xref: /freebsd/contrib/llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp (revision b3edf4467982447620505a28fc82e38a414c07dc) 
1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that X86 uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "X86ISelLowering.h"
15 #include "MCTargetDesc/X86ShuffleDecode.h"
16 #include "X86.h"
17 #include "X86CallingConv.h"
18 #include "X86FrameLowering.h"
19 #include "X86InstrBuilder.h"
20 #include "X86IntrinsicsInfo.h"
21 #include "X86MachineFunctionInfo.h"
22 #include "X86TargetMachine.h"
23 #include "X86TargetObjectFile.h"
24 #include "llvm/ADT/SmallBitVector.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/Statistic.h"
27 #include "llvm/ADT/StringExtras.h"
28 #include "llvm/ADT/StringSwitch.h"
29 #include "llvm/Analysis/BlockFrequencyInfo.h"
30 #include "llvm/Analysis/ObjCARCUtil.h"
31 #include "llvm/Analysis/ProfileSummaryInfo.h"
32 #include "llvm/Analysis/VectorUtils.h"
33 #include "llvm/CodeGen/IntrinsicLowering.h"
34 #include "llvm/CodeGen/MachineFrameInfo.h"
35 #include "llvm/CodeGen/MachineFunction.h"
36 #include "llvm/CodeGen/MachineInstrBuilder.h"
37 #include "llvm/CodeGen/MachineJumpTableInfo.h"
38 #include "llvm/CodeGen/MachineLoopInfo.h"
39 #include "llvm/CodeGen/MachineModuleInfo.h"
40 #include "llvm/CodeGen/MachineRegisterInfo.h"
41 #include "llvm/CodeGen/TargetLowering.h"
42 #include "llvm/CodeGen/WinEHFuncInfo.h"
43 #include "llvm/IR/CallingConv.h"
44 #include "llvm/IR/Constants.h"
45 #include "llvm/IR/DerivedTypes.h"
46 #include "llvm/IR/EHPersonalities.h"
47 #include "llvm/IR/Function.h"
48 #include "llvm/IR/GlobalAlias.h"
49 #include "llvm/IR/GlobalVariable.h"
50 #include "llvm/IR/IRBuilder.h"
51 #include "llvm/IR/Instructions.h"
52 #include "llvm/IR/Intrinsics.h"
53 #include "llvm/IR/PatternMatch.h"
54 #include "llvm/MC/MCAsmInfo.h"
55 #include "llvm/MC/MCContext.h"
56 #include "llvm/MC/MCExpr.h"
57 #include "llvm/MC/MCSymbol.h"
58 #include "llvm/Support/CommandLine.h"
59 #include "llvm/Support/Debug.h"
60 #include "llvm/Support/ErrorHandling.h"
61 #include "llvm/Support/KnownBits.h"
62 #include "llvm/Support/MathExtras.h"
63 #include "llvm/Target/TargetOptions.h"
64 #include <algorithm>
65 #include <bitset>
66 #include <cctype>
67 #include <numeric>
68 using namespace llvm;
69 
70 #define DEBUG_TYPE "x86-isel"
71 
72 static cl::opt<int> ExperimentalPrefInnermostLoopAlignment(
73     "x86-experimental-pref-innermost-loop-alignment", cl::init(4),
74     cl::desc(
75         "Sets the preferable loop alignment for experiments (as log2 bytes) "
76         "for innermost loops only. If specified, this option overrides "
77         "alignment set by x86-experimental-pref-loop-alignment."),
78     cl::Hidden);
79 
80 static cl::opt<bool> MulConstantOptimization(
81     "mul-constant-optimization", cl::init(true),
82     cl::desc("Replace 'mul x, Const' with more effective instructions like "
83              "SHIFT, LEA, etc."),
84     cl::Hidden);
85 
86 X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
87                                      const X86Subtarget &STI)
88     : TargetLowering(TM), Subtarget(STI) {
89   bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
90   MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
91 
92   // Set up the TargetLowering object.
93 
94   // X86 is weird. It always uses i8 for shift amounts and setcc results.
95   setBooleanContents(ZeroOrOneBooleanContent);
96   // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
97   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
98 
99   // For 64-bit, since we have so many registers, use the ILP scheduler.
100   // For 32-bit, use the register pressure specific scheduling.
101   // For Atom, always use ILP scheduling.
102   if (Subtarget.isAtom())
103     setSchedulingPreference(Sched::ILP);
104   else if (Subtarget.is64Bit())
105     setSchedulingPreference(Sched::ILP);
106   else
107     setSchedulingPreference(Sched::RegPressure);
108   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
109   setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
110 
111   // Bypass expensive divides and use cheaper ones.
112   if (TM.getOptLevel() >= CodeGenOptLevel::Default) {
113     if (Subtarget.hasSlowDivide32())
114       addBypassSlowDiv(32, 8);
115     if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
116       addBypassSlowDiv(64, 32);
117   }
118 
119   // Setup Windows compiler runtime calls.
120   if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) {
121     static const struct {
122       const RTLIB::Libcall Op;
123       const char * const Name;
124       const CallingConv::ID CC;
125     } LibraryCalls[] = {
126       { RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall },
127       { RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall },
128       { RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall },
129       { RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall },
130       { RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall },
131     };
132 
133     for (const auto &LC : LibraryCalls) {
134       setLibcallName(LC.Op, LC.Name);
135       setLibcallCallingConv(LC.Op, LC.CC);
136     }
137   }
138 
139   if (Subtarget.getTargetTriple().isOSMSVCRT()) {
140     // MSVCRT doesn't have powi; fall back to pow
141     setLibcallName(RTLIB::POWI_F32, nullptr);
142     setLibcallName(RTLIB::POWI_F64, nullptr);
143   }
144 
145   if (Subtarget.canUseCMPXCHG16B())
146     setMaxAtomicSizeInBitsSupported(128);
147   else if (Subtarget.canUseCMPXCHG8B())
148     setMaxAtomicSizeInBitsSupported(64);
149   else
150     setMaxAtomicSizeInBitsSupported(32);
151 
152   setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
153 
154   setMaxLargeFPConvertBitWidthSupported(128);
155 
156   // Set up the register classes.
157   addRegisterClass(MVT::i8, &X86::GR8RegClass);
158   addRegisterClass(MVT::i16, &X86::GR16RegClass);
159   addRegisterClass(MVT::i32, &X86::GR32RegClass);
160   if (Subtarget.is64Bit())
161     addRegisterClass(MVT::i64, &X86::GR64RegClass);
162 
163   for (MVT VT : MVT::integer_valuetypes())
164     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
165 
166   // We don't accept any truncstore of integer registers.
167   setTruncStoreAction(MVT::i64, MVT::i32, Expand);
168   setTruncStoreAction(MVT::i64, MVT::i16, Expand);
169   setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
170   setTruncStoreAction(MVT::i32, MVT::i16, Expand);
171   setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
172   setTruncStoreAction(MVT::i16, MVT::i8,  Expand);
173 
174   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
175 
176   // SETOEQ and SETUNE require checking two conditions.
177   for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) {
178     setCondCodeAction(ISD::SETOEQ, VT, Expand);
179     setCondCodeAction(ISD::SETUNE, VT, Expand);
180   }
181 
182   // Integer absolute.
183   if (Subtarget.canUseCMOV()) {
184     setOperationAction(ISD::ABS            , MVT::i16  , Custom);
185     setOperationAction(ISD::ABS            , MVT::i32  , Custom);
186     if (Subtarget.is64Bit())
187       setOperationAction(ISD::ABS          , MVT::i64  , Custom);
188   }
189 
190   // Absolute difference.
191   for (auto Op : {ISD::ABDS, ISD::ABDU}) {
192     setOperationAction(Op                  , MVT::i8   , Custom);
193     setOperationAction(Op                  , MVT::i16  , Custom);
194     setOperationAction(Op                  , MVT::i32  , Custom);
195     if (Subtarget.is64Bit())
196      setOperationAction(Op                 , MVT::i64  , Custom);
197   }
198 
199   // Signed saturation subtraction.
200   setOperationAction(ISD::SSUBSAT          , MVT::i8   , Custom);
201   setOperationAction(ISD::SSUBSAT          , MVT::i16  , Custom);
202   setOperationAction(ISD::SSUBSAT          , MVT::i32  , Custom);
203   if (Subtarget.is64Bit())
204     setOperationAction(ISD::SSUBSAT        , MVT::i64  , Custom);
205 
206   // Funnel shifts.
207   for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
208     // For slow shld targets we only lower for code size.
209     LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
210 
211     setOperationAction(ShiftOp             , MVT::i8   , Custom);
212     setOperationAction(ShiftOp             , MVT::i16  , Custom);
213     setOperationAction(ShiftOp             , MVT::i32  , ShiftDoubleAction);
214     if (Subtarget.is64Bit())
215       setOperationAction(ShiftOp           , MVT::i64  , ShiftDoubleAction);
216   }
217 
218   if (!Subtarget.useSoftFloat()) {
219     // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
220     // operation.
221     setOperationAction(ISD::UINT_TO_FP,        MVT::i8, Promote);
222     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
223     setOperationAction(ISD::UINT_TO_FP,        MVT::i16, Promote);
224     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
225     // We have an algorithm for SSE2, and we turn this into a 64-bit
226     // FILD or VCVTUSI2SS/SD for other targets.
227     setOperationAction(ISD::UINT_TO_FP,        MVT::i32, Custom);
228     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
229     // We have an algorithm for SSE2->double, and we turn this into a
230     // 64-bit FILD followed by conditional FADD for other targets.
231     setOperationAction(ISD::UINT_TO_FP,        MVT::i64, Custom);
232     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
233 
234     // Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
235     // this operation.
236     setOperationAction(ISD::SINT_TO_FP,        MVT::i8, Promote);
237     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
238     // SSE has no i16 to fp conversion, only i32. We promote in the handler
239     // to allow f80 to use i16 and f64 to use i16 with sse1 only
240     setOperationAction(ISD::SINT_TO_FP,        MVT::i16, Custom);
241     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
242     // f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
243     setOperationAction(ISD::SINT_TO_FP,        MVT::i32, Custom);
244     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
245     // In 32-bit mode these are custom lowered.  In 64-bit mode F32 and F64
246     // are Legal, f80 is custom lowered.
247     setOperationAction(ISD::SINT_TO_FP,        MVT::i64, Custom);
248     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
249 
250     // Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
251     // this operation.
252     setOperationAction(ISD::FP_TO_SINT,        MVT::i8,  Promote);
253     // FIXME: This doesn't generate invalid exception when it should. PR44019.
254     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8,  Promote);
255     setOperationAction(ISD::FP_TO_SINT,        MVT::i16, Custom);
256     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
257     setOperationAction(ISD::FP_TO_SINT,        MVT::i32, Custom);
258     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
259     // In 32-bit mode these are custom lowered.  In 64-bit mode F32 and F64
260     // are Legal, f80 is custom lowered.
261     setOperationAction(ISD::FP_TO_SINT,        MVT::i64, Custom);
262     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
263 
264     // Handle FP_TO_UINT by promoting the destination to a larger signed
265     // conversion.
266     setOperationAction(ISD::FP_TO_UINT,        MVT::i8,  Promote);
267     // FIXME: This doesn't generate invalid exception when it should. PR44019.
268     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8,  Promote);
269     setOperationAction(ISD::FP_TO_UINT,        MVT::i16, Promote);
270     // FIXME: This doesn't generate invalid exception when it should. PR44019.
271     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
272     setOperationAction(ISD::FP_TO_UINT,        MVT::i32, Custom);
273     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
274     setOperationAction(ISD::FP_TO_UINT,        MVT::i64, Custom);
275     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
276 
277     setOperationAction(ISD::LRINT,             MVT::f32, Custom);
278     setOperationAction(ISD::LRINT,             MVT::f64, Custom);
279     setOperationAction(ISD::LLRINT,            MVT::f32, Custom);
280     setOperationAction(ISD::LLRINT,            MVT::f64, Custom);
281 
282     if (!Subtarget.is64Bit()) {
283       setOperationAction(ISD::LRINT,  MVT::i64, Custom);
284       setOperationAction(ISD::LLRINT, MVT::i64, Custom);
285     }
286   }
287 
288   if (Subtarget.hasSSE2()) {
289     // Custom lowering for saturating float to int conversions.
290     // We handle promotion to larger result types manually.
291     for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {
292       setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
293       setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
294     }
295     if (Subtarget.is64Bit()) {
296       setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
297       setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
298     }
299   }
300 
301   // Handle address space casts between mixed sized pointers.
302   setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
303   setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
304 
305   // TODO: when we have SSE, these could be more efficient, by using movd/movq.
306   if (!Subtarget.hasSSE2()) {
307     setOperationAction(ISD::BITCAST        , MVT::f32  , Expand);
308     setOperationAction(ISD::BITCAST        , MVT::i32  , Expand);
309     if (Subtarget.is64Bit()) {
310       setOperationAction(ISD::BITCAST      , MVT::f64  , Expand);
311       // Without SSE, i64->f64 goes through memory.
312       setOperationAction(ISD::BITCAST      , MVT::i64  , Expand);
313     }
314   } else if (!Subtarget.is64Bit())
315     setOperationAction(ISD::BITCAST      , MVT::i64  , Custom);
316 
317   // Scalar integer divide and remainder are lowered to use operations that
318   // produce two results, to match the available instructions. This exposes
319   // the two-result form to trivial CSE, which is able to combine x/y and x%y
320   // into a single instruction.
321   //
322   // Scalar integer multiply-high is also lowered to use two-result
323   // operations, to match the available instructions. However, plain multiply
324   // (low) operations are left as Legal, as there are single-result
325   // instructions for this in x86. Using the two-result multiply instructions
326   // when both high and low results are needed must be arranged by dagcombine.
327   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
328     setOperationAction(ISD::MULHS, VT, Expand);
329     setOperationAction(ISD::MULHU, VT, Expand);
330     setOperationAction(ISD::SDIV, VT, Expand);
331     setOperationAction(ISD::UDIV, VT, Expand);
332     setOperationAction(ISD::SREM, VT, Expand);
333     setOperationAction(ISD::UREM, VT, Expand);
334   }
335 
336   setOperationAction(ISD::BR_JT            , MVT::Other, Expand);
337   setOperationAction(ISD::BRCOND           , MVT::Other, Custom);
338   for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
339                    MVT::i8,  MVT::i16, MVT::i32, MVT::i64 }) {
340     setOperationAction(ISD::BR_CC,     VT, Expand);
341     setOperationAction(ISD::SELECT_CC, VT, Expand);
342   }
343   if (Subtarget.is64Bit())
344     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
345   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16  , Legal);
346   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8   , Legal);
347   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1   , Expand);
348 
349   setOperationAction(ISD::FREM             , MVT::f32  , Expand);
350   setOperationAction(ISD::FREM             , MVT::f64  , Expand);
351   setOperationAction(ISD::FREM             , MVT::f80  , Expand);
352   setOperationAction(ISD::FREM             , MVT::f128 , Expand);
353 
354   if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
355     setOperationAction(ISD::GET_ROUNDING   , MVT::i32  , Custom);
356     setOperationAction(ISD::SET_ROUNDING   , MVT::Other, Custom);
357     setOperationAction(ISD::GET_FPENV_MEM  , MVT::Other, Custom);
358     setOperationAction(ISD::SET_FPENV_MEM  , MVT::Other, Custom);
359     setOperationAction(ISD::RESET_FPENV    , MVT::Other, Custom);
360   }
361 
362   // Promote the i8 variants and force them on up to i32 which has a shorter
363   // encoding.
364   setOperationPromotedToType(ISD::CTTZ           , MVT::i8   , MVT::i32);
365   setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8   , MVT::i32);
366   // Promoted i16. tzcntw has a false dependency on Intel CPUs. For BSF, we emit
367   // a REP prefix to encode it as TZCNT for modern CPUs so it makes sense to
368   // promote that too.
369   setOperationPromotedToType(ISD::CTTZ           , MVT::i16  , MVT::i32);
370   setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i16  , MVT::i32);
371 
372   if (!Subtarget.hasBMI()) {
373     setOperationAction(ISD::CTTZ           , MVT::i32  , Custom);
374     setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32  , Legal);
375     if (Subtarget.is64Bit()) {
376       setOperationAction(ISD::CTTZ         , MVT::i64  , Custom);
377       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal);
378     }
379   }
380 
381   if (Subtarget.hasLZCNT()) {
382     // When promoting the i8 variants, force them to i32 for a shorter
383     // encoding.
384     setOperationPromotedToType(ISD::CTLZ           , MVT::i8   , MVT::i32);
385     setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8   , MVT::i32);
386   } else {
387     for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
388       if (VT == MVT::i64 && !Subtarget.is64Bit())
389         continue;
390       setOperationAction(ISD::CTLZ           , VT, Custom);
391       setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
392     }
393   }
394 
395   for (auto Op : {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP, ISD::FP_TO_FP16,
396                   ISD::STRICT_FP_TO_FP16}) {
397     // Special handling for half-precision floating point conversions.
398     // If we don't have F16C support, then lower half float conversions
399     // into library calls.
400     setOperationAction(
401         Op, MVT::f32,
402         (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) ? Custom : Expand);
403     // There's never any support for operations beyond MVT::f32.
404     setOperationAction(Op, MVT::f64, Expand);
405     setOperationAction(Op, MVT::f80, Expand);
406     setOperationAction(Op, MVT::f128, Expand);
407   }
408 
409   for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
410     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
411     setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
412     setTruncStoreAction(VT, MVT::f16, Expand);
413     setTruncStoreAction(VT, MVT::bf16, Expand);
414 
415     setOperationAction(ISD::BF16_TO_FP, VT, Expand);
416     setOperationAction(ISD::FP_TO_BF16, VT, Custom);
417   }
418 
419   setOperationAction(ISD::PARITY, MVT::i8, Custom);
420   setOperationAction(ISD::PARITY, MVT::i16, Custom);
421   setOperationAction(ISD::PARITY, MVT::i32, Custom);
422   if (Subtarget.is64Bit())
423     setOperationAction(ISD::PARITY, MVT::i64, Custom);
424   if (Subtarget.hasPOPCNT()) {
425     setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);
426     // popcntw is longer to encode than popcntl and also has a false dependency
427     // on the dest that popcntl hasn't had since Cannon Lake.
428     setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);
429   } else {
430     setOperationAction(ISD::CTPOP          , MVT::i8   , Expand);
431     setOperationAction(ISD::CTPOP          , MVT::i16  , Expand);
432     setOperationAction(ISD::CTPOP          , MVT::i32  , Expand);
433     if (Subtarget.is64Bit())
434       setOperationAction(ISD::CTPOP        , MVT::i64  , Expand);
435     else
436       setOperationAction(ISD::CTPOP        , MVT::i64  , Custom);
437   }
438 
439   setOperationAction(ISD::READCYCLECOUNTER , MVT::i64  , Custom);
440 
441   if (!Subtarget.hasMOVBE())
442     setOperationAction(ISD::BSWAP          , MVT::i16  , Expand);
443 
444   // X86 wants to expand cmov itself.
445   for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
446     setOperationAction(ISD::SELECT, VT, Custom);
447     setOperationAction(ISD::SETCC, VT, Custom);
448     setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
449     setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
450   }
451   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
452     if (VT == MVT::i64 && !Subtarget.is64Bit())
453       continue;
454     setOperationAction(ISD::SELECT, VT, Custom);
455     setOperationAction(ISD::SETCC,  VT, Custom);
456   }
457 
458   // Custom action for SELECT MMX and expand action for SELECT_CC MMX
459   setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
460   setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
461 
462   setOperationAction(ISD::EH_RETURN       , MVT::Other, Custom);
463   // NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
464   // LLVM/Clang supports zero-cost DWARF and SEH exception handling.
465   setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
466   setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
467   setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
468   if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
469     setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
470 
471   // Darwin ABI issue.
472   for (auto VT : { MVT::i32, MVT::i64 }) {
473     if (VT == MVT::i64 && !Subtarget.is64Bit())
474       continue;
475     setOperationAction(ISD::ConstantPool    , VT, Custom);
476     setOperationAction(ISD::JumpTable       , VT, Custom);
477     setOperationAction(ISD::GlobalAddress   , VT, Custom);
478     setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
479     setOperationAction(ISD::ExternalSymbol  , VT, Custom);
480     setOperationAction(ISD::BlockAddress    , VT, Custom);
481   }
482 
483   // 64-bit shl, sra, srl (iff 32-bit x86)
484   for (auto VT : { MVT::i32, MVT::i64 }) {
485     if (VT == MVT::i64 && !Subtarget.is64Bit())
486       continue;
487     setOperationAction(ISD::SHL_PARTS, VT, Custom);
488     setOperationAction(ISD::SRA_PARTS, VT, Custom);
489     setOperationAction(ISD::SRL_PARTS, VT, Custom);
490   }
491 
492   if (Subtarget.hasSSEPrefetch() || Subtarget.hasThreeDNow())
493     setOperationAction(ISD::PREFETCH      , MVT::Other, Custom);
494 
495   setOperationAction(ISD::ATOMIC_FENCE  , MVT::Other, Custom);
496 
497   // Expand certain atomics
498   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
499     setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
500     setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
501     setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
502     setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);
503     setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
504     setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
505     setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
506   }
507 
508   if (!Subtarget.is64Bit())
509     setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
510 
511   if (Subtarget.canUseCMPXCHG16B())
512     setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
513 
514   // FIXME - use subtarget debug flags
515   if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() &&
516       !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() &&
517       TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
518     setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
519   }
520 
521   setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
522   setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
523 
524   setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
525   setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
526 
527   setOperationAction(ISD::TRAP, MVT::Other, Legal);
528   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
529   if (Subtarget.isTargetPS())
530     setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);
531   else
532     setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
533 
534   // VASTART needs to be custom lowered to use the VarArgsFrameIndex
535   setOperationAction(ISD::VASTART           , MVT::Other, Custom);
536   setOperationAction(ISD::VAEND             , MVT::Other, Expand);
537   bool Is64Bit = Subtarget.is64Bit();
538   setOperationAction(ISD::VAARG,  MVT::Other, Is64Bit ? Custom : Expand);
539   setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
540 
541   setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
542   setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
543 
544   setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
545 
546   // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
547   setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
548   setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
549 
550   setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
551 
552   auto setF16Action = [&] (MVT VT, LegalizeAction Action) {
553     setOperationAction(ISD::FABS, VT, Action);
554     setOperationAction(ISD::FNEG, VT, Action);
555     setOperationAction(ISD::FCOPYSIGN, VT, Expand);
556     setOperationAction(ISD::FREM, VT, Action);
557     setOperationAction(ISD::FMA, VT, Action);
558     setOperationAction(ISD::FMINNUM, VT, Action);
559     setOperationAction(ISD::FMAXNUM, VT, Action);
560     setOperationAction(ISD::FMINIMUM, VT, Action);
561     setOperationAction(ISD::FMAXIMUM, VT, Action);
562     setOperationAction(ISD::FSIN, VT, Action);
563     setOperationAction(ISD::FCOS, VT, Action);
564     setOperationAction(ISD::FSINCOS, VT, Action);
565     setOperationAction(ISD::FSQRT, VT, Action);
566     setOperationAction(ISD::FPOW, VT, Action);
567     setOperationAction(ISD::FLOG, VT, Action);
568     setOperationAction(ISD::FLOG2, VT, Action);
569     setOperationAction(ISD::FLOG10, VT, Action);
570     setOperationAction(ISD::FEXP, VT, Action);
571     setOperationAction(ISD::FEXP2, VT, Action);
572     setOperationAction(ISD::FEXP10, VT, Action);
573     setOperationAction(ISD::FCEIL, VT, Action);
574     setOperationAction(ISD::FFLOOR, VT, Action);
575     setOperationAction(ISD::FNEARBYINT, VT, Action);
576     setOperationAction(ISD::FRINT, VT, Action);
577     setOperationAction(ISD::BR_CC, VT, Action);
578     setOperationAction(ISD::SETCC, VT, Action);
579     setOperationAction(ISD::SELECT, VT, Custom);
580     setOperationAction(ISD::SELECT_CC, VT, Action);
581     setOperationAction(ISD::FROUND, VT, Action);
582     setOperationAction(ISD::FROUNDEVEN, VT, Action);
583     setOperationAction(ISD::FTRUNC, VT, Action);
584     setOperationAction(ISD::FLDEXP, VT, Action);
585   };
586 
587   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
588     // f16, f32 and f64 use SSE.
589     // Set up the FP register classes.
590     addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass
591                                                      : &X86::FR16RegClass);
592     addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
593                                                      : &X86::FR32RegClass);
594     addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
595                                                      : &X86::FR64RegClass);
596 
597     // Disable f32->f64 extload as we can only generate this in one instruction
598     // under optsize. So its easier to pattern match (fpext (load)) for that
599     // case instead of needing to emit 2 instructions for extload in the
600     // non-optsize case.
601     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
602 
603     for (auto VT : { MVT::f32, MVT::f64 }) {
604       // Use ANDPD to simulate FABS.
605       setOperationAction(ISD::FABS, VT, Custom);
606 
607       // Use XORP to simulate FNEG.
608       setOperationAction(ISD::FNEG, VT, Custom);
609 
610       // Use ANDPD and ORPD to simulate FCOPYSIGN.
611       setOperationAction(ISD::FCOPYSIGN, VT, Custom);
612 
613       // These might be better off as horizontal vector ops.
614       setOperationAction(ISD::FADD, VT, Custom);
615       setOperationAction(ISD::FSUB, VT, Custom);
616 
617       // We don't support sin/cos/fmod
618       setOperationAction(ISD::FSIN   , VT, Expand);
619       setOperationAction(ISD::FCOS   , VT, Expand);
620       setOperationAction(ISD::FSINCOS, VT, Expand);
621     }
622 
623     // Half type will be promoted by default.
624     setF16Action(MVT::f16, Promote);
625     setOperationAction(ISD::FADD, MVT::f16, Promote);
626     setOperationAction(ISD::FSUB, MVT::f16, Promote);
627     setOperationAction(ISD::FMUL, MVT::f16, Promote);
628     setOperationAction(ISD::FDIV, MVT::f16, Promote);
629     setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
630     setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
631     setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
632 
633     setOperationAction(ISD::STRICT_FADD, MVT::f16, Promote);
634     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Promote);
635     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Promote);
636     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Promote);
637     setOperationAction(ISD::STRICT_FMA, MVT::f16, Promote);
638     setOperationAction(ISD::STRICT_FMINNUM, MVT::f16, Promote);
639     setOperationAction(ISD::STRICT_FMAXNUM, MVT::f16, Promote);
640     setOperationAction(ISD::STRICT_FMINIMUM, MVT::f16, Promote);
641     setOperationAction(ISD::STRICT_FMAXIMUM, MVT::f16, Promote);
642     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Promote);
643     setOperationAction(ISD::STRICT_FPOW, MVT::f16, Promote);
644     setOperationAction(ISD::STRICT_FLDEXP, MVT::f16, Promote);
645     setOperationAction(ISD::STRICT_FLOG, MVT::f16, Promote);
646     setOperationAction(ISD::STRICT_FLOG2, MVT::f16, Promote);
647     setOperationAction(ISD::STRICT_FLOG10, MVT::f16, Promote);
648     setOperationAction(ISD::STRICT_FEXP, MVT::f16, Promote);
649     setOperationAction(ISD::STRICT_FEXP2, MVT::f16, Promote);
650     setOperationAction(ISD::STRICT_FCEIL, MVT::f16, Promote);
651     setOperationAction(ISD::STRICT_FFLOOR, MVT::f16, Promote);
652     setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f16, Promote);
653     setOperationAction(ISD::STRICT_FRINT, MVT::f16, Promote);
654     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Promote);
655     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Promote);
656     setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
657     setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Promote);
658     setOperationAction(ISD::STRICT_FTRUNC, MVT::f16, Promote);
659     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
660     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
661     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
662 
663     setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
664     setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
665 
666     // Lower this to MOVMSK plus an AND.
667     setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
668     setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
669 
670   } else if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1() &&
671              (UseX87 || Is64Bit)) {
672     // Use SSE for f32, x87 for f64.
673     // Set up the FP register classes.
674     addRegisterClass(MVT::f32, &X86::FR32RegClass);
675     if (UseX87)
676       addRegisterClass(MVT::f64, &X86::RFP64RegClass);
677 
678     // Use ANDPS to simulate FABS.
679     setOperationAction(ISD::FABS , MVT::f32, Custom);
680 
681     // Use XORP to simulate FNEG.
682     setOperationAction(ISD::FNEG , MVT::f32, Custom);
683 
684     if (UseX87)
685       setOperationAction(ISD::UNDEF, MVT::f64, Expand);
686 
687     // Use ANDPS and ORPS to simulate FCOPYSIGN.
688     if (UseX87)
689       setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
690     setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
691 
692     // We don't support sin/cos/fmod
693     setOperationAction(ISD::FSIN   , MVT::f32, Expand);
694     setOperationAction(ISD::FCOS   , MVT::f32, Expand);
695     setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
696 
697     if (UseX87) {
698       // Always expand sin/cos functions even though x87 has an instruction.
699       setOperationAction(ISD::FSIN, MVT::f64, Expand);
700       setOperationAction(ISD::FCOS, MVT::f64, Expand);
701       setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
702     }
703   } else if (UseX87) {
704     // f32 and f64 in x87.
705     // Set up the FP register classes.
706     addRegisterClass(MVT::f64, &X86::RFP64RegClass);
707     addRegisterClass(MVT::f32, &X86::RFP32RegClass);
708 
709     for (auto VT : { MVT::f32, MVT::f64 }) {
710       setOperationAction(ISD::UNDEF,     VT, Expand);
711       setOperationAction(ISD::FCOPYSIGN, VT, Expand);
712 
713       // Always expand sin/cos functions even though x87 has an instruction.
714       setOperationAction(ISD::FSIN   , VT, Expand);
715       setOperationAction(ISD::FCOS   , VT, Expand);
716       setOperationAction(ISD::FSINCOS, VT, Expand);
717     }
718   }
719 
720   // Expand FP32 immediates into loads from the stack, save special cases.
721   if (isTypeLegal(MVT::f32)) {
722     if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
723       addLegalFPImmediate(APFloat(+0.0f)); // FLD0
724       addLegalFPImmediate(APFloat(+1.0f)); // FLD1
725       addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
726       addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
727     } else // SSE immediates.
728       addLegalFPImmediate(APFloat(+0.0f)); // xorps
729   }
730   // Expand FP64 immediates into loads from the stack, save special cases.
731   if (isTypeLegal(MVT::f64)) {
732     if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
733       addLegalFPImmediate(APFloat(+0.0)); // FLD0
734       addLegalFPImmediate(APFloat(+1.0)); // FLD1
735       addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
736       addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
737     } else // SSE immediates.
738       addLegalFPImmediate(APFloat(+0.0)); // xorpd
739   }
740   // Support fp16 0 immediate.
741   if (isTypeLegal(MVT::f16))
742     addLegalFPImmediate(APFloat::getZero(APFloat::IEEEhalf()));
743 
744   // Handle constrained floating-point operations of scalar.
745   setOperationAction(ISD::STRICT_FADD,      MVT::f32, Legal);
746   setOperationAction(ISD::STRICT_FADD,      MVT::f64, Legal);
747   setOperationAction(ISD::STRICT_FSUB,      MVT::f32, Legal);
748   setOperationAction(ISD::STRICT_FSUB,      MVT::f64, Legal);
749   setOperationAction(ISD::STRICT_FMUL,      MVT::f32, Legal);
750   setOperationAction(ISD::STRICT_FMUL,      MVT::f64, Legal);
751   setOperationAction(ISD::STRICT_FDIV,      MVT::f32, Legal);
752   setOperationAction(ISD::STRICT_FDIV,      MVT::f64, Legal);
753   setOperationAction(ISD::STRICT_FP_ROUND,  MVT::f32, Legal);
754   setOperationAction(ISD::STRICT_FP_ROUND,  MVT::f64, Legal);
755   setOperationAction(ISD::STRICT_FSQRT,     MVT::f32, Legal);
756   setOperationAction(ISD::STRICT_FSQRT,     MVT::f64, Legal);
757 
758   // We don't support FMA.
759   setOperationAction(ISD::FMA, MVT::f64, Expand);
760   setOperationAction(ISD::FMA, MVT::f32, Expand);
761 
762   // f80 always uses X87.
763   if (UseX87) {
764     addRegisterClass(MVT::f80, &X86::RFP80RegClass);
765     setOperationAction(ISD::UNDEF,     MVT::f80, Expand);
766     setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
767     {
768       APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
769       addLegalFPImmediate(TmpFlt);  // FLD0
770       TmpFlt.changeSign();
771       addLegalFPImmediate(TmpFlt);  // FLD0/FCHS
772 
773       bool ignored;
774       APFloat TmpFlt2(+1.0);
775       TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
776                       &ignored);
777       addLegalFPImmediate(TmpFlt2);  // FLD1
778       TmpFlt2.changeSign();
779       addLegalFPImmediate(TmpFlt2);  // FLD1/FCHS
780     }
781 
782     // Always expand sin/cos functions even though x87 has an instruction.
783     setOperationAction(ISD::FSIN   , MVT::f80, Expand);
784     setOperationAction(ISD::FCOS   , MVT::f80, Expand);
785     setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
786 
787     setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
788     setOperationAction(ISD::FCEIL,  MVT::f80, Expand);
789     setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
790     setOperationAction(ISD::FRINT,  MVT::f80, Expand);
791     setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
792     setOperationAction(ISD::FROUNDEVEN, MVT::f80, Expand);
793     setOperationAction(ISD::FMA, MVT::f80, Expand);
794     setOperationAction(ISD::LROUND, MVT::f80, Expand);
795     setOperationAction(ISD::LLROUND, MVT::f80, Expand);
796     setOperationAction(ISD::LRINT, MVT::f80, Custom);
797     setOperationAction(ISD::LLRINT, MVT::f80, Custom);
798 
799     // Handle constrained floating-point operations of scalar.
800     setOperationAction(ISD::STRICT_FADD     , MVT::f80, Legal);
801     setOperationAction(ISD::STRICT_FSUB     , MVT::f80, Legal);
802     setOperationAction(ISD::STRICT_FMUL     , MVT::f80, Legal);
803     setOperationAction(ISD::STRICT_FDIV     , MVT::f80, Legal);
804     setOperationAction(ISD::STRICT_FSQRT    , MVT::f80, Legal);
805     if (isTypeLegal(MVT::f16)) {
806       setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
807       setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Custom);
808     } else {
809       setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Legal);
810     }
811     // FIXME: When the target is 64-bit, STRICT_FP_ROUND will be overwritten
812     // as Custom.
813     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Legal);
814   }
815 
816   // f128 uses xmm registers, but most operations require libcalls.
817   if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.hasSSE1()) {
818     addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
819                                                    : &X86::VR128RegClass);
820 
821     addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
822 
823     setOperationAction(ISD::FADD,        MVT::f128, LibCall);
824     setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
825     setOperationAction(ISD::FSUB,        MVT::f128, LibCall);
826     setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
827     setOperationAction(ISD::FDIV,        MVT::f128, LibCall);
828     setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
829     setOperationAction(ISD::FMUL,        MVT::f128, LibCall);
830     setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
831     setOperationAction(ISD::FMA,         MVT::f128, LibCall);
832     setOperationAction(ISD::STRICT_FMA,  MVT::f128, LibCall);
833 
834     setOperationAction(ISD::FABS, MVT::f128, Custom);
835     setOperationAction(ISD::FNEG, MVT::f128, Custom);
836     setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
837 
838     setOperationAction(ISD::FSIN,         MVT::f128, LibCall);
839     setOperationAction(ISD::STRICT_FSIN,  MVT::f128, LibCall);
840     setOperationAction(ISD::FCOS,         MVT::f128, LibCall);
841     setOperationAction(ISD::STRICT_FCOS,  MVT::f128, LibCall);
842     setOperationAction(ISD::FSINCOS,      MVT::f128, LibCall);
843     // No STRICT_FSINCOS
844     setOperationAction(ISD::FSQRT,        MVT::f128, LibCall);
845     setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
846 
847     setOperationAction(ISD::FP_EXTEND,        MVT::f128, Custom);
848     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
849     // We need to custom handle any FP_ROUND with an f128 input, but
850     // LegalizeDAG uses the result type to know when to run a custom handler.
851     // So we have to list all legal floating point result types here.
852     if (isTypeLegal(MVT::f32)) {
853       setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
854       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
855     }
856     if (isTypeLegal(MVT::f64)) {
857       setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
858       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
859     }
860     if (isTypeLegal(MVT::f80)) {
861       setOperationAction(ISD::FP_ROUND, MVT::f80, Custom);
862       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Custom);
863     }
864 
865     setOperationAction(ISD::SETCC, MVT::f128, Custom);
866 
867     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
868     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
869     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f80, Expand);
870     setTruncStoreAction(MVT::f128, MVT::f32, Expand);
871     setTruncStoreAction(MVT::f128, MVT::f64, Expand);
872     setTruncStoreAction(MVT::f128, MVT::f80, Expand);
873   }
874 
875   // Always use a library call for pow.
876   setOperationAction(ISD::FPOW             , MVT::f32  , Expand);
877   setOperationAction(ISD::FPOW             , MVT::f64  , Expand);
878   setOperationAction(ISD::FPOW             , MVT::f80  , Expand);
879   setOperationAction(ISD::FPOW             , MVT::f128 , Expand);
880 
881   setOperationAction(ISD::FLOG, MVT::f80, Expand);
882   setOperationAction(ISD::FLOG2, MVT::f80, Expand);
883   setOperationAction(ISD::FLOG10, MVT::f80, Expand);
884   setOperationAction(ISD::FEXP, MVT::f80, Expand);
885   setOperationAction(ISD::FEXP2, MVT::f80, Expand);
886   setOperationAction(ISD::FEXP10, MVT::f80, Expand);
887   setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
888   setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
889 
890   // Some FP actions are always expanded for vector types.
891   for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
892                    MVT::v4f32, MVT::v8f32,  MVT::v16f32,
893                    MVT::v2f64, MVT::v4f64,  MVT::v8f64 }) {
894     setOperationAction(ISD::FSIN,      VT, Expand);
895     setOperationAction(ISD::FSINCOS,   VT, Expand);
896     setOperationAction(ISD::FCOS,      VT, Expand);
897     setOperationAction(ISD::FREM,      VT, Expand);
898     setOperationAction(ISD::FCOPYSIGN, VT, Expand);
899     setOperationAction(ISD::FPOW,      VT, Expand);
900     setOperationAction(ISD::FLOG,      VT, Expand);
901     setOperationAction(ISD::FLOG2,     VT, Expand);
902     setOperationAction(ISD::FLOG10,    VT, Expand);
903     setOperationAction(ISD::FEXP,      VT, Expand);
904     setOperationAction(ISD::FEXP2,     VT, Expand);
905     setOperationAction(ISD::FEXP10,    VT, Expand);
906   }
907 
908   // First set operation action for all vector types to either promote
909   // (for widening) or expand (for scalarization). Then we will selectively
910   // turn on ones that can be effectively codegen'd.
911   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
912     setOperationAction(ISD::SDIV, VT, Expand);
913     setOperationAction(ISD::UDIV, VT, Expand);
914     setOperationAction(ISD::SREM, VT, Expand);
915     setOperationAction(ISD::UREM, VT, Expand);
916     setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
917     setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
918     setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
919     setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
920     setOperationAction(ISD::FMA,  VT, Expand);
921     setOperationAction(ISD::FFLOOR, VT, Expand);
922     setOperationAction(ISD::FCEIL, VT, Expand);
923     setOperationAction(ISD::FTRUNC, VT, Expand);
924     setOperationAction(ISD::FRINT, VT, Expand);
925     setOperationAction(ISD::FNEARBYINT, VT, Expand);
926     setOperationAction(ISD::FROUNDEVEN, VT, Expand);
927     setOperationAction(ISD::SMUL_LOHI, VT, Expand);
928     setOperationAction(ISD::MULHS, VT, Expand);
929     setOperationAction(ISD::UMUL_LOHI, VT, Expand);
930     setOperationAction(ISD::MULHU, VT, Expand);
931     setOperationAction(ISD::SDIVREM, VT, Expand);
932     setOperationAction(ISD::UDIVREM, VT, Expand);
933     setOperationAction(ISD::CTPOP, VT, Expand);
934     setOperationAction(ISD::CTTZ, VT, Expand);
935     setOperationAction(ISD::CTLZ, VT, Expand);
936     setOperationAction(ISD::ROTL, VT, Expand);
937     setOperationAction(ISD::ROTR, VT, Expand);
938     setOperationAction(ISD::BSWAP, VT, Expand);
939     setOperationAction(ISD::SETCC, VT, Expand);
940     setOperationAction(ISD::FP_TO_UINT, VT, Expand);
941     setOperationAction(ISD::FP_TO_SINT, VT, Expand);
942     setOperationAction(ISD::UINT_TO_FP, VT, Expand);
943     setOperationAction(ISD::SINT_TO_FP, VT, Expand);
944     setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
945     setOperationAction(ISD::TRUNCATE, VT, Expand);
946     setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
947     setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
948     setOperationAction(ISD::ANY_EXTEND, VT, Expand);
949     setOperationAction(ISD::SELECT_CC, VT, Expand);
950     for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
951       setTruncStoreAction(InnerVT, VT, Expand);
952 
953       setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
954       setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
955 
956       // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
957       // types, we have to deal with them whether we ask for Expansion or not.
958       // Setting Expand causes its own optimisation problems though, so leave
959       // them legal.
960       if (VT.getVectorElementType() == MVT::i1)
961         setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
962 
963       // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
964       // split/scalarized right now.
965       if (VT.getVectorElementType() == MVT::f16 ||
966           VT.getVectorElementType() == MVT::bf16)
967         setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
968     }
969   }
970 
971   // FIXME: In order to prevent SSE instructions being expanded to MMX ones
972   // with -msoft-float, disable use of MMX as well.
973   if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) {
974     addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
975     // No operations on x86mmx supported, everything uses intrinsics.
976   }
977 
978   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) {
979     addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
980                                                     : &X86::VR128RegClass);
981 
982     setOperationAction(ISD::FMAXIMUM,           MVT::f32, Custom);
983     setOperationAction(ISD::FMINIMUM,           MVT::f32, Custom);
984 
985     setOperationAction(ISD::FNEG,               MVT::v4f32, Custom);
986     setOperationAction(ISD::FABS,               MVT::v4f32, Custom);
987     setOperationAction(ISD::FCOPYSIGN,          MVT::v4f32, Custom);
988     setOperationAction(ISD::BUILD_VECTOR,       MVT::v4f32, Custom);
989     setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4f32, Custom);
990     setOperationAction(ISD::VSELECT,            MVT::v4f32, Custom);
991     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
992     setOperationAction(ISD::SELECT,             MVT::v4f32, Custom);
993 
994     setOperationAction(ISD::LOAD,               MVT::v2f32, Custom);
995     setOperationAction(ISD::STORE,              MVT::v2f32, Custom);
996 
997     setOperationAction(ISD::STRICT_FADD,        MVT::v4f32, Legal);
998     setOperationAction(ISD::STRICT_FSUB,        MVT::v4f32, Legal);
999     setOperationAction(ISD::STRICT_FMUL,        MVT::v4f32, Legal);
1000     setOperationAction(ISD::STRICT_FDIV,        MVT::v4f32, Legal);
1001     setOperationAction(ISD::STRICT_FSQRT,       MVT::v4f32, Legal);
1002   }
1003 
1004   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
1005     addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1006                                                     : &X86::VR128RegClass);
1007 
1008     // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
1009     // registers cannot be used even for integer operations.
1010     addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
1011                                                     : &X86::VR128RegClass);
1012     addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1013                                                     : &X86::VR128RegClass);
1014     addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1015                                                     : &X86::VR128RegClass);
1016     addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1017                                                     : &X86::VR128RegClass);
1018     addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1019                                                     : &X86::VR128RegClass);
1020 
1021     for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 }) {
1022       setOperationAction(ISD::FMAXIMUM, VT, Custom);
1023       setOperationAction(ISD::FMINIMUM, VT, Custom);
1024     }
1025 
1026     for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,
1027                      MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {
1028       setOperationAction(ISD::SDIV, VT, Custom);
1029       setOperationAction(ISD::SREM, VT, Custom);
1030       setOperationAction(ISD::UDIV, VT, Custom);
1031       setOperationAction(ISD::UREM, VT, Custom);
1032     }
1033 
1034     setOperationAction(ISD::MUL,                MVT::v2i8,  Custom);
1035     setOperationAction(ISD::MUL,                MVT::v4i8,  Custom);
1036     setOperationAction(ISD::MUL,                MVT::v8i8,  Custom);
1037 
1038     setOperationAction(ISD::MUL,                MVT::v16i8, Custom);
1039     setOperationAction(ISD::MUL,                MVT::v4i32, Custom);
1040     setOperationAction(ISD::MUL,                MVT::v2i64, Custom);
1041     setOperationAction(ISD::MULHU,              MVT::v4i32, Custom);
1042     setOperationAction(ISD::MULHS,              MVT::v4i32, Custom);
1043     setOperationAction(ISD::MULHU,              MVT::v16i8, Custom);
1044     setOperationAction(ISD::MULHS,              MVT::v16i8, Custom);
1045     setOperationAction(ISD::MULHU,              MVT::v8i16, Legal);
1046     setOperationAction(ISD::MULHS,              MVT::v8i16, Legal);
1047     setOperationAction(ISD::MUL,                MVT::v8i16, Legal);
1048     setOperationAction(ISD::AVGCEILU,           MVT::v16i8, Legal);
1049     setOperationAction(ISD::AVGCEILU,           MVT::v8i16, Legal);
1050 
1051     setOperationAction(ISD::SMULO,              MVT::v16i8, Custom);
1052     setOperationAction(ISD::UMULO,              MVT::v16i8, Custom);
1053     setOperationAction(ISD::UMULO,              MVT::v2i32, Custom);
1054 
1055     setOperationAction(ISD::FNEG,               MVT::v2f64, Custom);
1056     setOperationAction(ISD::FABS,               MVT::v2f64, Custom);
1057     setOperationAction(ISD::FCOPYSIGN,          MVT::v2f64, Custom);
1058 
1059     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1060       setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
1061       setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
1062       setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
1063       setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
1064     }
1065 
1066     setOperationAction(ISD::ABDU,               MVT::v16i8, Custom);
1067     setOperationAction(ISD::ABDS,               MVT::v16i8, Custom);
1068     setOperationAction(ISD::ABDU,               MVT::v8i16, Custom);
1069     setOperationAction(ISD::ABDS,               MVT::v8i16, Custom);
1070     setOperationAction(ISD::ABDU,               MVT::v4i32, Custom);
1071     setOperationAction(ISD::ABDS,               MVT::v4i32, Custom);
1072 
1073     setOperationAction(ISD::UADDSAT,            MVT::v16i8, Legal);
1074     setOperationAction(ISD::SADDSAT,            MVT::v16i8, Legal);
1075     setOperationAction(ISD::USUBSAT,            MVT::v16i8, Legal);
1076     setOperationAction(ISD::SSUBSAT,            MVT::v16i8, Legal);
1077     setOperationAction(ISD::UADDSAT,            MVT::v8i16, Legal);
1078     setOperationAction(ISD::SADDSAT,            MVT::v8i16, Legal);
1079     setOperationAction(ISD::USUBSAT,            MVT::v8i16, Legal);
1080     setOperationAction(ISD::SSUBSAT,            MVT::v8i16, Legal);
1081     setOperationAction(ISD::USUBSAT,            MVT::v4i32, Custom);
1082     setOperationAction(ISD::USUBSAT,            MVT::v2i64, Custom);
1083 
1084     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v16i8, Custom);
1085     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);
1086     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i32, Custom);
1087     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f32, Custom);
1088 
1089     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1090       setOperationAction(ISD::SETCC,              VT, Custom);
1091       setOperationAction(ISD::CTPOP,              VT, Custom);
1092       setOperationAction(ISD::ABS,                VT, Custom);
1093 
1094       // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1095       // setcc all the way to isel and prefer SETGT in some isel patterns.
1096       setCondCodeAction(ISD::SETLT, VT, Custom);
1097       setCondCodeAction(ISD::SETLE, VT, Custom);
1098     }
1099 
1100     setOperationAction(ISD::SETCC,          MVT::v2f64, Custom);
1101     setOperationAction(ISD::SETCC,          MVT::v4f32, Custom);
1102     setOperationAction(ISD::STRICT_FSETCC,  MVT::v2f64, Custom);
1103     setOperationAction(ISD::STRICT_FSETCC,  MVT::v4f32, Custom);
1104     setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
1105     setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
1106 
1107     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1108       setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
1109       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
1110       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
1111       setOperationAction(ISD::VSELECT,            VT, Custom);
1112       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1113     }
1114 
1115     for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {
1116       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
1117       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
1118       setOperationAction(ISD::VSELECT,            VT, Custom);
1119 
1120       if (VT == MVT::v2i64 && !Subtarget.is64Bit())
1121         continue;
1122 
1123       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
1124       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1125     }
1126     setF16Action(MVT::v8f16, Expand);
1127     setOperationAction(ISD::FADD, MVT::v8f16, Expand);
1128     setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
1129     setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
1130     setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
1131     setOperationAction(ISD::FNEG, MVT::v8f16, Custom);
1132     setOperationAction(ISD::FABS, MVT::v8f16, Custom);
1133     setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Custom);
1134 
1135     // Custom lower v2i64 and v2f64 selects.
1136     setOperationAction(ISD::SELECT,             MVT::v2f64, Custom);
1137     setOperationAction(ISD::SELECT,             MVT::v2i64, Custom);
1138     setOperationAction(ISD::SELECT,             MVT::v4i32, Custom);
1139     setOperationAction(ISD::SELECT,             MVT::v8i16, Custom);
1140     setOperationAction(ISD::SELECT,             MVT::v8f16, Custom);
1141     setOperationAction(ISD::SELECT,             MVT::v16i8, Custom);
1142 
1143     setOperationAction(ISD::FP_TO_SINT,         MVT::v4i32, Custom);
1144     setOperationAction(ISD::FP_TO_UINT,         MVT::v4i32, Custom);
1145     setOperationAction(ISD::FP_TO_SINT,         MVT::v2i32, Custom);
1146     setOperationAction(ISD::FP_TO_UINT,         MVT::v2i32, Custom);
1147     setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v4i32, Custom);
1148     setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v2i32, Custom);
1149 
1150     // Custom legalize these to avoid over promotion or custom promotion.
1151     for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
1152       setOperationAction(ISD::FP_TO_SINT,        VT, Custom);
1153       setOperationAction(ISD::FP_TO_UINT,        VT, Custom);
1154       setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1155       setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1156     }
1157 
1158     setOperationAction(ISD::SINT_TO_FP,         MVT::v4i32, Custom);
1159     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v4i32, Custom);
1160     setOperationAction(ISD::SINT_TO_FP,         MVT::v2i32, Custom);
1161     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v2i32, Custom);
1162 
1163     setOperationAction(ISD::UINT_TO_FP,         MVT::v2i32, Custom);
1164     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v2i32, Custom);
1165 
1166     setOperationAction(ISD::UINT_TO_FP,         MVT::v4i32, Custom);
1167     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v4i32, Custom);
1168 
1169     // Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
1170     setOperationAction(ISD::SINT_TO_FP,         MVT::v2f32, Custom);
1171     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v2f32, Custom);
1172     setOperationAction(ISD::UINT_TO_FP,         MVT::v2f32, Custom);
1173     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v2f32, Custom);
1174 
1175     setOperationAction(ISD::FP_EXTEND,          MVT::v2f32, Custom);
1176     setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v2f32, Custom);
1177     setOperationAction(ISD::FP_ROUND,           MVT::v2f32, Custom);
1178     setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v2f32, Custom);
1179 
1180     // We want to legalize this to an f64 load rather than an i64 load on
1181     // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
1182     // store.
1183     setOperationAction(ISD::LOAD,               MVT::v2i32, Custom);
1184     setOperationAction(ISD::LOAD,               MVT::v4i16, Custom);
1185     setOperationAction(ISD::LOAD,               MVT::v8i8,  Custom);
1186     setOperationAction(ISD::STORE,              MVT::v2i32, Custom);
1187     setOperationAction(ISD::STORE,              MVT::v4i16, Custom);
1188     setOperationAction(ISD::STORE,              MVT::v8i8,  Custom);
1189 
1190     // Add 32-bit vector stores to help vectorization opportunities.
1191     setOperationAction(ISD::STORE,              MVT::v2i16, Custom);
1192     setOperationAction(ISD::STORE,              MVT::v4i8,  Custom);
1193 
1194     setOperationAction(ISD::BITCAST,            MVT::v2i32, Custom);
1195     setOperationAction(ISD::BITCAST,            MVT::v4i16, Custom);
1196     setOperationAction(ISD::BITCAST,            MVT::v8i8,  Custom);
1197     if (!Subtarget.hasAVX512())
1198       setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
1199 
1200     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
1201     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
1202     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
1203 
1204     setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
1205 
1206     setOperationAction(ISD::TRUNCATE,    MVT::v2i8,  Custom);
1207     setOperationAction(ISD::TRUNCATE,    MVT::v2i16, Custom);
1208     setOperationAction(ISD::TRUNCATE,    MVT::v2i32, Custom);
1209     setOperationAction(ISD::TRUNCATE,    MVT::v2i64, Custom);
1210     setOperationAction(ISD::TRUNCATE,    MVT::v4i8,  Custom);
1211     setOperationAction(ISD::TRUNCATE,    MVT::v4i16, Custom);
1212     setOperationAction(ISD::TRUNCATE,    MVT::v4i32, Custom);
1213     setOperationAction(ISD::TRUNCATE,    MVT::v4i64, Custom);
1214     setOperationAction(ISD::TRUNCATE,    MVT::v8i8,  Custom);
1215     setOperationAction(ISD::TRUNCATE,    MVT::v8i16, Custom);
1216     setOperationAction(ISD::TRUNCATE,    MVT::v8i32, Custom);
1217     setOperationAction(ISD::TRUNCATE,    MVT::v8i64, Custom);
1218     setOperationAction(ISD::TRUNCATE,    MVT::v16i8, Custom);
1219     setOperationAction(ISD::TRUNCATE,    MVT::v16i16, Custom);
1220     setOperationAction(ISD::TRUNCATE,    MVT::v16i32, Custom);
1221     setOperationAction(ISD::TRUNCATE,    MVT::v16i64, Custom);
1222 
1223     // In the customized shift lowering, the legal v4i32/v2i64 cases
1224     // in AVX2 will be recognized.
1225     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1226       setOperationAction(ISD::SRL,              VT, Custom);
1227       setOperationAction(ISD::SHL,              VT, Custom);
1228       setOperationAction(ISD::SRA,              VT, Custom);
1229       if (VT == MVT::v2i64) continue;
1230       setOperationAction(ISD::ROTL,             VT, Custom);
1231       setOperationAction(ISD::ROTR,             VT, Custom);
1232       setOperationAction(ISD::FSHL,             VT, Custom);
1233       setOperationAction(ISD::FSHR,             VT, Custom);
1234     }
1235 
1236     setOperationAction(ISD::STRICT_FSQRT,       MVT::v2f64, Legal);
1237     setOperationAction(ISD::STRICT_FADD,        MVT::v2f64, Legal);
1238     setOperationAction(ISD::STRICT_FSUB,        MVT::v2f64, Legal);
1239     setOperationAction(ISD::STRICT_FMUL,        MVT::v2f64, Legal);
1240     setOperationAction(ISD::STRICT_FDIV,        MVT::v2f64, Legal);
1241   }
1242 
1243   if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
1244     setOperationAction(ISD::ABS,                MVT::v16i8, Legal);
1245     setOperationAction(ISD::ABS,                MVT::v8i16, Legal);
1246     setOperationAction(ISD::ABS,                MVT::v4i32, Legal);
1247     setOperationAction(ISD::BITREVERSE,         MVT::v16i8, Custom);
1248     setOperationAction(ISD::CTLZ,               MVT::v16i8, Custom);
1249     setOperationAction(ISD::CTLZ,               MVT::v8i16, Custom);
1250     setOperationAction(ISD::CTLZ,               MVT::v4i32, Custom);
1251     setOperationAction(ISD::CTLZ,               MVT::v2i64, Custom);
1252 
1253     // These might be better off as horizontal vector ops.
1254     setOperationAction(ISD::ADD,                MVT::i16, Custom);
1255     setOperationAction(ISD::ADD,                MVT::i32, Custom);
1256     setOperationAction(ISD::SUB,                MVT::i16, Custom);
1257     setOperationAction(ISD::SUB,                MVT::i32, Custom);
1258   }
1259 
1260   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
1261     for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
1262       setOperationAction(ISD::FFLOOR,            RoundedTy,  Legal);
1263       setOperationAction(ISD::STRICT_FFLOOR,     RoundedTy,  Legal);
1264       setOperationAction(ISD::FCEIL,             RoundedTy,  Legal);
1265       setOperationAction(ISD::STRICT_FCEIL,      RoundedTy,  Legal);
1266       setOperationAction(ISD::FTRUNC,            RoundedTy,  Legal);
1267       setOperationAction(ISD::STRICT_FTRUNC,     RoundedTy,  Legal);
1268       setOperationAction(ISD::FRINT,             RoundedTy,  Legal);
1269       setOperationAction(ISD::STRICT_FRINT,      RoundedTy,  Legal);
1270       setOperationAction(ISD::FNEARBYINT,        RoundedTy,  Legal);
1271       setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy,  Legal);
1272       setOperationAction(ISD::FROUNDEVEN,        RoundedTy,  Legal);
1273       setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy,  Legal);
1274 
1275       setOperationAction(ISD::FROUND,            RoundedTy,  Custom);
1276     }
1277 
1278     setOperationAction(ISD::SMAX,               MVT::v16i8, Legal);
1279     setOperationAction(ISD::SMAX,               MVT::v4i32, Legal);
1280     setOperationAction(ISD::UMAX,               MVT::v8i16, Legal);
1281     setOperationAction(ISD::UMAX,               MVT::v4i32, Legal);
1282     setOperationAction(ISD::SMIN,               MVT::v16i8, Legal);
1283     setOperationAction(ISD::SMIN,               MVT::v4i32, Legal);
1284     setOperationAction(ISD::UMIN,               MVT::v8i16, Legal);
1285     setOperationAction(ISD::UMIN,               MVT::v4i32, Legal);
1286 
1287     for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
1288       setOperationAction(ISD::ABDS,             VT, Custom);
1289       setOperationAction(ISD::ABDU,             VT, Custom);
1290     }
1291 
1292     setOperationAction(ISD::UADDSAT,            MVT::v4i32, Custom);
1293     setOperationAction(ISD::SADDSAT,            MVT::v2i64, Custom);
1294     setOperationAction(ISD::SSUBSAT,            MVT::v2i64, Custom);
1295 
1296     // FIXME: Do we need to handle scalar-to-vector here?
1297     setOperationAction(ISD::MUL,                MVT::v4i32, Legal);
1298     setOperationAction(ISD::SMULO,              MVT::v2i32, Custom);
1299 
1300     // We directly match byte blends in the backend as they match the VSELECT
1301     // condition form.
1302     setOperationAction(ISD::VSELECT,            MVT::v16i8, Legal);
1303 
1304     // SSE41 brings specific instructions for doing vector sign extend even in
1305     // cases where we don't have SRA.
1306     for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1307       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
1308       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
1309     }
1310 
1311     // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1312     for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1313       setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8,  Legal);
1314       setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8,  Legal);
1315       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8,  Legal);
1316       setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
1317       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
1318       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
1319     }
1320 
1321     if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
1322       // We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
1323       // do the pre and post work in the vector domain.
1324       setOperationAction(ISD::UINT_TO_FP,        MVT::v4i64, Custom);
1325       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
1326       // We need to mark SINT_TO_FP as Custom even though we want to expand it
1327       // so that DAG combine doesn't try to turn it into uint_to_fp.
1328       setOperationAction(ISD::SINT_TO_FP,        MVT::v4i64, Custom);
1329       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
1330     }
1331   }
1332 
1333   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
1334     setOperationAction(ISD::UADDSAT,            MVT::v2i64, Custom);
1335   }
1336 
1337   if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
1338     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,
1339                      MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1340       setOperationAction(ISD::ROTL, VT, Custom);
1341       setOperationAction(ISD::ROTR, VT, Custom);
1342     }
1343 
1344     // XOP can efficiently perform BITREVERSE with VPPERM.
1345     for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
1346       setOperationAction(ISD::BITREVERSE, VT, Custom);
1347 
1348     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,
1349                      MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1350       setOperationAction(ISD::BITREVERSE, VT, Custom);
1351   }
1352 
1353   if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
1354     bool HasInt256 = Subtarget.hasInt256();
1355 
1356     addRegisterClass(MVT::v32i8,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1357                                                      : &X86::VR256RegClass);
1358     addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1359                                                      : &X86::VR256RegClass);
1360     addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1361                                                      : &X86::VR256RegClass);
1362     addRegisterClass(MVT::v8i32,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1363                                                      : &X86::VR256RegClass);
1364     addRegisterClass(MVT::v8f32,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1365                                                      : &X86::VR256RegClass);
1366     addRegisterClass(MVT::v4i64,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1367                                                      : &X86::VR256RegClass);
1368     addRegisterClass(MVT::v4f64,  Subtarget.hasVLX() ? &X86::VR256XRegClass
1369                                                      : &X86::VR256RegClass);
1370 
1371     for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
1372       setOperationAction(ISD::FFLOOR,            VT, Legal);
1373       setOperationAction(ISD::STRICT_FFLOOR,     VT, Legal);
1374       setOperationAction(ISD::FCEIL,             VT, Legal);
1375       setOperationAction(ISD::STRICT_FCEIL,      VT, Legal);
1376       setOperationAction(ISD::FTRUNC,            VT, Legal);
1377       setOperationAction(ISD::STRICT_FTRUNC,     VT, Legal);
1378       setOperationAction(ISD::FRINT,             VT, Legal);
1379       setOperationAction(ISD::STRICT_FRINT,      VT, Legal);
1380       setOperationAction(ISD::FNEARBYINT,        VT, Legal);
1381       setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1382       setOperationAction(ISD::FROUNDEVEN,        VT, Legal);
1383       setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1384 
1385       setOperationAction(ISD::FROUND,            VT, Custom);
1386 
1387       setOperationAction(ISD::FNEG,              VT, Custom);
1388       setOperationAction(ISD::FABS,              VT, Custom);
1389       setOperationAction(ISD::FCOPYSIGN,         VT, Custom);
1390 
1391       setOperationAction(ISD::FMAXIMUM,          VT, Custom);
1392       setOperationAction(ISD::FMINIMUM,          VT, Custom);
1393     }
1394 
1395     // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1396     // even though v8i16 is a legal type.
1397     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v8i16, MVT::v8i32);
1398     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v8i16, MVT::v8i32);
1399     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1400     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1401     setOperationAction(ISD::FP_TO_SINT,                MVT::v8i32, Custom);
1402     setOperationAction(ISD::FP_TO_UINT,                MVT::v8i32, Custom);
1403     setOperationAction(ISD::STRICT_FP_TO_SINT,         MVT::v8i32, Custom);
1404 
1405     setOperationAction(ISD::SINT_TO_FP,         MVT::v8i32, Custom);
1406     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v8i32, Custom);
1407     setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Expand);
1408     setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Expand);
1409     setOperationAction(ISD::FP_EXTEND,          MVT::v4f64, Custom);
1410     setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v4f64, Custom);
1411 
1412     setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v4f32, Legal);
1413     setOperationAction(ISD::STRICT_FADD,        MVT::v8f32, Legal);
1414     setOperationAction(ISD::STRICT_FADD,        MVT::v4f64, Legal);
1415     setOperationAction(ISD::STRICT_FSUB,        MVT::v8f32, Legal);
1416     setOperationAction(ISD::STRICT_FSUB,        MVT::v4f64, Legal);
1417     setOperationAction(ISD::STRICT_FMUL,        MVT::v8f32, Legal);
1418     setOperationAction(ISD::STRICT_FMUL,        MVT::v4f64, Legal);
1419     setOperationAction(ISD::STRICT_FDIV,        MVT::v8f32, Legal);
1420     setOperationAction(ISD::STRICT_FDIV,        MVT::v4f64, Legal);
1421     setOperationAction(ISD::STRICT_FSQRT,       MVT::v8f32, Legal);
1422     setOperationAction(ISD::STRICT_FSQRT,       MVT::v4f64, Legal);
1423 
1424     if (!Subtarget.hasAVX512())
1425       setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
1426 
1427     // In the customized shift lowering, the legal v8i32/v4i64 cases
1428     // in AVX2 will be recognized.
1429     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1430       setOperationAction(ISD::SRL,             VT, Custom);
1431       setOperationAction(ISD::SHL,             VT, Custom);
1432       setOperationAction(ISD::SRA,             VT, Custom);
1433       setOperationAction(ISD::ABDS,            VT, Custom);
1434       setOperationAction(ISD::ABDU,            VT, Custom);
1435       if (VT == MVT::v4i64) continue;
1436       setOperationAction(ISD::ROTL,            VT, Custom);
1437       setOperationAction(ISD::ROTR,            VT, Custom);
1438       setOperationAction(ISD::FSHL,            VT, Custom);
1439       setOperationAction(ISD::FSHR,            VT, Custom);
1440     }
1441 
1442     // These types need custom splitting if their input is a 128-bit vector.
1443     setOperationAction(ISD::SIGN_EXTEND,       MVT::v8i64,  Custom);
1444     setOperationAction(ISD::SIGN_EXTEND,       MVT::v16i32, Custom);
1445     setOperationAction(ISD::ZERO_EXTEND,       MVT::v8i64,  Custom);
1446     setOperationAction(ISD::ZERO_EXTEND,       MVT::v16i32, Custom);
1447 
1448     setOperationAction(ISD::SELECT,            MVT::v4f64, Custom);
1449     setOperationAction(ISD::SELECT,            MVT::v4i64, Custom);
1450     setOperationAction(ISD::SELECT,            MVT::v8i32, Custom);
1451     setOperationAction(ISD::SELECT,            MVT::v16i16, Custom);
1452     setOperationAction(ISD::SELECT,            MVT::v16f16, Custom);
1453     setOperationAction(ISD::SELECT,            MVT::v32i8, Custom);
1454     setOperationAction(ISD::SELECT,            MVT::v8f32, Custom);
1455 
1456     for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1457       setOperationAction(ISD::SIGN_EXTEND,     VT, Custom);
1458       setOperationAction(ISD::ZERO_EXTEND,     VT, Custom);
1459       setOperationAction(ISD::ANY_EXTEND,      VT, Custom);
1460     }
1461 
1462     setOperationAction(ISD::TRUNCATE,          MVT::v32i8, Custom);
1463     setOperationAction(ISD::TRUNCATE,          MVT::v32i16, Custom);
1464     setOperationAction(ISD::TRUNCATE,          MVT::v32i32, Custom);
1465     setOperationAction(ISD::TRUNCATE,          MVT::v32i64, Custom);
1466 
1467     setOperationAction(ISD::BITREVERSE,        MVT::v32i8, Custom);
1468 
1469     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1470       setOperationAction(ISD::SETCC,           VT, Custom);
1471       setOperationAction(ISD::CTPOP,           VT, Custom);
1472       setOperationAction(ISD::CTLZ,            VT, Custom);
1473 
1474       // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1475       // setcc all the way to isel and prefer SETGT in some isel patterns.
1476       setCondCodeAction(ISD::SETLT, VT, Custom);
1477       setCondCodeAction(ISD::SETLE, VT, Custom);
1478     }
1479 
1480     setOperationAction(ISD::SETCC,          MVT::v4f64, Custom);
1481     setOperationAction(ISD::SETCC,          MVT::v8f32, Custom);
1482     setOperationAction(ISD::STRICT_FSETCC,  MVT::v4f64, Custom);
1483     setOperationAction(ISD::STRICT_FSETCC,  MVT::v8f32, Custom);
1484     setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f64, Custom);
1485     setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f32, Custom);
1486 
1487     if (Subtarget.hasAnyFMA()) {
1488       for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
1489                        MVT::v2f64, MVT::v4f64 }) {
1490         setOperationAction(ISD::FMA, VT, Legal);
1491         setOperationAction(ISD::STRICT_FMA, VT, Legal);
1492       }
1493     }
1494 
1495     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1496       setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
1497       setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
1498     }
1499 
1500     setOperationAction(ISD::MUL,       MVT::v4i64,  Custom);
1501     setOperationAction(ISD::MUL,       MVT::v8i32,  HasInt256 ? Legal : Custom);
1502     setOperationAction(ISD::MUL,       MVT::v16i16, HasInt256 ? Legal : Custom);
1503     setOperationAction(ISD::MUL,       MVT::v32i8,  Custom);
1504 
1505     setOperationAction(ISD::MULHU,     MVT::v8i32,  Custom);
1506     setOperationAction(ISD::MULHS,     MVT::v8i32,  Custom);
1507     setOperationAction(ISD::MULHU,     MVT::v16i16, HasInt256 ? Legal : Custom);
1508     setOperationAction(ISD::MULHS,     MVT::v16i16, HasInt256 ? Legal : Custom);
1509     setOperationAction(ISD::MULHU,     MVT::v32i8,  Custom);
1510     setOperationAction(ISD::MULHS,     MVT::v32i8,  Custom);
1511     setOperationAction(ISD::AVGCEILU,  MVT::v16i16, HasInt256 ? Legal : Custom);
1512     setOperationAction(ISD::AVGCEILU,  MVT::v32i8,  HasInt256 ? Legal : Custom);
1513 
1514     setOperationAction(ISD::SMULO,     MVT::v32i8, Custom);
1515     setOperationAction(ISD::UMULO,     MVT::v32i8, Custom);
1516 
1517     setOperationAction(ISD::ABS,       MVT::v4i64,  Custom);
1518     setOperationAction(ISD::SMAX,      MVT::v4i64,  Custom);
1519     setOperationAction(ISD::UMAX,      MVT::v4i64,  Custom);
1520     setOperationAction(ISD::SMIN,      MVT::v4i64,  Custom);
1521     setOperationAction(ISD::UMIN,      MVT::v4i64,  Custom);
1522 
1523     setOperationAction(ISD::UADDSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
1524     setOperationAction(ISD::SADDSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
1525     setOperationAction(ISD::USUBSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
1526     setOperationAction(ISD::SSUBSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
1527     setOperationAction(ISD::UADDSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
1528     setOperationAction(ISD::SADDSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
1529     setOperationAction(ISD::USUBSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
1530     setOperationAction(ISD::SSUBSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
1531     setOperationAction(ISD::UADDSAT,   MVT::v8i32, Custom);
1532     setOperationAction(ISD::USUBSAT,   MVT::v8i32, Custom);
1533     setOperationAction(ISD::UADDSAT,   MVT::v4i64, Custom);
1534     setOperationAction(ISD::USUBSAT,   MVT::v4i64, Custom);
1535 
1536     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
1537       setOperationAction(ISD::ABS,  VT, HasInt256 ? Legal : Custom);
1538       setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
1539       setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
1540       setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
1541       setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
1542     }
1543 
1544     for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
1545       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1546       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1547     }
1548 
1549     if (HasInt256) {
1550       // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1551       // when we have a 256bit-wide blend with immediate.
1552       setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1553       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
1554 
1555       // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1556       for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1557         setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
1558         setLoadExtAction(LoadExtOp, MVT::v8i32,  MVT::v8i8,  Legal);
1559         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i8,  Legal);
1560         setLoadExtAction(LoadExtOp, MVT::v8i32,  MVT::v8i16, Legal);
1561         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i16, Legal);
1562         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i32, Legal);
1563       }
1564     }
1565 
1566     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1567                      MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
1568       setOperationAction(ISD::MLOAD,  VT, Subtarget.hasVLX() ? Legal : Custom);
1569       setOperationAction(ISD::MSTORE, VT, Legal);
1570     }
1571 
1572     // Extract subvector is special because the value type
1573     // (result) is 128-bit but the source is 256-bit wide.
1574     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1575                      MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1576       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1577     }
1578 
1579     // Custom lower several nodes for 256-bit types.
1580     for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1581                     MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {
1582       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
1583       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
1584       setOperationAction(ISD::VSELECT,            VT, Custom);
1585       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
1586       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1587       setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
1588       setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Legal);
1589       setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
1590       setOperationAction(ISD::STORE,              VT, Custom);
1591     }
1592     setF16Action(MVT::v16f16, Expand);
1593     setOperationAction(ISD::FNEG, MVT::v16f16, Custom);
1594     setOperationAction(ISD::FABS, MVT::v16f16, Custom);
1595     setOperationAction(ISD::FCOPYSIGN, MVT::v16f16, Custom);
1596     setOperationAction(ISD::FADD, MVT::v16f16, Expand);
1597     setOperationAction(ISD::FSUB, MVT::v16f16, Expand);
1598     setOperationAction(ISD::FMUL, MVT::v16f16, Expand);
1599     setOperationAction(ISD::FDIV, MVT::v16f16, Expand);
1600 
1601     if (HasInt256) {
1602       setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1603 
1604       // Custom legalize 2x32 to get a little better code.
1605       setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
1606       setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
1607 
1608       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1609                        MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1610         setOperationAction(ISD::MGATHER,  VT, Custom);
1611     }
1612   }
1613 
1614   if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
1615       Subtarget.hasF16C()) {
1616     for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {
1617       setOperationAction(ISD::FP_ROUND,           VT, Custom);
1618       setOperationAction(ISD::STRICT_FP_ROUND,    VT, Custom);
1619     }
1620     for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) {
1621       setOperationAction(ISD::FP_EXTEND,          VT, Custom);
1622       setOperationAction(ISD::STRICT_FP_EXTEND,   VT, Custom);
1623     }
1624     for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
1625       setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);
1626       setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
1627     }
1628   }
1629 
1630   // This block controls legalization of the mask vector sizes that are
1631   // available with AVX512. 512-bit vectors are in a separate block controlled
1632   // by useAVX512Regs.
1633   if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1634     addRegisterClass(MVT::v1i1,   &X86::VK1RegClass);
1635     addRegisterClass(MVT::v2i1,   &X86::VK2RegClass);
1636     addRegisterClass(MVT::v4i1,   &X86::VK4RegClass);
1637     addRegisterClass(MVT::v8i1,   &X86::VK8RegClass);
1638     addRegisterClass(MVT::v16i1,  &X86::VK16RegClass);
1639 
1640     setOperationAction(ISD::SELECT,             MVT::v1i1, Custom);
1641     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
1642     setOperationAction(ISD::BUILD_VECTOR,       MVT::v1i1, Custom);
1643 
1644     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v8i1,  MVT::v8i32);
1645     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v8i1,  MVT::v8i32);
1646     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v4i1,  MVT::v4i32);
1647     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v4i1,  MVT::v4i32);
1648     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1,  MVT::v8i32);
1649     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1,  MVT::v8i32);
1650     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1,  MVT::v4i32);
1651     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1,  MVT::v4i32);
1652     setOperationAction(ISD::FP_TO_SINT,                MVT::v2i1,  Custom);
1653     setOperationAction(ISD::FP_TO_UINT,                MVT::v2i1,  Custom);
1654     setOperationAction(ISD::STRICT_FP_TO_SINT,         MVT::v2i1,  Custom);
1655     setOperationAction(ISD::STRICT_FP_TO_UINT,         MVT::v2i1,  Custom);
1656 
1657     // There is no byte sized k-register load or store without AVX512DQ.
1658     if (!Subtarget.hasDQI()) {
1659       setOperationAction(ISD::LOAD, MVT::v1i1, Custom);
1660       setOperationAction(ISD::LOAD, MVT::v2i1, Custom);
1661       setOperationAction(ISD::LOAD, MVT::v4i1, Custom);
1662       setOperationAction(ISD::LOAD, MVT::v8i1, Custom);
1663 
1664       setOperationAction(ISD::STORE, MVT::v1i1, Custom);
1665       setOperationAction(ISD::STORE, MVT::v2i1, Custom);
1666       setOperationAction(ISD::STORE, MVT::v4i1, Custom);
1667       setOperationAction(ISD::STORE, MVT::v8i1, Custom);
1668     }
1669 
1670     // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
1671     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1672       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1673       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1674       setOperationAction(ISD::ANY_EXTEND,  VT, Custom);
1675     }
1676 
1677     for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })
1678       setOperationAction(ISD::VSELECT,          VT, Expand);
1679 
1680     for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
1681       setOperationAction(ISD::SETCC,            VT, Custom);
1682       setOperationAction(ISD::SELECT,           VT, Custom);
1683       setOperationAction(ISD::TRUNCATE,         VT, Custom);
1684 
1685       setOperationAction(ISD::BUILD_VECTOR,     VT, Custom);
1686       setOperationAction(ISD::CONCAT_VECTORS,   VT, Custom);
1687       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1688       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1689       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1690       setOperationAction(ISD::VECTOR_SHUFFLE,   VT,  Custom);
1691     }
1692 
1693     for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
1694       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1695   }
1696 
1697   // This block controls legalization for 512-bit operations with 8/16/32/64 bit
1698   // elements. 512-bits can be disabled based on prefer-vector-width and
1699   // required-vector-width function attributes.
1700   if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
1701     bool HasBWI = Subtarget.hasBWI();
1702 
1703     addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1704     addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1705     addRegisterClass(MVT::v8i64,  &X86::VR512RegClass);
1706     addRegisterClass(MVT::v8f64,  &X86::VR512RegClass);
1707     addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1708     addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
1709     addRegisterClass(MVT::v64i8,  &X86::VR512RegClass);
1710 
1711     for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1712       setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8,  Legal);
1713       setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
1714       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i8,   Legal);
1715       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i16,  Legal);
1716       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i32,  Legal);
1717       if (HasBWI)
1718         setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
1719     }
1720 
1721     for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
1722       setOperationAction(ISD::FMAXIMUM, VT, Custom);
1723       setOperationAction(ISD::FMINIMUM, VT, Custom);
1724       setOperationAction(ISD::FNEG,  VT, Custom);
1725       setOperationAction(ISD::FABS,  VT, Custom);
1726       setOperationAction(ISD::FMA,   VT, Legal);
1727       setOperationAction(ISD::STRICT_FMA, VT, Legal);
1728       setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1729     }
1730 
1731     for (MVT VT : { MVT::v16i1, MVT::v16i8 }) {
1732       setOperationPromotedToType(ISD::FP_TO_SINT       , VT, MVT::v16i32);
1733       setOperationPromotedToType(ISD::FP_TO_UINT       , VT, MVT::v16i32);
1734       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
1735       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
1736     }
1737 
1738     for (MVT VT : { MVT::v16i16, MVT::v16i32 }) {
1739       setOperationAction(ISD::FP_TO_SINT,        VT, Custom);
1740       setOperationAction(ISD::FP_TO_UINT,        VT, Custom);
1741       setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1742       setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1743     }
1744 
1745     setOperationAction(ISD::SINT_TO_FP,        MVT::v16i32, Custom);
1746     setOperationAction(ISD::UINT_TO_FP,        MVT::v16i32, Custom);
1747     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);
1748     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);
1749     setOperationAction(ISD::FP_EXTEND,         MVT::v8f64,  Custom);
1750     setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v8f64,  Custom);
1751 
1752     setOperationAction(ISD::STRICT_FADD,      MVT::v16f32, Legal);
1753     setOperationAction(ISD::STRICT_FADD,      MVT::v8f64,  Legal);
1754     setOperationAction(ISD::STRICT_FSUB,      MVT::v16f32, Legal);
1755     setOperationAction(ISD::STRICT_FSUB,      MVT::v8f64,  Legal);
1756     setOperationAction(ISD::STRICT_FMUL,      MVT::v16f32, Legal);
1757     setOperationAction(ISD::STRICT_FMUL,      MVT::v8f64,  Legal);
1758     setOperationAction(ISD::STRICT_FDIV,      MVT::v16f32, Legal);
1759     setOperationAction(ISD::STRICT_FDIV,      MVT::v8f64,  Legal);
1760     setOperationAction(ISD::STRICT_FSQRT,     MVT::v16f32, Legal);
1761     setOperationAction(ISD::STRICT_FSQRT,     MVT::v8f64,  Legal);
1762     setOperationAction(ISD::STRICT_FP_ROUND,  MVT::v8f32,  Legal);
1763 
1764     setTruncStoreAction(MVT::v8i64,   MVT::v8i8,   Legal);
1765     setTruncStoreAction(MVT::v8i64,   MVT::v8i16,  Legal);
1766     setTruncStoreAction(MVT::v8i64,   MVT::v8i32,  Legal);
1767     setTruncStoreAction(MVT::v16i32,  MVT::v16i8,  Legal);
1768     setTruncStoreAction(MVT::v16i32,  MVT::v16i16, Legal);
1769     if (HasBWI)
1770       setTruncStoreAction(MVT::v32i16,  MVT::v32i8, Legal);
1771 
1772     // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
1773     // to 512-bit rather than use the AVX2 instructions so that we can use
1774     // k-masks.
1775     if (!Subtarget.hasVLX()) {
1776       for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1777            MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1778         setOperationAction(ISD::MLOAD,  VT, Custom);
1779         setOperationAction(ISD::MSTORE, VT, Custom);
1780       }
1781     }
1782 
1783     setOperationAction(ISD::TRUNCATE,    MVT::v8i32,  Legal);
1784     setOperationAction(ISD::TRUNCATE,    MVT::v16i16, Legal);
1785     setOperationAction(ISD::TRUNCATE,    MVT::v32i8,  HasBWI ? Legal : Custom);
1786     setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
1787     setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1788     setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64,  Custom);
1789     setOperationAction(ISD::ANY_EXTEND,  MVT::v32i16, Custom);
1790     setOperationAction(ISD::ANY_EXTEND,  MVT::v16i32, Custom);
1791     setOperationAction(ISD::ANY_EXTEND,  MVT::v8i64,  Custom);
1792     setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
1793     setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1794     setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64,  Custom);
1795 
1796     if (HasBWI) {
1797       // Extends from v64i1 masks to 512-bit vectors.
1798       setOperationAction(ISD::SIGN_EXTEND,        MVT::v64i8, Custom);
1799       setOperationAction(ISD::ZERO_EXTEND,        MVT::v64i8, Custom);
1800       setOperationAction(ISD::ANY_EXTEND,         MVT::v64i8, Custom);
1801     }
1802 
1803     for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
1804       setOperationAction(ISD::FFLOOR,            VT, Legal);
1805       setOperationAction(ISD::STRICT_FFLOOR,     VT, Legal);
1806       setOperationAction(ISD::FCEIL,             VT, Legal);
1807       setOperationAction(ISD::STRICT_FCEIL,      VT, Legal);
1808       setOperationAction(ISD::FTRUNC,            VT, Legal);
1809       setOperationAction(ISD::STRICT_FTRUNC,     VT, Legal);
1810       setOperationAction(ISD::FRINT,             VT, Legal);
1811       setOperationAction(ISD::STRICT_FRINT,      VT, Legal);
1812       setOperationAction(ISD::FNEARBYINT,        VT, Legal);
1813       setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1814       setOperationAction(ISD::FROUNDEVEN,        VT, Legal);
1815       setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1816 
1817       setOperationAction(ISD::FROUND,            VT, Custom);
1818     }
1819 
1820     for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
1821       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1822       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1823     }
1824 
1825     setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);
1826     setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);
1827     setOperationAction(ISD::ADD, MVT::v64i8,  HasBWI ? Legal : Custom);
1828     setOperationAction(ISD::SUB, MVT::v64i8,  HasBWI ? Legal : Custom);
1829 
1830     setOperationAction(ISD::MUL, MVT::v8i64,  Custom);
1831     setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1832     setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);
1833     setOperationAction(ISD::MUL, MVT::v64i8,  Custom);
1834 
1835     setOperationAction(ISD::MULHU, MVT::v16i32, Custom);
1836     setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
1837     setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);
1838     setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);
1839     setOperationAction(ISD::MULHS, MVT::v64i8,  Custom);
1840     setOperationAction(ISD::MULHU, MVT::v64i8,  Custom);
1841     setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);
1842     setOperationAction(ISD::AVGCEILU, MVT::v64i8,  HasBWI ? Legal : Custom);
1843 
1844     setOperationAction(ISD::SMULO, MVT::v64i8, Custom);
1845     setOperationAction(ISD::UMULO, MVT::v64i8, Custom);
1846 
1847     setOperationAction(ISD::BITREVERSE, MVT::v64i8,  Custom);
1848 
1849     for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
1850       setOperationAction(ISD::SRL,              VT, Custom);
1851       setOperationAction(ISD::SHL,              VT, Custom);
1852       setOperationAction(ISD::SRA,              VT, Custom);
1853       setOperationAction(ISD::ROTL,             VT, Custom);
1854       setOperationAction(ISD::ROTR,             VT, Custom);
1855       setOperationAction(ISD::SETCC,            VT, Custom);
1856       setOperationAction(ISD::ABDS,             VT, Custom);
1857       setOperationAction(ISD::ABDU,             VT, Custom);
1858 
1859       // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1860       // setcc all the way to isel and prefer SETGT in some isel patterns.
1861       setCondCodeAction(ISD::SETLT, VT, Custom);
1862       setCondCodeAction(ISD::SETLE, VT, Custom);
1863     }
1864 
1865     setOperationAction(ISD::SETCC,          MVT::v8f64, Custom);
1866     setOperationAction(ISD::SETCC,          MVT::v16f32, Custom);
1867     setOperationAction(ISD::STRICT_FSETCC,  MVT::v8f64, Custom);
1868     setOperationAction(ISD::STRICT_FSETCC,  MVT::v16f32, Custom);
1869     setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f64, Custom);
1870     setOperationAction(ISD::STRICT_FSETCCS, MVT::v16f32, Custom);
1871 
1872     for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1873       setOperationAction(ISD::SMAX,             VT, Legal);
1874       setOperationAction(ISD::UMAX,             VT, Legal);
1875       setOperationAction(ISD::SMIN,             VT, Legal);
1876       setOperationAction(ISD::UMIN,             VT, Legal);
1877       setOperationAction(ISD::ABS,              VT, Legal);
1878       setOperationAction(ISD::CTPOP,            VT, Custom);
1879     }
1880 
1881     for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1882       setOperationAction(ISD::ABS,     VT, HasBWI ? Legal : Custom);
1883       setOperationAction(ISD::CTPOP,   VT, Subtarget.hasBITALG() ? Legal : Custom);
1884       setOperationAction(ISD::CTLZ,    VT, Custom);
1885       setOperationAction(ISD::SMAX,    VT, HasBWI ? Legal : Custom);
1886       setOperationAction(ISD::UMAX,    VT, HasBWI ? Legal : Custom);
1887       setOperationAction(ISD::SMIN,    VT, HasBWI ? Legal : Custom);
1888       setOperationAction(ISD::UMIN,    VT, HasBWI ? Legal : Custom);
1889       setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom);
1890       setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom);
1891       setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom);
1892       setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom);
1893     }
1894 
1895     setOperationAction(ISD::FSHL,       MVT::v64i8, Custom);
1896     setOperationAction(ISD::FSHR,       MVT::v64i8, Custom);
1897     setOperationAction(ISD::FSHL,      MVT::v32i16, Custom);
1898     setOperationAction(ISD::FSHR,      MVT::v32i16, Custom);
1899     setOperationAction(ISD::FSHL,      MVT::v16i32, Custom);
1900     setOperationAction(ISD::FSHR,      MVT::v16i32, Custom);
1901 
1902     if (Subtarget.hasDQI()) {
1903       for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
1904                        ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1905                        ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
1906         setOperationAction(Opc,           MVT::v8i64, Custom);
1907       setOperationAction(ISD::MUL,        MVT::v8i64, Legal);
1908     }
1909 
1910     if (Subtarget.hasCDI()) {
1911       // NonVLX sub-targets extend 128/256 vectors to use the 512 version.
1912       for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
1913         setOperationAction(ISD::CTLZ,            VT, Legal);
1914       }
1915     } // Subtarget.hasCDI()
1916 
1917     if (Subtarget.hasVPOPCNTDQ()) {
1918       for (auto VT : { MVT::v16i32, MVT::v8i64 })
1919         setOperationAction(ISD::CTPOP, VT, Legal);
1920     }
1921 
1922     // Extract subvector is special because the value type
1923     // (result) is 256-bit but the source is 512-bit wide.
1924     // 128-bit was made Legal under AVX1.
1925     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1926                      MVT::v16f16, MVT::v8f32, MVT::v4f64 })
1927       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1928 
1929     for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
1930                      MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {
1931       setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
1932       setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Legal);
1933       setOperationAction(ISD::SELECT,             VT, Custom);
1934       setOperationAction(ISD::VSELECT,            VT, Custom);
1935       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
1936       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1937       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
1938       setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
1939       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
1940     }
1941     setF16Action(MVT::v32f16, Expand);
1942     setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);
1943     setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Custom);
1944     setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
1945     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Custom);
1946     for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
1947       setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
1948       setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
1949     }
1950 
1951     for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
1952       setOperationAction(ISD::MLOAD,               VT, Legal);
1953       setOperationAction(ISD::MSTORE,              VT, Legal);
1954       setOperationAction(ISD::MGATHER,             VT, Custom);
1955       setOperationAction(ISD::MSCATTER,            VT, Custom);
1956     }
1957     if (HasBWI) {
1958       for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1959         setOperationAction(ISD::MLOAD,        VT, Legal);
1960         setOperationAction(ISD::MSTORE,       VT, Legal);
1961       }
1962     } else {
1963       setOperationAction(ISD::STORE, MVT::v32i16, Custom);
1964       setOperationAction(ISD::STORE, MVT::v64i8,  Custom);
1965     }
1966 
1967     if (Subtarget.hasVBMI2()) {
1968       for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
1969         setOperationAction(ISD::FSHL, VT, Custom);
1970         setOperationAction(ISD::FSHR, VT, Custom);
1971       }
1972 
1973       setOperationAction(ISD::ROTL, MVT::v32i16, Custom);
1974       setOperationAction(ISD::ROTR, MVT::v32i16, Custom);
1975     }
1976   }// useAVX512Regs
1977 
1978   if (!Subtarget.useSoftFloat() && Subtarget.hasVBMI2()) {
1979     for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v16i16, MVT::v8i32,
1980                     MVT::v4i64}) {
1981       setOperationAction(ISD::FSHL, VT, Custom);
1982       setOperationAction(ISD::FSHR, VT, Custom);
1983     }
1984   }
1985 
1986   // This block controls legalization for operations that don't have
1987   // pre-AVX512 equivalents. Without VLX we use 512-bit operations for
1988   // narrower widths.
1989   if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1990     // These operations are handled on non-VLX by artificially widening in
1991     // isel patterns.
1992 
1993     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i32, Custom);
1994     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v4i32, Custom);
1995     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v2i32, Custom);
1996 
1997     if (Subtarget.hasDQI()) {
1998       // Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
1999       // v2f32 UINT_TO_FP is already custom under SSE2.
2000       assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&
2001              isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&
2002              "Unexpected operation action!");
2003       // v2i64 FP_TO_S/UINT(v2f32) custom conversion.
2004       setOperationAction(ISD::FP_TO_SINT,        MVT::v2f32, Custom);
2005       setOperationAction(ISD::FP_TO_UINT,        MVT::v2f32, Custom);
2006       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
2007       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
2008     }
2009 
2010     for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
2011       setOperationAction(ISD::SMAX, VT, Legal);
2012       setOperationAction(ISD::UMAX, VT, Legal);
2013       setOperationAction(ISD::SMIN, VT, Legal);
2014       setOperationAction(ISD::UMIN, VT, Legal);
2015       setOperationAction(ISD::ABS,  VT, Legal);
2016     }
2017 
2018     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2019       setOperationAction(ISD::ROTL,     VT, Custom);
2020       setOperationAction(ISD::ROTR,     VT, Custom);
2021     }
2022 
2023     // Custom legalize 2x32 to get a little better code.
2024     setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
2025     setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
2026 
2027     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
2028                      MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
2029       setOperationAction(ISD::MSCATTER, VT, Custom);
2030 
2031     if (Subtarget.hasDQI()) {
2032       for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
2033                        ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2034                        ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT}) {
2035         setOperationAction(Opc, MVT::v2i64, Custom);
2036         setOperationAction(Opc, MVT::v4i64, Custom);
2037       }
2038       setOperationAction(ISD::MUL, MVT::v2i64, Legal);
2039       setOperationAction(ISD::MUL, MVT::v4i64, Legal);
2040     }
2041 
2042     if (Subtarget.hasCDI()) {
2043       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2044         setOperationAction(ISD::CTLZ,            VT, Legal);
2045       }
2046     } // Subtarget.hasCDI()
2047 
2048     if (Subtarget.hasVPOPCNTDQ()) {
2049       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })
2050         setOperationAction(ISD::CTPOP, VT, Legal);
2051     }
2052     setOperationAction(ISD::FNEG, MVT::v32f16, Custom);
2053     setOperationAction(ISD::FABS, MVT::v32f16, Custom);
2054     setOperationAction(ISD::FCOPYSIGN, MVT::v32f16, Custom);
2055   }
2056 
2057   // This block control legalization of v32i1/v64i1 which are available with
2058   // AVX512BW..
2059   if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
2060     addRegisterClass(MVT::v32i1,  &X86::VK32RegClass);
2061     addRegisterClass(MVT::v64i1,  &X86::VK64RegClass);
2062 
2063     for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
2064       setOperationAction(ISD::VSELECT,            VT, Expand);
2065       setOperationAction(ISD::TRUNCATE,           VT, Custom);
2066       setOperationAction(ISD::SETCC,              VT, Custom);
2067       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2068       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
2069       setOperationAction(ISD::SELECT,             VT, Custom);
2070       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
2071       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
2072       setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
2073       setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Custom);
2074     }
2075 
2076     for (auto VT : { MVT::v16i1, MVT::v32i1 })
2077       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
2078 
2079     // Extends from v32i1 masks to 256-bit vectors.
2080     setOperationAction(ISD::SIGN_EXTEND,        MVT::v32i8, Custom);
2081     setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i8, Custom);
2082     setOperationAction(ISD::ANY_EXTEND,         MVT::v32i8, Custom);
2083 
2084     for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {
2085       setOperationAction(ISD::MLOAD,  VT, Subtarget.hasVLX() ? Legal : Custom);
2086       setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
2087     }
2088 
2089     // These operations are handled on non-VLX by artificially widening in
2090     // isel patterns.
2091     // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
2092 
2093     if (Subtarget.hasBITALG()) {
2094       for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
2095         setOperationAction(ISD::CTPOP, VT, Legal);
2096     }
2097   }
2098 
2099   if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
2100     auto setGroup = [&] (MVT VT) {
2101       setOperationAction(ISD::FADD,               VT, Legal);
2102       setOperationAction(ISD::STRICT_FADD,        VT, Legal);
2103       setOperationAction(ISD::FSUB,               VT, Legal);
2104       setOperationAction(ISD::STRICT_FSUB,        VT, Legal);
2105       setOperationAction(ISD::FMUL,               VT, Legal);
2106       setOperationAction(ISD::STRICT_FMUL,        VT, Legal);
2107       setOperationAction(ISD::FDIV,               VT, Legal);
2108       setOperationAction(ISD::STRICT_FDIV,        VT, Legal);
2109       setOperationAction(ISD::FSQRT,              VT, Legal);
2110       setOperationAction(ISD::STRICT_FSQRT,       VT, Legal);
2111 
2112       setOperationAction(ISD::FFLOOR,             VT, Legal);
2113       setOperationAction(ISD::STRICT_FFLOOR,      VT, Legal);
2114       setOperationAction(ISD::FCEIL,              VT, Legal);
2115       setOperationAction(ISD::STRICT_FCEIL,       VT, Legal);
2116       setOperationAction(ISD::FTRUNC,             VT, Legal);
2117       setOperationAction(ISD::STRICT_FTRUNC,      VT, Legal);
2118       setOperationAction(ISD::FRINT,              VT, Legal);
2119       setOperationAction(ISD::STRICT_FRINT,       VT, Legal);
2120       setOperationAction(ISD::FNEARBYINT,         VT, Legal);
2121       setOperationAction(ISD::STRICT_FNEARBYINT,  VT, Legal);
2122       setOperationAction(ISD::FROUNDEVEN, VT, Legal);
2123       setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
2124 
2125       setOperationAction(ISD::FROUND,             VT, Custom);
2126 
2127       setOperationAction(ISD::LOAD,               VT, Legal);
2128       setOperationAction(ISD::STORE,              VT, Legal);
2129 
2130       setOperationAction(ISD::FMA,                VT, Legal);
2131       setOperationAction(ISD::STRICT_FMA,         VT, Legal);
2132       setOperationAction(ISD::VSELECT,            VT, Legal);
2133       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
2134       setOperationAction(ISD::SELECT,             VT, Custom);
2135 
2136       setOperationAction(ISD::FNEG,               VT, Custom);
2137       setOperationAction(ISD::FABS,               VT, Custom);
2138       setOperationAction(ISD::FCOPYSIGN,          VT, Custom);
2139       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2140       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
2141 
2142       setOperationAction(ISD::SETCC,              VT, Custom);
2143       setOperationAction(ISD::STRICT_FSETCC,      VT, Custom);
2144       setOperationAction(ISD::STRICT_FSETCCS,     VT, Custom);
2145     };
2146 
2147     // AVX512_FP16 scalar operations
2148     setGroup(MVT::f16);
2149     setOperationAction(ISD::FREM,                 MVT::f16, Promote);
2150     setOperationAction(ISD::STRICT_FREM,          MVT::f16, Promote);
2151     setOperationAction(ISD::SELECT_CC,            MVT::f16, Expand);
2152     setOperationAction(ISD::BR_CC,                MVT::f16, Expand);
2153     setOperationAction(ISD::STRICT_FROUND,        MVT::f16, Promote);
2154     setOperationAction(ISD::FROUNDEVEN,           MVT::f16, Legal);
2155     setOperationAction(ISD::STRICT_FROUNDEVEN,    MVT::f16, Legal);
2156     setOperationAction(ISD::FP_ROUND,             MVT::f16, Custom);
2157     setOperationAction(ISD::STRICT_FP_ROUND,      MVT::f16, Custom);
2158     setOperationAction(ISD::FMAXIMUM,             MVT::f16, Custom);
2159     setOperationAction(ISD::FMINIMUM,             MVT::f16, Custom);
2160     setOperationAction(ISD::FP_EXTEND,            MVT::f32, Legal);
2161     setOperationAction(ISD::STRICT_FP_EXTEND,     MVT::f32, Legal);
2162 
2163     setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);
2164     setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);
2165 
2166     if (Subtarget.useAVX512Regs()) {
2167       setGroup(MVT::v32f16);
2168       setOperationAction(ISD::SCALAR_TO_VECTOR,       MVT::v32f16, Custom);
2169       setOperationAction(ISD::SINT_TO_FP,             MVT::v32i16, Legal);
2170       setOperationAction(ISD::STRICT_SINT_TO_FP,      MVT::v32i16, Legal);
2171       setOperationAction(ISD::UINT_TO_FP,             MVT::v32i16, Legal);
2172       setOperationAction(ISD::STRICT_UINT_TO_FP,      MVT::v32i16, Legal);
2173       setOperationAction(ISD::FP_ROUND,               MVT::v16f16, Legal);
2174       setOperationAction(ISD::STRICT_FP_ROUND,        MVT::v16f16, Legal);
2175       setOperationAction(ISD::FP_EXTEND,              MVT::v16f32, Custom);
2176       setOperationAction(ISD::STRICT_FP_EXTEND,       MVT::v16f32, Legal);
2177       setOperationAction(ISD::FP_EXTEND,              MVT::v8f64,  Custom);
2178       setOperationAction(ISD::STRICT_FP_EXTEND,       MVT::v8f64,  Legal);
2179       setOperationAction(ISD::INSERT_VECTOR_ELT,      MVT::v32f16, Custom);
2180 
2181       setOperationAction(ISD::FP_TO_SINT,             MVT::v32i16, Custom);
2182       setOperationAction(ISD::STRICT_FP_TO_SINT,      MVT::v32i16, Custom);
2183       setOperationAction(ISD::FP_TO_UINT,             MVT::v32i16, Custom);
2184       setOperationAction(ISD::STRICT_FP_TO_UINT,      MVT::v32i16, Custom);
2185       setOperationPromotedToType(ISD::FP_TO_SINT,     MVT::v32i8,  MVT::v32i16);
2186       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,
2187                                  MVT::v32i16);
2188       setOperationPromotedToType(ISD::FP_TO_UINT,     MVT::v32i8,  MVT::v32i16);
2189       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,
2190                                  MVT::v32i16);
2191       setOperationPromotedToType(ISD::FP_TO_SINT,     MVT::v32i1,  MVT::v32i16);
2192       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,
2193                                  MVT::v32i16);
2194       setOperationPromotedToType(ISD::FP_TO_UINT,     MVT::v32i1,  MVT::v32i16);
2195       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,
2196                                  MVT::v32i16);
2197 
2198       setOperationAction(ISD::EXTRACT_SUBVECTOR,      MVT::v16f16, Legal);
2199       setOperationAction(ISD::INSERT_SUBVECTOR,       MVT::v32f16, Legal);
2200       setOperationAction(ISD::CONCAT_VECTORS,         MVT::v32f16, Custom);
2201 
2202       setLoadExtAction(ISD::EXTLOAD, MVT::v8f64,  MVT::v8f16,  Legal);
2203       setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);
2204     }
2205 
2206     if (Subtarget.hasVLX()) {
2207       setGroup(MVT::v8f16);
2208       setGroup(MVT::v16f16);
2209 
2210       setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v8f16,  Legal);
2211       setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v16f16, Custom);
2212       setOperationAction(ISD::SINT_TO_FP,         MVT::v16i16, Legal);
2213       setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v16i16, Legal);
2214       setOperationAction(ISD::SINT_TO_FP,         MVT::v8i16,  Legal);
2215       setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v8i16,  Legal);
2216       setOperationAction(ISD::UINT_TO_FP,         MVT::v16i16, Legal);
2217       setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v16i16, Legal);
2218       setOperationAction(ISD::UINT_TO_FP,         MVT::v8i16,  Legal);
2219       setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v8i16,  Legal);
2220 
2221       setOperationAction(ISD::FP_TO_SINT,         MVT::v8i16, Custom);
2222       setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v8i16, Custom);
2223       setOperationAction(ISD::FP_TO_UINT,         MVT::v8i16, Custom);
2224       setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i16, Custom);
2225       setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Legal);
2226       setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v8f16, Legal);
2227       setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Custom);
2228       setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v8f32, Legal);
2229       setOperationAction(ISD::FP_EXTEND,          MVT::v4f64, Custom);
2230       setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v4f64, Legal);
2231 
2232       // INSERT_VECTOR_ELT v8f16 extended to VECTOR_SHUFFLE
2233       setOperationAction(ISD::INSERT_VECTOR_ELT,    MVT::v8f16,  Custom);
2234       setOperationAction(ISD::INSERT_VECTOR_ELT,    MVT::v16f16, Custom);
2235 
2236       setOperationAction(ISD::EXTRACT_SUBVECTOR,    MVT::v8f16, Legal);
2237       setOperationAction(ISD::INSERT_SUBVECTOR,     MVT::v16f16, Legal);
2238       setOperationAction(ISD::CONCAT_VECTORS,       MVT::v16f16, Custom);
2239 
2240       setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);
2241       setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);
2242       setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Legal);
2243       setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);
2244 
2245       // Need to custom widen these to prevent scalarization.
2246       setOperationAction(ISD::LOAD,  MVT::v4f16, Custom);
2247       setOperationAction(ISD::STORE, MVT::v4f16, Custom);
2248     }
2249   }
2250 
2251   if (!Subtarget.useSoftFloat() &&
2252       (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16())) {
2253     addRegisterClass(MVT::v8bf16, Subtarget.hasAVX512() ? &X86::VR128XRegClass
2254                                                         : &X86::VR128RegClass);
2255     addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512() ? &X86::VR256XRegClass
2256                                                          : &X86::VR256RegClass);
2257     // We set the type action of bf16 to TypeSoftPromoteHalf, but we don't
2258     // provide the method to promote BUILD_VECTOR and INSERT_VECTOR_ELT.
2259     // Set the operation action Custom to do the customization later.
2260     setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
2261     setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
2262     for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
2263       setF16Action(VT, Expand);
2264       setOperationAction(ISD::FADD, VT, Expand);
2265       setOperationAction(ISD::FSUB, VT, Expand);
2266       setOperationAction(ISD::FMUL, VT, Expand);
2267       setOperationAction(ISD::FDIV, VT, Expand);
2268       setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2269       setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2270       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
2271       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2272     }
2273     setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom);
2274     addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
2275   }
2276 
2277   if (!Subtarget.useSoftFloat() && Subtarget.hasBF16()) {
2278     addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
2279     setF16Action(MVT::v32bf16, Expand);
2280     setOperationAction(ISD::FADD, MVT::v32bf16, Expand);
2281     setOperationAction(ISD::FSUB, MVT::v32bf16, Expand);
2282     setOperationAction(ISD::FMUL, MVT::v32bf16, Expand);
2283     setOperationAction(ISD::FDIV, MVT::v32bf16, Expand);
2284     setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom);
2285     setOperationAction(ISD::FP_ROUND, MVT::v16bf16, Custom);
2286     setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32bf16, Custom);
2287     setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32bf16, Legal);
2288     setOperationAction(ISD::CONCAT_VECTORS, MVT::v32bf16, Custom);
2289   }
2290 
2291   if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
2292     setTruncStoreAction(MVT::v4i64, MVT::v4i8,  Legal);
2293     setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
2294     setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
2295     setTruncStoreAction(MVT::v8i32, MVT::v8i8,  Legal);
2296     setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
2297 
2298     setTruncStoreAction(MVT::v2i64, MVT::v2i8,  Legal);
2299     setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
2300     setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
2301     setTruncStoreAction(MVT::v4i32, MVT::v4i8,  Legal);
2302     setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
2303 
2304     if (Subtarget.hasBWI()) {
2305       setTruncStoreAction(MVT::v16i16,  MVT::v16i8, Legal);
2306       setTruncStoreAction(MVT::v8i16,   MVT::v8i8,  Legal);
2307     }
2308 
2309     if (Subtarget.hasFP16()) {
2310       // vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64
2311       setOperationAction(ISD::FP_TO_SINT,        MVT::v2f16, Custom);
2312       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);
2313       setOperationAction(ISD::FP_TO_UINT,        MVT::v2f16, Custom);
2314       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);
2315       setOperationAction(ISD::FP_TO_SINT,        MVT::v4f16, Custom);
2316       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);
2317       setOperationAction(ISD::FP_TO_UINT,        MVT::v4f16, Custom);
2318       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);
2319       // vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16
2320       setOperationAction(ISD::SINT_TO_FP,        MVT::v2f16, Custom);
2321       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);
2322       setOperationAction(ISD::UINT_TO_FP,        MVT::v2f16, Custom);
2323       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);
2324       setOperationAction(ISD::SINT_TO_FP,        MVT::v4f16, Custom);
2325       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);
2326       setOperationAction(ISD::UINT_TO_FP,        MVT::v4f16, Custom);
2327       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);
2328       // vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16
2329       setOperationAction(ISD::FP_ROUND,          MVT::v2f16, Custom);
2330       setOperationAction(ISD::STRICT_FP_ROUND,   MVT::v2f16, Custom);
2331       setOperationAction(ISD::FP_ROUND,          MVT::v4f16, Custom);
2332       setOperationAction(ISD::STRICT_FP_ROUND,   MVT::v4f16, Custom);
2333       // vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32
2334       setOperationAction(ISD::FP_EXTEND,         MVT::v2f16, Custom);
2335       setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v2f16, Custom);
2336       setOperationAction(ISD::FP_EXTEND,         MVT::v4f16, Custom);
2337       setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v4f16, Custom);
2338     }
2339   }
2340 
2341   if (!Subtarget.useSoftFloat() && Subtarget.hasAMXTILE()) {
2342     addRegisterClass(MVT::x86amx, &X86::TILERegClass);
2343   }
2344 
2345   // We want to custom lower some of our intrinsics.
2346   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
2347   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
2348   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
2349   if (!Subtarget.is64Bit()) {
2350     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
2351   }
2352 
2353   // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
2354   // handle type legalization for these operations here.
2355   //
2356   // FIXME: We really should do custom legalization for addition and
2357   // subtraction on x86-32 once PR3203 is fixed.  We really can't do much better
2358   // than generic legalization for 64-bit multiplication-with-overflow, though.
2359   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
2360     if (VT == MVT::i64 && !Subtarget.is64Bit())
2361       continue;
2362     // Add/Sub/Mul with overflow operations are custom lowered.
2363     setOperationAction(ISD::SADDO, VT, Custom);
2364     setOperationAction(ISD::UADDO, VT, Custom);
2365     setOperationAction(ISD::SSUBO, VT, Custom);
2366     setOperationAction(ISD::USUBO, VT, Custom);
2367     setOperationAction(ISD::SMULO, VT, Custom);
2368     setOperationAction(ISD::UMULO, VT, Custom);
2369 
2370     // Support carry in as value rather than glue.
2371     setOperationAction(ISD::UADDO_CARRY, VT, Custom);
2372     setOperationAction(ISD::USUBO_CARRY, VT, Custom);
2373     setOperationAction(ISD::SETCCCARRY, VT, Custom);
2374     setOperationAction(ISD::SADDO_CARRY, VT, Custom);
2375     setOperationAction(ISD::SSUBO_CARRY, VT, Custom);
2376   }
2377 
2378   if (!Subtarget.is64Bit()) {
2379     // These libcalls are not available in 32-bit.
2380     setLibcallName(RTLIB::SHL_I128, nullptr);
2381     setLibcallName(RTLIB::SRL_I128, nullptr);
2382     setLibcallName(RTLIB::SRA_I128, nullptr);
2383     setLibcallName(RTLIB::MUL_I128, nullptr);
2384     // The MULO libcall is not part of libgcc, only compiler-rt.
2385     setLibcallName(RTLIB::MULO_I64, nullptr);
2386   }
2387   // The MULO libcall is not part of libgcc, only compiler-rt.
2388   setLibcallName(RTLIB::MULO_I128, nullptr);
2389 
2390   // Combine sin / cos into _sincos_stret if it is available.
2391   if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
2392       getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
2393     setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
2394     setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
2395   }
2396 
2397   if (Subtarget.isTargetWin64()) {
2398     setOperationAction(ISD::SDIV, MVT::i128, Custom);
2399     setOperationAction(ISD::UDIV, MVT::i128, Custom);
2400     setOperationAction(ISD::SREM, MVT::i128, Custom);
2401     setOperationAction(ISD::UREM, MVT::i128, Custom);
2402     setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
2403     setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
2404     setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
2405     setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
2406     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
2407     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
2408     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
2409     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
2410   }
2411 
2412   // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)`
2413   // is. We should promote the value to 64-bits to solve this.
2414   // This is what the CRT headers do - `fmodf` is an inline header
2415   // function casting to f64 and calling `fmod`.
2416   if (Subtarget.is32Bit() &&
2417       (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()))
2418     for (ISD::NodeType Op :
2419          {ISD::FCEIL,  ISD::STRICT_FCEIL,
2420           ISD::FCOS,   ISD::STRICT_FCOS,
2421           ISD::FEXP,   ISD::STRICT_FEXP,
2422           ISD::FFLOOR, ISD::STRICT_FFLOOR,
2423           ISD::FREM,   ISD::STRICT_FREM,
2424           ISD::FLOG,   ISD::STRICT_FLOG,
2425           ISD::FLOG10, ISD::STRICT_FLOG10,
2426           ISD::FPOW,   ISD::STRICT_FPOW,
2427           ISD::FSIN,   ISD::STRICT_FSIN})
2428       if (isOperationExpand(Op, MVT::f32))
2429         setOperationAction(Op, MVT::f32, Promote);
2430 
2431   // We have target-specific dag combine patterns for the following nodes:
2432   setTargetDAGCombine({ISD::VECTOR_SHUFFLE,
2433                        ISD::SCALAR_TO_VECTOR,
2434                        ISD::INSERT_VECTOR_ELT,
2435                        ISD::EXTRACT_VECTOR_ELT,
2436                        ISD::CONCAT_VECTORS,
2437                        ISD::INSERT_SUBVECTOR,
2438                        ISD::EXTRACT_SUBVECTOR,
2439                        ISD::BITCAST,
2440                        ISD::VSELECT,
2441                        ISD::SELECT,
2442                        ISD::SHL,
2443                        ISD::SRA,
2444                        ISD::SRL,
2445                        ISD::OR,
2446                        ISD::AND,
2447                        ISD::BITREVERSE,
2448                        ISD::ADD,
2449                        ISD::FADD,
2450                        ISD::FSUB,
2451                        ISD::FNEG,
2452                        ISD::FMA,
2453                        ISD::STRICT_FMA,
2454                        ISD::FMINNUM,
2455                        ISD::FMAXNUM,
2456                        ISD::SUB,
2457                        ISD::LOAD,
2458                        ISD::MLOAD,
2459                        ISD::STORE,
2460                        ISD::MSTORE,
2461                        ISD::TRUNCATE,
2462                        ISD::ZERO_EXTEND,
2463                        ISD::ANY_EXTEND,
2464                        ISD::SIGN_EXTEND,
2465                        ISD::SIGN_EXTEND_INREG,
2466                        ISD::ANY_EXTEND_VECTOR_INREG,
2467                        ISD::SIGN_EXTEND_VECTOR_INREG,
2468                        ISD::ZERO_EXTEND_VECTOR_INREG,
2469                        ISD::SINT_TO_FP,
2470                        ISD::UINT_TO_FP,
2471                        ISD::STRICT_SINT_TO_FP,
2472                        ISD::STRICT_UINT_TO_FP,
2473                        ISD::SETCC,
2474                        ISD::MUL,
2475                        ISD::XOR,
2476                        ISD::MSCATTER,
2477                        ISD::MGATHER,
2478                        ISD::FP16_TO_FP,
2479                        ISD::FP_EXTEND,
2480                        ISD::STRICT_FP_EXTEND,
2481                        ISD::FP_ROUND,
2482                        ISD::STRICT_FP_ROUND});
2483 
2484   computeRegisterProperties(Subtarget.getRegisterInfo());
2485 
2486   MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
2487   MaxStoresPerMemsetOptSize = 8;
2488   MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
2489   MaxStoresPerMemcpyOptSize = 4;
2490   MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
2491   MaxStoresPerMemmoveOptSize = 4;
2492 
2493   // TODO: These control memcmp expansion in CGP and could be raised higher, but
2494   // that needs to benchmarked and balanced with the potential use of vector
2495   // load/store types (PR33329, PR33914).
2496   MaxLoadsPerMemcmp = 2;
2497   MaxLoadsPerMemcmpOptSize = 2;
2498 
2499   // Default loop alignment, which can be overridden by -align-loops.
2500   setPrefLoopAlignment(Align(16));
2501 
2502   // An out-of-order CPU can speculatively execute past a predictable branch,
2503   // but a conditional move could be stalled by an expensive earlier operation.
2504   PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
2505   EnableExtLdPromotion = true;
2506   setPrefFunctionAlignment(Align(16));
2507 
2508   verifyIntrinsicTables();
2509 
2510   // Default to having -disable-strictnode-mutation on
2511   IsStrictFPEnabled = true;
2512 }
2513 
2514 // This has so far only been implemented for 64-bit MachO.
2515 bool X86TargetLowering::useLoadStackGuardNode() const {
2516   return Subtarget.isTargetMachO() && Subtarget.is64Bit();
2517 }
2518 
2519 bool X86TargetLowering::useStackGuardXorFP() const {
2520   // Currently only MSVC CRTs XOR the frame pointer into the stack guard value.
2521   return Subtarget.getTargetTriple().isOSMSVCRT() && !Subtarget.isTargetMachO();
2522 }
2523 
2524 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2525                                                const SDLoc &DL) const {
2526   EVT PtrTy = getPointerTy(DAG.getDataLayout());
2527   unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
2528   MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
2529   return SDValue(Node, 0);
2530 }
2531 
2532 TargetLoweringBase::LegalizeTypeAction
2533 X86TargetLowering::getPreferredVectorAction(MVT VT) const {
2534   if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() &&
2535       !Subtarget.hasBWI())
2536     return TypeSplitVector;
2537 
2538   if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2539       !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16)
2540     return TypeSplitVector;
2541 
2542   if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2543       VT.getVectorElementType() != MVT::i1)
2544     return TypeWidenVector;
2545 
2546   return TargetLoweringBase::getPreferredVectorAction(VT);
2547 }
2548 
2549 FastISel *
2550 X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
2551                                   const TargetLibraryInfo *libInfo) const {
2552   return X86::createFastISel(funcInfo, libInfo);
2553 }
2554 
2555 //===----------------------------------------------------------------------===//
2556 //                           Other Lowering Hooks
2557 //===----------------------------------------------------------------------===//
2558 
2559 bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
2560                       bool AssumeSingleUse) {
2561   if (!AssumeSingleUse && !Op.hasOneUse())
2562     return false;
2563   if (!ISD::isNormalLoad(Op.getNode()))
2564     return false;
2565 
2566   // If this is an unaligned vector, make sure the target supports folding it.
2567   auto *Ld = cast<LoadSDNode>(Op.getNode());
2568   if (!Subtarget.hasAVX() && !Subtarget.hasSSEUnalignedMem() &&
2569       Ld->getValueSizeInBits(0) == 128 && Ld->getAlign() < Align(16))
2570     return false;
2571 
2572   // TODO: If this is a non-temporal load and the target has an instruction
2573   //       for it, it should not be folded. See "useNonTemporalLoad()".
2574 
2575   return true;
2576 }
2577 
2578 bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
2579                                           const X86Subtarget &Subtarget,
2580                                           bool AssumeSingleUse) {
2581   assert(Subtarget.hasAVX() && "Expected AVX for broadcast from memory");
2582   if (!X86::mayFoldLoad(Op, Subtarget, AssumeSingleUse))
2583     return false;
2584 
2585   // We can not replace a wide volatile load with a broadcast-from-memory,
2586   // because that would narrow the load, which isn't legal for volatiles.
2587   auto *Ld = cast<LoadSDNode>(Op.getNode());
2588   return !Ld->isVolatile() ||
2589          Ld->getValueSizeInBits(0) == EltVT.getScalarSizeInBits();
2590 }
2591 
2592 bool X86::mayFoldIntoStore(SDValue Op) {
2593   return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
2594 }
2595 
2596 bool X86::mayFoldIntoZeroExtend(SDValue Op) {
2597   if (Op.hasOneUse()) {
2598     unsigned Opcode = Op.getNode()->use_begin()->getOpcode();
2599     return (ISD::ZERO_EXTEND == Opcode);
2600   }
2601   return false;
2602 }
2603 
2604 static bool isTargetShuffle(unsigned Opcode) {
2605   switch(Opcode) {
2606   default: return false;
2607   case X86ISD::BLENDI:
2608   case X86ISD::PSHUFB:
2609   case X86ISD::PSHUFD:
2610   case X86ISD::PSHUFHW:
2611   case X86ISD::PSHUFLW:
2612   case X86ISD::SHUFP:
2613   case X86ISD::INSERTPS:
2614   case X86ISD::EXTRQI:
2615   case X86ISD::INSERTQI:
2616   case X86ISD::VALIGN:
2617   case X86ISD::PALIGNR:
2618   case X86ISD::VSHLDQ:
2619   case X86ISD::VSRLDQ:
2620   case X86ISD::MOVLHPS:
2621   case X86ISD::MOVHLPS:
2622   case X86ISD::MOVSHDUP:
2623   case X86ISD::MOVSLDUP:
2624   case X86ISD::MOVDDUP:
2625   case X86ISD::MOVSS:
2626   case X86ISD::MOVSD:
2627   case X86ISD::MOVSH:
2628   case X86ISD::UNPCKL:
2629   case X86ISD::UNPCKH:
2630   case X86ISD::VBROADCAST:
2631   case X86ISD::VPERMILPI:
2632   case X86ISD::VPERMILPV:
2633   case X86ISD::VPERM2X128:
2634   case X86ISD::SHUF128:
2635   case X86ISD::VPERMIL2:
2636   case X86ISD::VPERMI:
2637   case X86ISD::VPPERM:
2638   case X86ISD::VPERMV:
2639   case X86ISD::VPERMV3:
2640   case X86ISD::VZEXT_MOVL:
2641     return true;
2642   }
2643 }
2644 
2645 static bool isTargetShuffleVariableMask(unsigned Opcode) {
2646   switch (Opcode) {
2647   default: return false;
2648   // Target Shuffles.
2649   case X86ISD::PSHUFB:
2650   case X86ISD::VPERMILPV:
2651   case X86ISD::VPERMIL2:
2652   case X86ISD::VPPERM:
2653   case X86ISD::VPERMV:
2654   case X86ISD::VPERMV3:
2655     return true;
2656   // 'Faux' Target Shuffles.
2657   case ISD::OR:
2658   case ISD::AND:
2659   case X86ISD::ANDNP:
2660     return true;
2661   }
2662 }
2663 
2664 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
2665   MachineFunction &MF = DAG.getMachineFunction();
2666   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
2667   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2668   int ReturnAddrIndex = FuncInfo->getRAIndex();
2669 
2670   if (ReturnAddrIndex == 0) {
2671     // Set up a frame object for the return address.
2672     unsigned SlotSize = RegInfo->getSlotSize();
2673     ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
2674                                                           -(int64_t)SlotSize,
2675                                                           false);
2676     FuncInfo->setRAIndex(ReturnAddrIndex);
2677   }
2678 
2679   return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
2680 }
2681 
2682 bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model CM,
2683                                        bool HasSymbolicDisplacement) {
2684   // Offset should fit into 32 bit immediate field.
2685   if (!isInt<32>(Offset))
2686     return false;
2687 
2688   // If we don't have a symbolic displacement - we don't have any extra
2689   // restrictions.
2690   if (!HasSymbolicDisplacement)
2691     return true;
2692 
2693   // We can fold large offsets in the large code model because we always use
2694   // 64-bit offsets.
2695   if (CM == CodeModel::Large)
2696     return true;
2697 
2698   // For kernel code model we know that all object resist in the negative half
2699   // of 32bits address space. We may not accept negative offsets, since they may
2700   // be just off and we may accept pretty large positive ones.
2701   if (CM == CodeModel::Kernel)
2702     return Offset >= 0;
2703 
2704   // For other non-large code models we assume that latest small object is 16MB
2705   // before end of 31 bits boundary. We may also accept pretty large negative
2706   // constants knowing that all objects are in the positive half of address
2707   // space.
2708   return Offset < 16 * 1024 * 1024;
2709 }
2710 
2711 /// Return true if the condition is an signed comparison operation.
2712 static bool isX86CCSigned(unsigned X86CC) {
2713   switch (X86CC) {
2714   default:
2715     llvm_unreachable("Invalid integer condition!");
2716   case X86::COND_E:
2717   case X86::COND_NE:
2718   case X86::COND_B:
2719   case X86::COND_A:
2720   case X86::COND_BE:
2721   case X86::COND_AE:
2722     return false;
2723   case X86::COND_G:
2724   case X86::COND_GE:
2725   case X86::COND_L:
2726   case X86::COND_LE:
2727     return true;
2728   }
2729 }
2730 
2731 static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
2732   switch (SetCCOpcode) {
2733   default: llvm_unreachable("Invalid integer condition!");
2734   case ISD::SETEQ:  return X86::COND_E;
2735   case ISD::SETGT:  return X86::COND_G;
2736   case ISD::SETGE:  return X86::COND_GE;
2737   case ISD::SETLT:  return X86::COND_L;
2738   case ISD::SETLE:  return X86::COND_LE;
2739   case ISD::SETNE:  return X86::COND_NE;
2740   case ISD::SETULT: return X86::COND_B;
2741   case ISD::SETUGT: return X86::COND_A;
2742   case ISD::SETULE: return X86::COND_BE;
2743   case ISD::SETUGE: return X86::COND_AE;
2744   }
2745 }
2746 
2747 /// Do a one-to-one translation of a ISD::CondCode to the X86-specific
2748 /// condition code, returning the condition code and the LHS/RHS of the
2749 /// comparison to make.
2750 static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
2751                                     bool isFP, SDValue &LHS, SDValue &RHS,
2752                                     SelectionDAG &DAG) {
2753   if (!isFP) {
2754     if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2755       if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnes()) {
2756         // X > -1   -> X == 0, jump !sign.
2757         RHS = DAG.getConstant(0, DL, RHS.getValueType());
2758         return X86::COND_NS;
2759       }
2760       if (SetCCOpcode == ISD::SETLT && RHSC->isZero()) {
2761         // X < 0   -> X == 0, jump on sign.
2762         return X86::COND_S;
2763       }
2764       if (SetCCOpcode == ISD::SETGE && RHSC->isZero()) {
2765         // X >= 0   -> X == 0, jump on !sign.
2766         return X86::COND_NS;
2767       }
2768       if (SetCCOpcode == ISD::SETLT && RHSC->isOne()) {
2769         // X < 1   -> X <= 0
2770         RHS = DAG.getConstant(0, DL, RHS.getValueType());
2771         return X86::COND_LE;
2772       }
2773     }
2774 
2775     return TranslateIntegerX86CC(SetCCOpcode);
2776   }
2777 
2778   // First determine if it is required or is profitable to flip the operands.
2779 
2780   // If LHS is a foldable load, but RHS is not, flip the condition.
2781   if (ISD::isNON_EXTLoad(LHS.getNode()) &&
2782       !ISD::isNON_EXTLoad(RHS.getNode())) {
2783     SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
2784     std::swap(LHS, RHS);
2785   }
2786 
2787   switch (SetCCOpcode) {
2788   default: break;
2789   case ISD::SETOLT:
2790   case ISD::SETOLE:
2791   case ISD::SETUGT:
2792   case ISD::SETUGE:
2793     std::swap(LHS, RHS);
2794     break;
2795   }
2796 
2797   // On a floating point condition, the flags are set as follows:
2798   // ZF  PF  CF   op
2799   //  0 | 0 | 0 | X > Y
2800   //  0 | 0 | 1 | X < Y
2801   //  1 | 0 | 0 | X == Y
2802   //  1 | 1 | 1 | unordered
2803   switch (SetCCOpcode) {
2804   default: llvm_unreachable("Condcode should be pre-legalized away");
2805   case ISD::SETUEQ:
2806   case ISD::SETEQ:   return X86::COND_E;
2807   case ISD::SETOLT:              // flipped
2808   case ISD::SETOGT:
2809   case ISD::SETGT:   return X86::COND_A;
2810   case ISD::SETOLE:              // flipped
2811   case ISD::SETOGE:
2812   case ISD::SETGE:   return X86::COND_AE;
2813   case ISD::SETUGT:              // flipped
2814   case ISD::SETULT:
2815   case ISD::SETLT:   return X86::COND_B;
2816   case ISD::SETUGE:              // flipped
2817   case ISD::SETULE:
2818   case ISD::SETLE:   return X86::COND_BE;
2819   case ISD::SETONE:
2820   case ISD::SETNE:   return X86::COND_NE;
2821   case ISD::SETUO:   return X86::COND_P;
2822   case ISD::SETO:    return X86::COND_NP;
2823   case ISD::SETOEQ:
2824   case ISD::SETUNE:  return X86::COND_INVALID;
2825   }
2826 }
2827 
2828 /// Is there a floating point cmov for the specific X86 condition code?
2829 /// Current x86 isa includes the following FP cmov instructions:
2830 /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
2831 static bool hasFPCMov(unsigned X86CC) {
2832   switch (X86CC) {
2833   default:
2834     return false;
2835   case X86::COND_B:
2836   case X86::COND_BE:
2837   case X86::COND_E:
2838   case X86::COND_P:
2839   case X86::COND_A:
2840   case X86::COND_AE:
2841   case X86::COND_NE:
2842   case X86::COND_NP:
2843     return true;
2844   }
2845 }
2846 
2847 static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
2848   return Subtarget.hasVLX() || Subtarget.canExtendTo512DQ() ||
2849          VT.is512BitVector();
2850 }
2851 
2852 bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
2853                                            const CallInst &I,
2854                                            MachineFunction &MF,
2855                                            unsigned Intrinsic) const {
2856   Info.flags = MachineMemOperand::MONone;
2857   Info.offset = 0;
2858 
2859   const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
2860   if (!IntrData) {
2861     switch (Intrinsic) {
2862     case Intrinsic::x86_aesenc128kl:
2863     case Intrinsic::x86_aesdec128kl:
2864       Info.opc = ISD::INTRINSIC_W_CHAIN;
2865       Info.ptrVal = I.getArgOperand(1);
2866       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2867       Info.align = Align(1);
2868       Info.flags |= MachineMemOperand::MOLoad;
2869       return true;
2870     case Intrinsic::x86_aesenc256kl:
2871     case Intrinsic::x86_aesdec256kl:
2872       Info.opc = ISD::INTRINSIC_W_CHAIN;
2873       Info.ptrVal = I.getArgOperand(1);
2874       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2875       Info.align = Align(1);
2876       Info.flags |= MachineMemOperand::MOLoad;
2877       return true;
2878     case Intrinsic::x86_aesencwide128kl:
2879     case Intrinsic::x86_aesdecwide128kl:
2880       Info.opc = ISD::INTRINSIC_W_CHAIN;
2881       Info.ptrVal = I.getArgOperand(0);
2882       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2883       Info.align = Align(1);
2884       Info.flags |= MachineMemOperand::MOLoad;
2885       return true;
2886     case Intrinsic::x86_aesencwide256kl:
2887     case Intrinsic::x86_aesdecwide256kl:
2888       Info.opc = ISD::INTRINSIC_W_CHAIN;
2889       Info.ptrVal = I.getArgOperand(0);
2890       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2891       Info.align = Align(1);
2892       Info.flags |= MachineMemOperand::MOLoad;
2893       return true;
2894     case Intrinsic::x86_cmpccxadd32:
2895     case Intrinsic::x86_cmpccxadd64:
2896     case Intrinsic::x86_atomic_bts:
2897     case Intrinsic::x86_atomic_btc:
2898     case Intrinsic::x86_atomic_btr: {
2899       Info.opc = ISD::INTRINSIC_W_CHAIN;
2900       Info.ptrVal = I.getArgOperand(0);
2901       unsigned Size = I.getType()->getScalarSizeInBits();
2902       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2903       Info.align = Align(Size);
2904       Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2905                     MachineMemOperand::MOVolatile;
2906       return true;
2907     }
2908     case Intrinsic::x86_atomic_bts_rm:
2909     case Intrinsic::x86_atomic_btc_rm:
2910     case Intrinsic::x86_atomic_btr_rm: {
2911       Info.opc = ISD::INTRINSIC_W_CHAIN;
2912       Info.ptrVal = I.getArgOperand(0);
2913       unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
2914       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2915       Info.align = Align(Size);
2916       Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2917                     MachineMemOperand::MOVolatile;
2918       return true;
2919     }
2920     case Intrinsic::x86_aadd32:
2921     case Intrinsic::x86_aadd64:
2922     case Intrinsic::x86_aand32:
2923     case Intrinsic::x86_aand64:
2924     case Intrinsic::x86_aor32:
2925     case Intrinsic::x86_aor64:
2926     case Intrinsic::x86_axor32:
2927     case Intrinsic::x86_axor64:
2928     case Intrinsic::x86_atomic_add_cc:
2929     case Intrinsic::x86_atomic_sub_cc:
2930     case Intrinsic::x86_atomic_or_cc:
2931     case Intrinsic::x86_atomic_and_cc:
2932     case Intrinsic::x86_atomic_xor_cc: {
2933       Info.opc = ISD::INTRINSIC_W_CHAIN;
2934       Info.ptrVal = I.getArgOperand(0);
2935       unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
2936       Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2937       Info.align = Align(Size);
2938       Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2939                     MachineMemOperand::MOVolatile;
2940       return true;
2941     }
2942     }
2943     return false;
2944   }
2945 
2946   switch (IntrData->Type) {
2947   case TRUNCATE_TO_MEM_VI8:
2948   case TRUNCATE_TO_MEM_VI16:
2949   case TRUNCATE_TO_MEM_VI32: {
2950     Info.opc = ISD::INTRINSIC_VOID;
2951     Info.ptrVal = I.getArgOperand(0);
2952     MVT VT  = MVT::getVT(I.getArgOperand(1)->getType());
2953     MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
2954     if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
2955       ScalarVT = MVT::i8;
2956     else if (IntrData->Type == TRUNCATE_TO_MEM_VI16)
2957       ScalarVT = MVT::i16;
2958     else if (IntrData->Type == TRUNCATE_TO_MEM_VI32)
2959       ScalarVT = MVT::i32;
2960 
2961     Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());
2962     Info.align = Align(1);
2963     Info.flags |= MachineMemOperand::MOStore;
2964     break;
2965   }
2966   case GATHER:
2967   case GATHER_AVX2: {
2968     Info.opc = ISD::INTRINSIC_W_CHAIN;
2969     Info.ptrVal = nullptr;
2970     MVT DataVT = MVT::getVT(I.getType());
2971     MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
2972     unsigned NumElts = std::min(DataVT.getVectorNumElements(),
2973                                 IndexVT.getVectorNumElements());
2974     Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
2975     Info.align = Align(1);
2976     Info.flags |= MachineMemOperand::MOLoad;
2977     break;
2978   }
2979   case SCATTER: {
2980     Info.opc = ISD::INTRINSIC_VOID;
2981     Info.ptrVal = nullptr;
2982     MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
2983     MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
2984     unsigned NumElts = std::min(DataVT.getVectorNumElements(),
2985                                 IndexVT.getVectorNumElements());
2986     Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
2987     Info.align = Align(1);
2988     Info.flags |= MachineMemOperand::MOStore;
2989     break;
2990   }
2991   default:
2992     return false;
2993   }
2994 
2995   return true;
2996 }
2997 
2998 /// Returns true if the target can instruction select the
2999 /// specified FP immediate natively. If false, the legalizer will
3000 /// materialize the FP immediate as a load from a constant pool.
3001 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3002                                      bool ForCodeSize) const {
3003   for (const APFloat &FPImm : LegalFPImmediates)
3004     if (Imm.bitwiseIsEqual(FPImm))
3005       return true;
3006   return false;
3007 }
3008 
3009 bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load,
3010                                               ISD::LoadExtType ExtTy,
3011                                               EVT NewVT) const {
3012   assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow");
3013 
3014   // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
3015   // relocation target a movq or addq instruction: don't let the load shrink.
3016   SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
3017   if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
3018     if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
3019       return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
3020 
3021   // If this is an (1) AVX vector load with (2) multiple uses and (3) all of
3022   // those uses are extracted directly into a store, then the extract + store
3023   // can be store-folded. Therefore, it's probably not worth splitting the load.
3024   EVT VT = Load->getValueType(0);
3025   if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) {
3026     for (auto UI = Load->use_begin(), UE = Load->use_end(); UI != UE; ++UI) {
3027       // Skip uses of the chain value. Result 0 of the node is the load value.
3028       if (UI.getUse().getResNo() != 0)
3029         continue;
3030 
3031       // If this use is not an extract + store, it's probably worth splitting.
3032       if (UI->getOpcode() != ISD::EXTRACT_SUBVECTOR || !UI->hasOneUse() ||
3033           UI->use_begin()->getOpcode() != ISD::STORE)
3034         return true;
3035     }
3036     // All non-chain uses are extract + store.
3037     return false;
3038   }
3039 
3040   return true;
3041 }
3042 
3043 /// Returns true if it is beneficial to convert a load of a constant
3044 /// to just the constant itself.
3045 bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3046                                                           Type *Ty) const {
3047   assert(Ty->isIntegerTy());
3048 
3049   unsigned BitSize = Ty->getPrimitiveSizeInBits();
3050   if (BitSize == 0 || BitSize > 64)
3051     return false;
3052   return true;
3053 }
3054 
3055 bool X86TargetLowering::reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
3056   // If we are using XMM registers in the ABI and the condition of the select is
3057   // a floating-point compare and we have blendv or conditional move, then it is
3058   // cheaper to select instead of doing a cross-register move and creating a
3059   // load that depends on the compare result.
3060   bool IsFPSetCC = CmpOpVT.isFloatingPoint() && CmpOpVT != MVT::f128;
3061   return !IsFPSetCC || !Subtarget.isTarget64BitLP64() || !Subtarget.hasAVX();
3062 }
3063 
3064 bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const {
3065   // TODO: It might be a win to ease or lift this restriction, but the generic
3066   // folds in DAGCombiner conflict with vector folds for an AVX512 target.
3067   if (VT.isVector() && Subtarget.hasAVX512())
3068     return false;
3069 
3070   return true;
3071 }
3072 
3073 bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
3074                                                SDValue C) const {
3075   // TODO: We handle scalars using custom code, but generic combining could make
3076   // that unnecessary.
3077   APInt MulC;
3078   if (!ISD::isConstantSplatVector(C.getNode(), MulC))
3079     return false;
3080 
3081   // Find the type this will be legalized too. Otherwise we might prematurely
3082   // convert this to shl+add/sub and then still have to type legalize those ops.
3083   // Another choice would be to defer the decision for illegal types until
3084   // after type legalization. But constant splat vectors of i64 can't make it
3085   // through type legalization on 32-bit targets so we would need to special
3086   // case vXi64.
3087   while (getTypeAction(Context, VT) != TypeLegal)
3088     VT = getTypeToTransformTo(Context, VT);
3089 
3090   // If vector multiply is legal, assume that's faster than shl + add/sub.
3091   // Multiply is a complex op with higher latency and lower throughput in
3092   // most implementations, sub-vXi32 vector multiplies are always fast,
3093   // vXi32 mustn't have a SlowMULLD implementation, and anything larger (vXi64)
3094   // is always going to be slow.
3095   unsigned EltSizeInBits = VT.getScalarSizeInBits();
3096   if (isOperationLegal(ISD::MUL, VT) && EltSizeInBits <= 32 &&
3097       (EltSizeInBits != 32 || !Subtarget.isPMULLDSlow()))
3098     return false;
3099 
3100   // shl+add, shl+sub, shl+add+neg
3101   return (MulC + 1).isPowerOf2() || (MulC - 1).isPowerOf2() ||
3102          (1 - MulC).isPowerOf2() || (-(MulC + 1)).isPowerOf2();
3103 }
3104 
3105 bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
3106                                                 unsigned Index) const {
3107   if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
3108     return false;
3109 
3110   // Mask vectors support all subregister combinations and operations that
3111   // extract half of vector.
3112   if (ResVT.getVectorElementType() == MVT::i1)
3113     return Index == 0 || ((ResVT.getSizeInBits() == SrcVT.getSizeInBits()*2) &&
3114                           (Index == ResVT.getVectorNumElements()));
3115 
3116   return (Index % ResVT.getVectorNumElements()) == 0;
3117 }
3118 
3119 bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
3120   unsigned Opc = VecOp.getOpcode();
3121 
3122   // Assume target opcodes can't be scalarized.
3123   // TODO - do we have any exceptions?
3124   if (Opc >= ISD::BUILTIN_OP_END)
3125     return false;
3126 
3127   // If the vector op is not supported, try to convert to scalar.
3128   EVT VecVT = VecOp.getValueType();
3129   if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
3130     return true;
3131 
3132   // If the vector op is supported, but the scalar op is not, the transform may
3133   // not be worthwhile.
3134   EVT ScalarVT = VecVT.getScalarType();
3135   return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
3136 }
3137 
3138 bool X86TargetLowering::shouldFormOverflowOp(unsigned Opcode, EVT VT,
3139                                              bool) const {
3140   // TODO: Allow vectors?
3141   if (VT.isVector())
3142     return false;
3143   return VT.isSimple() || !isOperationExpand(Opcode, VT);
3144 }
3145 
3146 bool X86TargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
3147   // Speculate cttz only if we can directly use TZCNT or can promote to i32.
3148   return Subtarget.hasBMI() ||
3149          (!Ty->isVectorTy() && Ty->getScalarSizeInBits() < 32);
3150 }
3151 
3152 bool X86TargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
3153   // Speculate ctlz only if we can directly use LZCNT.
3154   return Subtarget.hasLZCNT();
3155 }
3156 
3157 bool X86TargetLowering::ShouldShrinkFPConstant(EVT VT) const {
3158   // Don't shrink FP constpool if SSE2 is available since cvtss2sd is more
3159   // expensive than a straight movsd. On the other hand, it's important to
3160   // shrink long double fp constant since fldt is very slow.
3161   return !Subtarget.hasSSE2() || VT == MVT::f80;
3162 }
3163 
3164 bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const {
3165   return (VT == MVT::f64 && Subtarget.hasSSE2()) ||
3166          (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16;
3167 }
3168 
3169 bool X86TargetLowering::isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
3170                                                 const SelectionDAG &DAG,
3171                                                 const MachineMemOperand &MMO) const {
3172   if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() &&
3173       BitcastVT.getVectorElementType() == MVT::i1)
3174     return false;
3175 
3176   if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8)
3177     return false;
3178 
3179   // If both types are legal vectors, it's always ok to convert them.
3180   if (LoadVT.isVector() && BitcastVT.isVector() &&
3181       isTypeLegal(LoadVT) && isTypeLegal(BitcastVT))
3182     return true;
3183 
3184   return TargetLowering::isLoadBitCastBeneficial(LoadVT, BitcastVT, DAG, MMO);
3185 }
3186 
3187 bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
3188                                          const MachineFunction &MF) const {
3189   // Do not merge to float value size (128 bytes) if no implicit
3190   // float attribute is set.
3191   bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
3192 
3193   if (NoFloat) {
3194     unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
3195     return (MemVT.getSizeInBits() <= MaxIntSize);
3196   }
3197   // Make sure we don't merge greater than our preferred vector
3198   // width.
3199   if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth())
3200     return false;
3201 
3202   return true;
3203 }
3204 
3205 bool X86TargetLowering::isCtlzFast() const {
3206   return Subtarget.hasFastLZCNT();
3207 }
3208 
3209 bool X86TargetLowering::isMaskAndCmp0FoldingBeneficial(
3210     const Instruction &AndI) const {
3211   return true;
3212 }
3213 
3214 bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
3215   EVT VT = Y.getValueType();
3216 
3217   if (VT.isVector())
3218     return false;
3219 
3220   if (!Subtarget.hasBMI())
3221     return false;
3222 
3223   // There are only 32-bit and 64-bit forms for 'andn'.
3224   if (VT != MVT::i32 && VT != MVT::i64)
3225     return false;
3226 
3227   return !isa<ConstantSDNode>(Y);
3228 }
3229 
3230 bool X86TargetLowering::hasAndNot(SDValue Y) const {
3231   EVT VT = Y.getValueType();
3232 
3233   if (!VT.isVector())
3234     return hasAndNotCompare(Y);
3235 
3236   // Vector.
3237 
3238   if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128)
3239     return false;
3240 
3241   if (VT == MVT::v4i32)
3242     return true;
3243 
3244   return Subtarget.hasSSE2();
3245 }
3246 
3247 bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
3248   return X.getValueType().isScalarInteger(); // 'bt'
3249 }
3250 
3251 bool X86TargetLowering::
3252     shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3253         SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
3254         unsigned OldShiftOpcode, unsigned NewShiftOpcode,
3255         SelectionDAG &DAG) const {
3256   // Does baseline recommend not to perform the fold by default?
3257   if (!TargetLowering::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3258           X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG))
3259     return false;
3260   // For scalars this transform is always beneficial.
3261   if (X.getValueType().isScalarInteger())
3262     return true;
3263   // If all the shift amounts are identical, then transform is beneficial even
3264   // with rudimentary SSE2 shifts.
3265   if (DAG.isSplatValue(Y, /*AllowUndefs=*/true))
3266     return true;
3267   // If we have AVX2 with it's powerful shift operations, then it's also good.
3268   if (Subtarget.hasAVX2())
3269     return true;
3270   // Pre-AVX2 vector codegen for this pattern is best for variant with 'shl'.
3271   return NewShiftOpcode == ISD::SHL;
3272 }
3273 
3274 unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand(
3275     EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
3276     const APInt &ShiftOrRotateAmt, const std::optional<APInt> &AndMask) const {
3277   if (!VT.isInteger())
3278     return ShiftOpc;
3279 
3280   bool PreferRotate = false;
3281   if (VT.isVector()) {
3282     // For vectors, if we have rotate instruction support, then its definetly
3283     // best. Otherwise its not clear what the best so just don't make changed.
3284     PreferRotate = Subtarget.hasAVX512() && (VT.getScalarType() == MVT::i32 ||
3285                                              VT.getScalarType() == MVT::i64);
3286   } else {
3287     // For scalar, if we have bmi prefer rotate for rorx. Otherwise prefer
3288     // rotate unless we have a zext mask+shr.
3289     PreferRotate = Subtarget.hasBMI2();
3290     if (!PreferRotate) {
3291       unsigned MaskBits =
3292           VT.getScalarSizeInBits() - ShiftOrRotateAmt.getZExtValue();
3293       PreferRotate = (MaskBits != 8) && (MaskBits != 16) && (MaskBits != 32);
3294     }
3295   }
3296 
3297   if (ShiftOpc == ISD::SHL || ShiftOpc == ISD::SRL) {
3298     assert(AndMask.has_value() && "Null andmask when querying about shift+and");
3299 
3300     if (PreferRotate && MayTransformRotate)
3301       return ISD::ROTL;
3302 
3303     // If vector we don't really get much benefit swapping around constants.
3304     // Maybe we could check if the DAG has the flipped node already in the
3305     // future.
3306     if (VT.isVector())
3307       return ShiftOpc;
3308 
3309     // See if the beneficial to swap shift type.
3310     if (ShiftOpc == ISD::SHL) {
3311       // If the current setup has imm64 mask, then inverse will have
3312       // at least imm32 mask (or be zext i32 -> i64).
3313       if (VT == MVT::i64)
3314         return AndMask->getSignificantBits() > 32 ? (unsigned)ISD::SRL
3315                                                   : ShiftOpc;
3316 
3317       // We can only benefit if req at least 7-bit for the mask. We
3318       // don't want to replace shl of 1,2,3 as they can be implemented
3319       // with lea/add.
3320       return ShiftOrRotateAmt.uge(7) ? (unsigned)ISD::SRL : ShiftOpc;
3321     }
3322 
3323     if (VT == MVT::i64)
3324       // Keep exactly 32-bit imm64, this is zext i32 -> i64 which is
3325       // extremely efficient.
3326       return AndMask->getSignificantBits() > 33 ? (unsigned)ISD::SHL : ShiftOpc;
3327 
3328     // Keep small shifts as shl so we can generate add/lea.
3329     return ShiftOrRotateAmt.ult(7) ? (unsigned)ISD::SHL : ShiftOpc;
3330   }
3331 
3332   // We prefer rotate for vectors of if we won't get a zext mask with SRL
3333   // (PreferRotate will be set in the latter case).
3334   if (PreferRotate || VT.isVector())
3335     return ShiftOpc;
3336 
3337   // Non-vector type and we have a zext mask with SRL.
3338   return ISD::SRL;
3339 }
3340 
3341 bool X86TargetLowering::preferScalarizeSplat(SDNode *N) const {
3342   return N->getOpcode() != ISD::FP_EXTEND;
3343 }
3344 
3345 bool X86TargetLowering::shouldFoldConstantShiftPairToMask(
3346     const SDNode *N, CombineLevel Level) const {
3347   assert(((N->getOpcode() == ISD::SHL &&
3348            N->getOperand(0).getOpcode() == ISD::SRL) ||
3349           (N->getOpcode() == ISD::SRL &&
3350            N->getOperand(0).getOpcode() == ISD::SHL)) &&
3351          "Expected shift-shift mask");
3352   // TODO: Should we always create i64 masks? Or only folded immediates?
3353   EVT VT = N->getValueType(0);
3354   if ((Subtarget.hasFastVectorShiftMasks() && VT.isVector()) ||
3355       (Subtarget.hasFastScalarShiftMasks() && !VT.isVector())) {
3356     // Only fold if the shift values are equal - so it folds to AND.
3357     // TODO - we should fold if either is a non-uniform vector but we don't do
3358     // the fold for non-splats yet.
3359     return N->getOperand(1) == N->getOperand(0).getOperand(1);
3360   }
3361   return TargetLoweringBase::shouldFoldConstantShiftPairToMask(N, Level);
3362 }
3363 
3364 bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
3365   EVT VT = Y.getValueType();
3366 
3367   // For vectors, we don't have a preference, but we probably want a mask.
3368   if (VT.isVector())
3369     return false;
3370 
3371   // 64-bit shifts on 32-bit targets produce really bad bloated code.
3372   if (VT == MVT::i64 && !Subtarget.is64Bit())
3373     return false;
3374 
3375   return true;
3376 }
3377 
3378 TargetLowering::ShiftLegalizationStrategy
3379 X86TargetLowering::preferredShiftLegalizationStrategy(
3380     SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
3381   if (DAG.getMachineFunction().getFunction().hasMinSize() &&
3382       !Subtarget.isOSWindows())
3383     return ShiftLegalizationStrategy::LowerToLibcall;
3384   return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
3385                                                             ExpansionFactor);
3386 }
3387 
3388 bool X86TargetLowering::shouldSplatInsEltVarIndex(EVT VT) const {
3389   // Any legal vector type can be splatted more efficiently than
3390   // loading/spilling from memory.
3391   return isTypeLegal(VT);
3392 }
3393 
3394 MVT X86TargetLowering::hasFastEqualityCompare(unsigned NumBits) const {
3395   MVT VT = MVT::getIntegerVT(NumBits);
3396   if (isTypeLegal(VT))
3397     return VT;
3398 
3399   // PMOVMSKB can handle this.
3400   if (NumBits == 128 && isTypeLegal(MVT::v16i8))
3401     return MVT::v16i8;
3402 
3403   // VPMOVMSKB can handle this.
3404   if (NumBits == 256 && isTypeLegal(MVT::v32i8))
3405     return MVT::v32i8;
3406 
3407   // TODO: Allow 64-bit type for 32-bit target.
3408   // TODO: 512-bit types should be allowed, but make sure that those
3409   // cases are handled in combineVectorSizedSetCCEquality().
3410 
3411   return MVT::INVALID_SIMPLE_VALUE_TYPE;
3412 }
3413 
3414 /// Val is the undef sentinel value or equal to the specified value.
3415 static bool isUndefOrEqual(int Val, int CmpVal) {
3416   return ((Val == SM_SentinelUndef) || (Val == CmpVal));
3417 }
3418 
3419 /// Return true if every element in Mask is the undef sentinel value or equal to
3420 /// the specified value.
3421 static bool isUndefOrEqual(ArrayRef<int> Mask, int CmpVal) {
3422   return llvm::all_of(Mask, [CmpVal](int M) {
3423     return (M == SM_SentinelUndef) || (M == CmpVal);
3424   });
3425 }
3426 
3427 /// Return true if every element in Mask, beginning from position Pos and ending
3428 /// in Pos+Size is the undef sentinel value or equal to the specified value.
3429 static bool isUndefOrEqualInRange(ArrayRef<int> Mask, int CmpVal, unsigned Pos,
3430                                   unsigned Size) {
3431   return llvm::all_of(Mask.slice(Pos, Size),
3432                       [CmpVal](int M) { return isUndefOrEqual(M, CmpVal); });
3433 }
3434 
3435 /// Val is either the undef or zero sentinel value.
3436 static bool isUndefOrZero(int Val) {
3437   return ((Val == SM_SentinelUndef) || (Val == SM_SentinelZero));
3438 }
3439 
3440 /// Return true if every element in Mask, beginning from position Pos and ending
3441 /// in Pos+Size is the undef sentinel value.
3442 static bool isUndefInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) {
3443   return llvm::all_of(Mask.slice(Pos, Size),
3444                       [](int M) { return M == SM_SentinelUndef; });
3445 }
3446 
3447 /// Return true if the mask creates a vector whose lower half is undefined.
3448 static bool isUndefLowerHalf(ArrayRef<int> Mask) {
3449   unsigned NumElts = Mask.size();
3450   return isUndefInRange(Mask, 0, NumElts / 2);
3451 }
3452 
3453 /// Return true if the mask creates a vector whose upper half is undefined.
3454 static bool isUndefUpperHalf(ArrayRef<int> Mask) {
3455   unsigned NumElts = Mask.size();
3456   return isUndefInRange(Mask, NumElts / 2, NumElts / 2);
3457 }
3458 
3459 /// Return true if Val falls within the specified range (L, H].
3460 static bool isInRange(int Val, int Low, int Hi) {
3461   return (Val >= Low && Val < Hi);
3462 }
3463 
3464 /// Return true if the value of any element in Mask falls within the specified
3465 /// range (L, H].
3466 static bool isAnyInRange(ArrayRef<int> Mask, int Low, int Hi) {
3467   return llvm::any_of(Mask, [Low, Hi](int M) { return isInRange(M, Low, Hi); });
3468 }
3469 
3470 /// Return true if the value of any element in Mask is the zero sentinel value.
3471 static bool isAnyZero(ArrayRef<int> Mask) {
3472   return llvm::any_of(Mask, [](int M) { return M == SM_SentinelZero; });
3473 }
3474 
3475 /// Return true if the value of any element in Mask is the zero or undef
3476 /// sentinel values.
3477 static bool isAnyZeroOrUndef(ArrayRef<int> Mask) {
3478   return llvm::any_of(Mask, [](int M) {
3479     return M == SM_SentinelZero || M == SM_SentinelUndef;
3480   });
3481 }
3482 
3483 /// Return true if Val is undef or if its value falls within the
3484 /// specified range (L, H].
3485 static bool isUndefOrInRange(int Val, int Low, int Hi) {
3486   return (Val == SM_SentinelUndef) || isInRange(Val, Low, Hi);
3487 }
3488 
3489 /// Return true if every element in Mask is undef or if its value
3490 /// falls within the specified range (L, H].
3491 static bool isUndefOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
3492   return llvm::all_of(
3493       Mask, [Low, Hi](int M) { return isUndefOrInRange(M, Low, Hi); });
3494 }
3495 
3496 /// Return true if Val is undef, zero or if its value falls within the
3497 /// specified range (L, H].
3498 static bool isUndefOrZeroOrInRange(int Val, int Low, int Hi) {
3499   return isUndefOrZero(Val) || isInRange(Val, Low, Hi);
3500 }
3501 
3502 /// Return true if every element in Mask is undef, zero or if its value
3503 /// falls within the specified range (L, H].
3504 static bool isUndefOrZeroOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
3505   return llvm::all_of(
3506       Mask, [Low, Hi](int M) { return isUndefOrZeroOrInRange(M, Low, Hi); });
3507 }
3508 
3509 /// Return true if every element in Mask, beginning
3510 /// from position Pos and ending in Pos + Size, falls within the specified
3511 /// sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
3512 static bool isSequentialOrUndefInRange(ArrayRef<int> Mask, unsigned Pos,
3513                                        unsigned Size, int Low, int Step = 1) {
3514   for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
3515     if (!isUndefOrEqual(Mask[i], Low))
3516       return false;
3517   return true;
3518 }
3519 
3520 /// Return true if every element in Mask, beginning
3521 /// from position Pos and ending in Pos+Size, falls within the specified
3522 /// sequential range (Low, Low+Size], or is undef or is zero.
3523 static bool isSequentialOrUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
3524                                              unsigned Size, int Low,
3525                                              int Step = 1) {
3526   for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
3527     if (!isUndefOrZero(Mask[i]) && Mask[i] != Low)
3528       return false;
3529   return true;
3530 }
3531 
3532 /// Return true if every element in Mask, beginning
3533 /// from position Pos and ending in Pos+Size is undef or is zero.
3534 static bool isUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
3535                                  unsigned Size) {
3536   return llvm::all_of(Mask.slice(Pos, Size), isUndefOrZero);
3537 }
3538 
3539 /// Helper function to test whether a shuffle mask could be
3540 /// simplified by widening the elements being shuffled.
3541 ///
3542 /// Appends the mask for wider elements in WidenedMask if valid. Otherwise
3543 /// leaves it in an unspecified state.
3544 ///
3545 /// NOTE: This must handle normal vector shuffle masks and *target* vector
3546 /// shuffle masks. The latter have the special property of a '-2' representing
3547 /// a zero-ed lane of a vector.
3548 static bool canWidenShuffleElements(ArrayRef<int> Mask,
3549                                     SmallVectorImpl<int> &WidenedMask) {
3550   WidenedMask.assign(Mask.size() / 2, 0);
3551   for (int i = 0, Size = Mask.size(); i < Size; i += 2) {
3552     int M0 = Mask[i];
3553     int M1 = Mask[i + 1];
3554 
3555     // If both elements are undef, its trivial.
3556     if (M0 == SM_SentinelUndef && M1 == SM_SentinelUndef) {
3557       WidenedMask[i / 2] = SM_SentinelUndef;
3558       continue;
3559     }
3560 
3561     // Check for an undef mask and a mask value properly aligned to fit with
3562     // a pair of values. If we find such a case, use the non-undef mask's value.
3563     if (M0 == SM_SentinelUndef && M1 >= 0 && (M1 % 2) == 1) {
3564       WidenedMask[i / 2] = M1 / 2;
3565       continue;
3566     }
3567     if (M1 == SM_SentinelUndef && M0 >= 0 && (M0 % 2) == 0) {
3568       WidenedMask[i / 2] = M0 / 2;
3569       continue;
3570     }
3571 
3572     // When zeroing, we need to spread the zeroing across both lanes to widen.
3573     if (M0 == SM_SentinelZero || M1 == SM_SentinelZero) {
3574       if ((M0 == SM_SentinelZero || M0 == SM_SentinelUndef) &&
3575           (M1 == SM_SentinelZero || M1 == SM_SentinelUndef)) {
3576         WidenedMask[i / 2] = SM_SentinelZero;
3577         continue;
3578       }
3579       return false;
3580     }
3581 
3582     // Finally check if the two mask values are adjacent and aligned with
3583     // a pair.
3584     if (M0 != SM_SentinelUndef && (M0 % 2) == 0 && (M0 + 1) == M1) {
3585       WidenedMask[i / 2] = M0 / 2;
3586       continue;
3587     }
3588 
3589     // Otherwise we can't safely widen the elements used in this shuffle.
3590     return false;
3591   }
3592   assert(WidenedMask.size() == Mask.size() / 2 &&
3593          "Incorrect size of mask after widening the elements!");
3594 
3595   return true;
3596 }
3597 
3598 static bool canWidenShuffleElements(ArrayRef<int> Mask,
3599                                     const APInt &Zeroable,
3600                                     bool V2IsZero,
3601                                     SmallVectorImpl<int> &WidenedMask) {
3602   // Create an alternative mask with info about zeroable elements.
3603   // Here we do not set undef elements as zeroable.
3604   SmallVector<int, 64> ZeroableMask(Mask);
3605   if (V2IsZero) {
3606     assert(!Zeroable.isZero() && "V2's non-undef elements are used?!");
3607     for (int i = 0, Size = Mask.size(); i != Size; ++i)
3608       if (Mask[i] != SM_SentinelUndef && Zeroable[i])
3609         ZeroableMask[i] = SM_SentinelZero;
3610   }
3611   return canWidenShuffleElements(ZeroableMask, WidenedMask);
3612 }
3613 
3614 static bool canWidenShuffleElements(ArrayRef<int> Mask) {
3615   SmallVector<int, 32> WidenedMask;
3616   return canWidenShuffleElements(Mask, WidenedMask);
3617 }
3618 
3619 // Attempt to narrow/widen shuffle mask until it matches the target number of
3620 // elements.
3621 static bool scaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts,
3622                                  SmallVectorImpl<int> &ScaledMask) {
3623   unsigned NumSrcElts = Mask.size();
3624   assert(((NumSrcElts % NumDstElts) == 0 || (NumDstElts % NumSrcElts) == 0) &&
3625          "Illegal shuffle scale factor");
3626 
3627   // Narrowing is guaranteed to work.
3628   if (NumDstElts >= NumSrcElts) {
3629     int Scale = NumDstElts / NumSrcElts;
3630     llvm::narrowShuffleMaskElts(Scale, Mask, ScaledMask);
3631     return true;
3632   }
3633 
3634   // We have to repeat the widening until we reach the target size, but we can
3635   // split out the first widening as it sets up ScaledMask for us.
3636   if (canWidenShuffleElements(Mask, ScaledMask)) {
3637     while (ScaledMask.size() > NumDstElts) {
3638       SmallVector<int, 16> WidenedMask;
3639       if (!canWidenShuffleElements(ScaledMask, WidenedMask))
3640         return false;
3641       ScaledMask = std::move(WidenedMask);
3642     }
3643     return true;
3644   }
3645 
3646   return false;
3647 }
3648 
3649 /// Returns true if Elt is a constant zero or a floating point constant +0.0.
3650 bool X86::isZeroNode(SDValue Elt) {
3651   return isNullConstant(Elt) || isNullFPConstant(Elt);
3652 }
3653 
3654 // Build a vector of constants.
3655 // Use an UNDEF node if MaskElt == -1.
3656 // Split 64-bit constants in the 32-bit mode.
3657 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
3658                               const SDLoc &dl, bool IsMask = false) {
3659 
3660   SmallVector<SDValue, 32>  Ops;
3661   bool Split = false;
3662 
3663   MVT ConstVecVT = VT;
3664   unsigned NumElts = VT.getVectorNumElements();
3665   bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
3666   if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
3667     ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
3668     Split = true;
3669   }
3670 
3671   MVT EltVT = ConstVecVT.getVectorElementType();
3672   for (unsigned i = 0; i < NumElts; ++i) {
3673     bool IsUndef = Values[i] < 0 && IsMask;
3674     SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
3675       DAG.getConstant(Values[i], dl, EltVT);
3676     Ops.push_back(OpNode);
3677     if (Split)
3678       Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) :
3679                     DAG.getConstant(0, dl, EltVT));
3680   }
3681   SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
3682   if (Split)
3683     ConstsNode = DAG.getBitcast(VT, ConstsNode);
3684   return ConstsNode;
3685 }
3686 
3687 static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs,
3688                               MVT VT, SelectionDAG &DAG, const SDLoc &dl) {
3689   assert(Bits.size() == Undefs.getBitWidth() &&
3690          "Unequal constant and undef arrays");
3691   SmallVector<SDValue, 32> Ops;
3692   bool Split = false;
3693 
3694   MVT ConstVecVT = VT;
3695   unsigned NumElts = VT.getVectorNumElements();
3696   bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
3697   if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
3698     ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
3699     Split = true;
3700   }
3701 
3702   MVT EltVT = ConstVecVT.getVectorElementType();
3703   for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
3704     if (Undefs[i]) {
3705       Ops.append(Split ? 2 : 1, DAG.getUNDEF(EltVT));
3706       continue;
3707     }
3708     const APInt &V = Bits[i];
3709     assert(V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes");
3710     if (Split) {
3711       Ops.push_back(DAG.getConstant(V.trunc(32), dl, EltVT));
3712       Ops.push_back(DAG.getConstant(V.lshr(32).trunc(32), dl, EltVT));
3713     } else if (EltVT == MVT::f32) {
3714       APFloat FV(APFloat::IEEEsingle(), V);
3715       Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
3716     } else if (EltVT == MVT::f64) {
3717       APFloat FV(APFloat::IEEEdouble(), V);
3718       Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
3719     } else {
3720       Ops.push_back(DAG.getConstant(V, dl, EltVT));
3721     }
3722   }
3723 
3724   SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
3725   return DAG.getBitcast(VT, ConstsNode);
3726 }
3727 
3728 static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT,
3729                               SelectionDAG &DAG, const SDLoc &dl) {
3730   APInt Undefs = APInt::getZero(Bits.size());
3731   return getConstVector(Bits, Undefs, VT, DAG, dl);
3732 }
3733 
3734 /// Returns a vector of specified type with all zero elements.
3735 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,
3736                              SelectionDAG &DAG, const SDLoc &dl) {
3737   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() ||
3738           VT.getVectorElementType() == MVT::i1) &&
3739          "Unexpected vector type");
3740 
3741   // Try to build SSE/AVX zero vectors as <N x i32> bitcasted to their dest
3742   // type. This ensures they get CSE'd. But if the integer type is not
3743   // available, use a floating-point +0.0 instead.
3744   SDValue Vec;
3745   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3746   if (!Subtarget.hasSSE2() && VT.is128BitVector()) {
3747     Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32);
3748   } else if (VT.isFloatingPoint() &&
3749              TLI.isTypeLegal(VT.getVectorElementType())) {
3750     Vec = DAG.getConstantFP(+0.0, dl, VT);
3751   } else if (VT.getVectorElementType() == MVT::i1) {
3752     assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) &&
3753            "Unexpected vector type");
3754     Vec = DAG.getConstant(0, dl, VT);
3755   } else {
3756     unsigned Num32BitElts = VT.getSizeInBits() / 32;
3757     Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts));
3758   }
3759   return DAG.getBitcast(VT, Vec);
3760 }
3761 
3762 // Helper to determine if the ops are all the extracted subvectors come from a
3763 // single source. If we allow commute they don't have to be in order (Lo/Hi).
3764 static SDValue getSplitVectorSrc(SDValue LHS, SDValue RHS, bool AllowCommute) {
3765   if (LHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
3766       RHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
3767       LHS.getValueType() != RHS.getValueType() ||
3768       LHS.getOperand(0) != RHS.getOperand(0))
3769     return SDValue();
3770 
3771   SDValue Src = LHS.getOperand(0);
3772   if (Src.getValueSizeInBits() != (LHS.getValueSizeInBits() * 2))
3773     return SDValue();
3774 
3775   unsigned NumElts = LHS.getValueType().getVectorNumElements();
3776   if ((LHS.getConstantOperandAPInt(1) == 0 &&
3777        RHS.getConstantOperandAPInt(1) == NumElts) ||
3778       (AllowCommute && RHS.getConstantOperandAPInt(1) == 0 &&
3779        LHS.getConstantOperandAPInt(1) == NumElts))
3780     return Src;
3781 
3782   return SDValue();
3783 }
3784 
3785 static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
3786                                 const SDLoc &dl, unsigned vectorWidth) {
3787   EVT VT = Vec.getValueType();
3788   EVT ElVT = VT.getVectorElementType();
3789   unsigned Factor = VT.getSizeInBits() / vectorWidth;
3790   EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
3791                                   VT.getVectorNumElements() / Factor);
3792 
3793   // Extract the relevant vectorWidth bits.  Generate an EXTRACT_SUBVECTOR
3794   unsigned ElemsPerChunk = vectorWidth / ElVT.getSizeInBits();
3795   assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3796 
3797   // This is the index of the first element of the vectorWidth-bit chunk
3798   // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3799   IdxVal &= ~(ElemsPerChunk - 1);
3800 
3801   // If the input is a buildvector just emit a smaller one.
3802   if (Vec.getOpcode() == ISD::BUILD_VECTOR)
3803     return DAG.getBuildVector(ResultVT, dl,
3804                               Vec->ops().slice(IdxVal, ElemsPerChunk));
3805 
3806   // Check if we're extracting the upper undef of a widening pattern.
3807   if (Vec.getOpcode() == ISD::INSERT_SUBVECTOR && Vec.getOperand(0).isUndef() &&
3808       Vec.getOperand(1).getValueType().getVectorNumElements() <= IdxVal &&
3809       isNullConstant(Vec.getOperand(2)))
3810     return DAG.getUNDEF(ResultVT);
3811 
3812   SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
3813   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
3814 }
3815 
3816 /// Generate a DAG to grab 128-bits from a vector > 128 bits.  This
3817 /// sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128
3818 /// or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4
3819 /// instructions or a simple subregister reference. Idx is an index in the
3820 /// 128 bits we want.  It need not be aligned to a 128-bit boundary.  That makes
3821 /// lowering EXTRACT_VECTOR_ELT operations easier.
3822 static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
3823                                    SelectionDAG &DAG, const SDLoc &dl) {
3824   assert((Vec.getValueType().is256BitVector() ||
3825           Vec.getValueType().is512BitVector()) && "Unexpected vector size!");
3826   return extractSubVector(Vec, IdxVal, DAG, dl, 128);
3827 }
3828 
3829 /// Generate a DAG to grab 256-bits from a 512-bit vector.
3830 static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
3831                                    SelectionDAG &DAG, const SDLoc &dl) {
3832   assert(Vec.getValueType().is512BitVector() && "Unexpected vector size!");
3833   return extractSubVector(Vec, IdxVal, DAG, dl, 256);
3834 }
3835 
3836 static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
3837                                SelectionDAG &DAG, const SDLoc &dl,
3838                                unsigned vectorWidth) {
3839   assert((vectorWidth == 128 || vectorWidth == 256) &&
3840          "Unsupported vector width");
3841   // Inserting UNDEF is Result
3842   if (Vec.isUndef())
3843     return Result;
3844   EVT VT = Vec.getValueType();
3845   EVT ElVT = VT.getVectorElementType();
3846   EVT ResultVT = Result.getValueType();
3847 
3848   // Insert the relevant vectorWidth bits.
3849   unsigned ElemsPerChunk = vectorWidth/ElVT.getSizeInBits();
3850   assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3851 
3852   // This is the index of the first element of the vectorWidth-bit chunk
3853   // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3854   IdxVal &= ~(ElemsPerChunk - 1);
3855 
3856   SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
3857   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
3858 }
3859 
3860 /// Generate a DAG to put 128-bits into a vector > 128 bits.  This
3861 /// sets things up to match to an AVX VINSERTF128/VINSERTI128 or
3862 /// AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a
3863 /// simple superregister reference.  Idx is an index in the 128 bits
3864 /// we want.  It need not be aligned to a 128-bit boundary.  That makes
3865 /// lowering INSERT_VECTOR_ELT operations easier.
3866 static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
3867                                   SelectionDAG &DAG, const SDLoc &dl) {
3868   assert(Vec.getValueType().is128BitVector() && "Unexpected vector size!");
3869   return insertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
3870 }
3871 
3872 /// Widen a vector to a larger size with the same scalar type, with the new
3873 /// elements either zero or undef.
3874 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
3875                               const X86Subtarget &Subtarget, SelectionDAG &DAG,
3876                               const SDLoc &dl) {
3877   assert(Vec.getValueSizeInBits().getFixedValue() <= VT.getFixedSizeInBits() &&
3878          Vec.getValueType().getScalarType() == VT.getScalarType() &&
3879          "Unsupported vector widening type");
3880   SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
3881                                 : DAG.getUNDEF(VT);
3882   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec,
3883                      DAG.getIntPtrConstant(0, dl));
3884 }
3885 
3886 /// Widen a vector to a larger size with the same scalar type, with the new
3887 /// elements either zero or undef.
3888 static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
3889                               const X86Subtarget &Subtarget, SelectionDAG &DAG,
3890                               const SDLoc &dl, unsigned WideSizeInBits) {
3891   assert(Vec.getValueSizeInBits() <= WideSizeInBits &&
3892          (WideSizeInBits % Vec.getScalarValueSizeInBits()) == 0 &&
3893          "Unsupported vector widening type");
3894   unsigned WideNumElts = WideSizeInBits / Vec.getScalarValueSizeInBits();
3895   MVT SVT = Vec.getSimpleValueType().getScalarType();
3896   MVT VT = MVT::getVectorVT(SVT, WideNumElts);
3897   return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
3898 }
3899 
3900 /// Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT
3901 /// and bitcast with integer types.
3902 static MVT widenMaskVectorType(MVT VT, const X86Subtarget &Subtarget) {
3903   assert(VT.getVectorElementType() == MVT::i1 && "Expected bool vector");
3904   unsigned NumElts = VT.getVectorNumElements();
3905   if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8)
3906     return Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
3907   return VT;
3908 }
3909 
3910 /// Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and
3911 /// bitcast with integer types.
3912 static SDValue widenMaskVector(SDValue Vec, bool ZeroNewElements,
3913                                const X86Subtarget &Subtarget, SelectionDAG &DAG,
3914                                const SDLoc &dl) {
3915   MVT VT = widenMaskVectorType(Vec.getSimpleValueType(), Subtarget);
3916   return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
3917 }
3918 
3919 // Helper function to collect subvector ops that are concatenated together,
3920 // either by ISD::CONCAT_VECTORS or a ISD::INSERT_SUBVECTOR series.
3921 // The subvectors in Ops are guaranteed to be the same type.
3922 static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops,
3923                              SelectionDAG &DAG) {
3924   assert(Ops.empty() && "Expected an empty ops vector");
3925 
3926   if (N->getOpcode() == ISD::CONCAT_VECTORS) {
3927     Ops.append(N->op_begin(), N->op_end());
3928     return true;
3929   }
3930 
3931   if (N->getOpcode() == ISD::INSERT_SUBVECTOR) {
3932     SDValue Src = N->getOperand(0);
3933     SDValue Sub = N->getOperand(1);
3934     const APInt &Idx = N->getConstantOperandAPInt(2);
3935     EVT VT = Src.getValueType();
3936     EVT SubVT = Sub.getValueType();
3937 
3938     // TODO - Handle more general insert_subvector chains.
3939     if (VT.getSizeInBits() == (SubVT.getSizeInBits() * 2)) {
3940       // insert_subvector(undef, x, lo)
3941       if (Idx == 0 && Src.isUndef()) {
3942         Ops.push_back(Sub);
3943         Ops.push_back(DAG.getUNDEF(SubVT));
3944         return true;
3945       }
3946       if (Idx == (VT.getVectorNumElements() / 2)) {
3947         // insert_subvector(insert_subvector(undef, x, lo), y, hi)
3948         if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
3949             Src.getOperand(1).getValueType() == SubVT &&
3950             isNullConstant(Src.getOperand(2))) {
3951           Ops.push_back(Src.getOperand(1));
3952           Ops.push_back(Sub);
3953           return true;
3954         }
3955         // insert_subvector(x, extract_subvector(x, lo), hi)
3956         if (Sub.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
3957             Sub.getOperand(0) == Src && isNullConstant(Sub.getOperand(1))) {
3958           Ops.append(2, Sub);
3959           return true;
3960         }
3961         // insert_subvector(undef, x, hi)
3962         if (Src.isUndef()) {
3963           Ops.push_back(DAG.getUNDEF(SubVT));
3964           Ops.push_back(Sub);
3965           return true;
3966         }
3967       }
3968     }
3969   }
3970 
3971   return false;
3972 }
3973 
3974 // Helper to check if \p V can be split into subvectors and the upper subvectors
3975 // are all undef. In which case return the lower subvector.
3976 static SDValue isUpperSubvectorUndef(SDValue V, const SDLoc &DL,
3977                                      SelectionDAG &DAG) {
3978   SmallVector<SDValue> SubOps;
3979   if (!collectConcatOps(V.getNode(), SubOps, DAG))
3980     return SDValue();
3981 
3982   unsigned NumSubOps = SubOps.size();
3983   unsigned HalfNumSubOps = NumSubOps / 2;
3984   assert((NumSubOps % 2) == 0 && "Unexpected number of subvectors");
3985 
3986   ArrayRef<SDValue> UpperOps(SubOps.begin() + HalfNumSubOps, SubOps.end());
3987   if (any_of(UpperOps, [](SDValue Op) { return !Op.isUndef(); }))
3988     return SDValue();
3989 
3990   EVT HalfVT = V.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
3991   ArrayRef<SDValue> LowerOps(SubOps.begin(), SubOps.begin() + HalfNumSubOps);
3992   return DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, LowerOps);
3993 }
3994 
3995 // Helper to check if we can access all the constituent subvectors without any
3996 // extract ops.
3997 static bool isFreeToSplitVector(SDNode *N, SelectionDAG &DAG) {
3998   SmallVector<SDValue> Ops;
3999   return collectConcatOps(N, Ops, DAG);
4000 }
4001 
4002 static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG,
4003                                                const SDLoc &dl) {
4004   EVT VT = Op.getValueType();
4005   unsigned NumElems = VT.getVectorNumElements();
4006   unsigned SizeInBits = VT.getSizeInBits();
4007   assert((NumElems % 2) == 0 && (SizeInBits % 2) == 0 &&
4008          "Can't split odd sized vector");
4009 
4010   // If this is a splat value (with no-undefs) then use the lower subvector,
4011   // which should be a free extraction.
4012   SDValue Lo = extractSubVector(Op, 0, DAG, dl, SizeInBits / 2);
4013   if (DAG.isSplatValue(Op, /*AllowUndefs*/ false))
4014     return std::make_pair(Lo, Lo);
4015 
4016   SDValue Hi = extractSubVector(Op, NumElems / 2, DAG, dl, SizeInBits / 2);
4017   return std::make_pair(Lo, Hi);
4018 }
4019 
4020 /// Break an operation into 2 half sized ops and then concatenate the results.
4021 static SDValue splitVectorOp(SDValue Op, SelectionDAG &DAG) {
4022   unsigned NumOps = Op.getNumOperands();
4023   EVT VT = Op.getValueType();
4024   SDLoc dl(Op);
4025 
4026   // Extract the LHS Lo/Hi vectors
4027   SmallVector<SDValue> LoOps(NumOps, SDValue());
4028   SmallVector<SDValue> HiOps(NumOps, SDValue());
4029   for (unsigned I = 0; I != NumOps; ++I) {
4030     SDValue SrcOp = Op.getOperand(I);
4031     if (!SrcOp.getValueType().isVector()) {
4032       LoOps[I] = HiOps[I] = SrcOp;
4033       continue;
4034     }
4035     std::tie(LoOps[I], HiOps[I]) = splitVector(SrcOp, DAG, dl);
4036   }
4037 
4038   EVT LoVT, HiVT;
4039   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
4040   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
4041                      DAG.getNode(Op.getOpcode(), dl, LoVT, LoOps),
4042                      DAG.getNode(Op.getOpcode(), dl, HiVT, HiOps));
4043 }
4044 
4045 /// Break an unary integer operation into 2 half sized ops and then
4046 /// concatenate the result back.
4047 static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG) {
4048   // Make sure we only try to split 256/512-bit types to avoid creating
4049   // narrow vectors.
4050   EVT VT = Op.getValueType();
4051   (void)VT;
4052   assert((Op.getOperand(0).getValueType().is256BitVector() ||
4053           Op.getOperand(0).getValueType().is512BitVector()) &&
4054          (VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!");
4055   assert(Op.getOperand(0).getValueType().getVectorNumElements() ==
4056              VT.getVectorNumElements() &&
4057          "Unexpected VTs!");
4058   return splitVectorOp(Op, DAG);
4059 }
4060 
4061 /// Break a binary integer operation into 2 half sized ops and then
4062 /// concatenate the result back.
4063 static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG) {
4064   // Assert that all the types match.
4065   EVT VT = Op.getValueType();
4066   (void)VT;
4067   assert(Op.getOperand(0).getValueType() == VT &&
4068          Op.getOperand(1).getValueType() == VT && "Unexpected VTs!");
4069   assert((VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!");
4070   return splitVectorOp(Op, DAG);
4071 }
4072 
4073 // Helper for splitting operands of an operation to legal target size and
4074 // apply a function on each part.
4075 // Useful for operations that are available on SSE2 in 128-bit, on AVX2 in
4076 // 256-bit and on AVX512BW in 512-bit. The argument VT is the type used for
4077 // deciding if/how to split Ops. Ops elements do *not* have to be of type VT.
4078 // The argument Builder is a function that will be applied on each split part:
4079 // SDValue Builder(SelectionDAG&G, SDLoc, ArrayRef<SDValue>)
4080 template <typename F>
4081 SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4082                          const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
4083                          F Builder, bool CheckBWI = true) {
4084   assert(Subtarget.hasSSE2() && "Target assumed to support at least SSE2");
4085   unsigned NumSubs = 1;
4086   if ((CheckBWI && Subtarget.useBWIRegs()) ||
4087       (!CheckBWI && Subtarget.useAVX512Regs())) {
4088     if (VT.getSizeInBits() > 512) {
4089       NumSubs = VT.getSizeInBits() / 512;
4090       assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size");
4091     }
4092   } else if (Subtarget.hasAVX2()) {
4093     if (VT.getSizeInBits() > 256) {
4094       NumSubs = VT.getSizeInBits() / 256;
4095       assert((VT.getSizeInBits() % 256) == 0 && "Illegal vector size");
4096     }
4097   } else {
4098     if (VT.getSizeInBits() > 128) {
4099       NumSubs = VT.getSizeInBits() / 128;
4100       assert((VT.getSizeInBits() % 128) == 0 && "Illegal vector size");
4101     }
4102   }
4103 
4104   if (NumSubs == 1)
4105     return Builder(DAG, DL, Ops);
4106 
4107   SmallVector<SDValue, 4> Subs;
4108   for (unsigned i = 0; i != NumSubs; ++i) {
4109     SmallVector<SDValue, 2> SubOps;
4110     for (SDValue Op : Ops) {
4111       EVT OpVT = Op.getValueType();
4112       unsigned NumSubElts = OpVT.getVectorNumElements() / NumSubs;
4113       unsigned SizeSub = OpVT.getSizeInBits() / NumSubs;
4114       SubOps.push_back(extractSubVector(Op, i * NumSubElts, DAG, DL, SizeSub));
4115     }
4116     Subs.push_back(Builder(DAG, DL, SubOps));
4117   }
4118   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
4119 }
4120 
4121 // Helper function that extends a non-512-bit vector op to 512-bits on non-VLX
4122 // targets.
4123 static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT,
4124                              ArrayRef<SDValue> Ops, SelectionDAG &DAG,
4125                              const X86Subtarget &Subtarget) {
4126   assert(Subtarget.hasAVX512() && "AVX512 target expected");
4127   MVT SVT = VT.getScalarType();
4128 
4129   // If we have a 32/64 splatted constant, splat it to DstTy to
4130   // encourage a foldable broadcast'd operand.
4131   auto MakeBroadcastOp = [&](SDValue Op, MVT OpVT, MVT DstVT) {
4132     unsigned OpEltSizeInBits = OpVT.getScalarSizeInBits();
4133     // AVX512 broadcasts 32/64-bit operands.
4134     // TODO: Support float once getAVX512Node is used by fp-ops.
4135     if (!OpVT.isInteger() || OpEltSizeInBits < 32 ||
4136         !DAG.getTargetLoweringInfo().isTypeLegal(SVT))
4137       return SDValue();
4138     // If we're not widening, don't bother if we're not bitcasting.
4139     if (OpVT == DstVT && Op.getOpcode() != ISD::BITCAST)
4140       return SDValue();
4141     if (auto *BV = dyn_cast<BuildVectorSDNode>(peekThroughBitcasts(Op))) {
4142       APInt SplatValue, SplatUndef;
4143       unsigned SplatBitSize;
4144       bool HasAnyUndefs;
4145       if (BV->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
4146                               HasAnyUndefs, OpEltSizeInBits) &&
4147           !HasAnyUndefs && SplatValue.getBitWidth() == OpEltSizeInBits)
4148         return DAG.getConstant(SplatValue, DL, DstVT);
4149     }
4150     return SDValue();
4151   };
4152 
4153   bool Widen = !(Subtarget.hasVLX() || VT.is512BitVector());
4154 
4155   MVT DstVT = VT;
4156   if (Widen)
4157     DstVT = MVT::getVectorVT(SVT, 512 / SVT.getSizeInBits());
4158 
4159   // Canonicalize src operands.
4160   SmallVector<SDValue> SrcOps(Ops.begin(), Ops.end());
4161   for (SDValue &Op : SrcOps) {
4162     MVT OpVT = Op.getSimpleValueType();
4163     // Just pass through scalar operands.
4164     if (!OpVT.isVector())
4165       continue;
4166     assert(OpVT == VT && "Vector type mismatch");
4167 
4168     if (SDValue BroadcastOp = MakeBroadcastOp(Op, OpVT, DstVT)) {
4169       Op = BroadcastOp;
4170       continue;
4171     }
4172 
4173     // Just widen the subvector by inserting into an undef wide vector.
4174     if (Widen)
4175       Op = widenSubVector(Op, false, Subtarget, DAG, DL, 512);
4176   }
4177 
4178   SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps);
4179 
4180   // Perform the 512-bit op then extract the bottom subvector.
4181   if (Widen)
4182     Res = extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
4183   return Res;
4184 }
4185 
4186 /// Insert i1-subvector to i1-vector.
4187 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
4188                                 const X86Subtarget &Subtarget) {
4189 
4190   SDLoc dl(Op);
4191   SDValue Vec = Op.getOperand(0);
4192   SDValue SubVec = Op.getOperand(1);
4193   SDValue Idx = Op.getOperand(2);
4194   unsigned IdxVal = Op.getConstantOperandVal(2);
4195 
4196   // Inserting undef is a nop. We can just return the original vector.
4197   if (SubVec.isUndef())
4198     return Vec;
4199 
4200   if (IdxVal == 0 && Vec.isUndef()) // the operation is legal
4201     return Op;
4202 
4203   MVT OpVT = Op.getSimpleValueType();
4204   unsigned NumElems = OpVT.getVectorNumElements();
4205   SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
4206 
4207   // Extend to natively supported kshift.
4208   MVT WideOpVT = widenMaskVectorType(OpVT, Subtarget);
4209 
4210   // Inserting into the lsbs of a zero vector is legal. ISel will insert shifts
4211   // if necessary.
4212   if (IdxVal == 0 && ISD::isBuildVectorAllZeros(Vec.getNode())) {
4213     // May need to promote to a legal type.
4214     Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4215                      DAG.getConstant(0, dl, WideOpVT),
4216                      SubVec, Idx);
4217     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4218   }
4219 
4220   MVT SubVecVT = SubVec.getSimpleValueType();
4221   unsigned SubVecNumElems = SubVecVT.getVectorNumElements();
4222   assert(IdxVal + SubVecNumElems <= NumElems &&
4223          IdxVal % SubVecVT.getSizeInBits() == 0 &&
4224          "Unexpected index value in INSERT_SUBVECTOR");
4225 
4226   SDValue Undef = DAG.getUNDEF(WideOpVT);
4227 
4228   if (IdxVal == 0) {
4229     // Zero lower bits of the Vec
4230     SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
4231     Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
4232                       ZeroIdx);
4233     Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
4234     Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
4235     // Merge them together, SubVec should be zero extended.
4236     SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4237                          DAG.getConstant(0, dl, WideOpVT),
4238                          SubVec, ZeroIdx);
4239     Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4240     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4241   }
4242 
4243   SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4244                        Undef, SubVec, ZeroIdx);
4245 
4246   if (Vec.isUndef()) {
4247     assert(IdxVal != 0 && "Unexpected index");
4248     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4249                          DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4250     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4251   }
4252 
4253   if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
4254     assert(IdxVal != 0 && "Unexpected index");
4255     // If upper elements of Vec are known undef, then just shift into place.
4256     if (llvm::all_of(Vec->ops().slice(IdxVal + SubVecNumElems),
4257                      [](SDValue V) { return V.isUndef(); })) {
4258       SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4259                            DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4260     } else {
4261       NumElems = WideOpVT.getVectorNumElements();
4262       unsigned ShiftLeft = NumElems - SubVecNumElems;
4263       unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
4264       SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4265                            DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4266       if (ShiftRight != 0)
4267         SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4268                              DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4269     }
4270     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4271   }
4272 
4273   // Simple case when we put subvector in the upper part
4274   if (IdxVal + SubVecNumElems == NumElems) {
4275     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4276                          DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4277     if (SubVecNumElems * 2 == NumElems) {
4278       // Special case, use legal zero extending insert_subvector. This allows
4279       // isel to optimize when bits are known zero.
4280       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx);
4281       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4282                         DAG.getConstant(0, dl, WideOpVT),
4283                         Vec, ZeroIdx);
4284     } else {
4285       // Otherwise use explicit shifts to zero the bits.
4286       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4287                         Undef, Vec, ZeroIdx);
4288       NumElems = WideOpVT.getVectorNumElements();
4289       SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
4290       Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
4291       Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
4292     }
4293     Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4294     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4295   }
4296 
4297   // Inserting into the middle is more complicated.
4298 
4299   NumElems = WideOpVT.getVectorNumElements();
4300 
4301   // Widen the vector if needed.
4302   Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
4303 
4304   unsigned ShiftLeft = NumElems - SubVecNumElems;
4305   unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
4306 
4307   // Do an optimization for the most frequently used types.
4308   if (WideOpVT != MVT::v64i1 || Subtarget.is64Bit()) {
4309     APInt Mask0 = APInt::getBitsSet(NumElems, IdxVal, IdxVal + SubVecNumElems);
4310     Mask0.flipAllBits();
4311     SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));
4312     SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0);
4313     Vec = DAG.getNode(ISD::AND, dl, WideOpVT, Vec, VMask0);
4314     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4315                          DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4316     SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4317                          DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4318     Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4319 
4320     // Reduce to original width if needed.
4321     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4322   }
4323 
4324   // Clear the upper bits of the subvector and move it to its insert position.
4325   SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4326                        DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4327   SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4328                        DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4329 
4330   // Isolate the bits below the insertion point.
4331   unsigned LowShift = NumElems - IdxVal;
4332   SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
4333                             DAG.getTargetConstant(LowShift, dl, MVT::i8));
4334   Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low,
4335                     DAG.getTargetConstant(LowShift, dl, MVT::i8));
4336 
4337   // Isolate the bits after the last inserted bit.
4338   unsigned HighShift = IdxVal + SubVecNumElems;
4339   SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
4340                             DAG.getTargetConstant(HighShift, dl, MVT::i8));
4341   High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
4342                     DAG.getTargetConstant(HighShift, dl, MVT::i8));
4343 
4344   // Now OR all 3 pieces together.
4345   Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
4346   SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec);
4347 
4348   // Reduce to original width if needed.
4349   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4350 }
4351 
4352 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
4353                                 const SDLoc &dl) {
4354   assert(V1.getValueType() == V2.getValueType() && "subvector type mismatch");
4355   EVT SubVT = V1.getValueType();
4356   EVT SubSVT = SubVT.getScalarType();
4357   unsigned SubNumElts = SubVT.getVectorNumElements();
4358   unsigned SubVectorWidth = SubVT.getSizeInBits();
4359   EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts);
4360   SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
4361   return insertSubVector(V, V2, SubNumElts, DAG, dl, SubVectorWidth);
4362 }
4363 
4364 /// Returns a vector of specified type with all bits set.
4365 /// Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>.
4366 /// Then bitcast to their original type, ensuring they get CSE'd.
4367 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
4368   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
4369          "Expected a 128/256/512-bit vector type");
4370 
4371   APInt Ones = APInt::getAllOnes(32);
4372   unsigned NumElts = VT.getSizeInBits() / 32;
4373   SDValue Vec = DAG.getConstant(Ones, dl, MVT::getVectorVT(MVT::i32, NumElts));
4374   return DAG.getBitcast(VT, Vec);
4375 }
4376 
4377 static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT,
4378                                       SDValue In, SelectionDAG &DAG) {
4379   EVT InVT = In.getValueType();
4380   assert(VT.isVector() && InVT.isVector() && "Expected vector VTs.");
4381   assert((ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode ||
4382           ISD::ZERO_EXTEND == Opcode) &&
4383          "Unknown extension opcode");
4384 
4385   // For 256-bit vectors, we only need the lower (128-bit) input half.
4386   // For 512-bit vectors, we only need the lower input half or quarter.
4387   if (InVT.getSizeInBits() > 128) {
4388     assert(VT.getSizeInBits() == InVT.getSizeInBits() &&
4389            "Expected VTs to be the same size!");
4390     unsigned Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits();
4391     In = extractSubVector(In, 0, DAG, DL,
4392                           std::max(128U, (unsigned)VT.getSizeInBits() / Scale));
4393     InVT = In.getValueType();
4394   }
4395 
4396   if (VT.getVectorNumElements() != InVT.getVectorNumElements())
4397     Opcode = DAG.getOpcode_EXTEND_VECTOR_INREG(Opcode);
4398 
4399   return DAG.getNode(Opcode, DL, VT, In);
4400 }
4401 
4402 // Create OR(AND(LHS,MASK),AND(RHS,~MASK)) bit select pattern
4403 static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS,
4404                             SDValue Mask, SelectionDAG &DAG) {
4405   LHS = DAG.getNode(ISD::AND, DL, VT, LHS, Mask);
4406   RHS = DAG.getNode(X86ISD::ANDNP, DL, VT, Mask, RHS);
4407   return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
4408 }
4409 
4410 void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask,
4411                                    bool Lo, bool Unary) {
4412   assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 &&
4413          "Illegal vector type to unpack");
4414   assert(Mask.empty() && "Expected an empty shuffle mask vector");
4415   int NumElts = VT.getVectorNumElements();
4416   int NumEltsInLane = 128 / VT.getScalarSizeInBits();
4417   for (int i = 0; i < NumElts; ++i) {
4418     unsigned LaneStart = (i / NumEltsInLane) * NumEltsInLane;
4419     int Pos = (i % NumEltsInLane) / 2 + LaneStart;
4420     Pos += (Unary ? 0 : NumElts * (i % 2));
4421     Pos += (Lo ? 0 : NumEltsInLane / 2);
4422     Mask.push_back(Pos);
4423   }
4424 }
4425 
4426 /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
4427 /// imposed by AVX and specific to the unary pattern. Example:
4428 /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
4429 /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
4430 void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
4431                                    bool Lo) {
4432   assert(Mask.empty() && "Expected an empty shuffle mask vector");
4433   int NumElts = VT.getVectorNumElements();
4434   for (int i = 0; i < NumElts; ++i) {
4435     int Pos = i / 2;
4436     Pos += (Lo ? 0 : NumElts / 2);
4437     Mask.push_back(Pos);
4438   }
4439 }
4440 
4441 // Attempt to constant fold, else just create a VECTOR_SHUFFLE.
4442 static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl,
4443                                 SDValue V1, SDValue V2, ArrayRef<int> Mask) {
4444   if ((ISD::isBuildVectorOfConstantSDNodes(V1.getNode()) || V1.isUndef()) &&
4445       (ISD::isBuildVectorOfConstantSDNodes(V2.getNode()) || V2.isUndef())) {
4446     SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType()));
4447     for (int I = 0, NumElts = Mask.size(); I != NumElts; ++I) {
4448       int M = Mask[I];
4449       if (M < 0)
4450         continue;
4451       SDValue V = (M < NumElts) ? V1 : V2;
4452       if (V.isUndef())
4453         continue;
4454       Ops[I] = V.getOperand(M % NumElts);
4455     }
4456     return DAG.getBuildVector(VT, dl, Ops);
4457   }
4458 
4459   return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
4460 }
4461 
4462 /// Returns a vector_shuffle node for an unpackl operation.
4463 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4464                           SDValue V1, SDValue V2) {
4465   SmallVector<int, 8> Mask;
4466   createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false);
4467   return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
4468 }
4469 
4470 /// Returns a vector_shuffle node for an unpackh operation.
4471 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4472                           SDValue V1, SDValue V2) {
4473   SmallVector<int, 8> Mask;
4474   createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false);
4475   return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
4476 }
4477 
4478 /// Returns a node that packs the LHS + RHS nodes together at half width.
4479 /// May return X86ISD::PACKSS/PACKUS, packing the top/bottom half.
4480 /// TODO: Add subvector splitting if/when we have a need for it.
4481 static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4482                        const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS,
4483                        bool PackHiHalf = false) {
4484   MVT OpVT = LHS.getSimpleValueType();
4485   unsigned EltSizeInBits = VT.getScalarSizeInBits();
4486   bool UsePackUS = Subtarget.hasSSE41() || EltSizeInBits == 8;
4487   assert(OpVT == RHS.getSimpleValueType() &&
4488          VT.getSizeInBits() == OpVT.getSizeInBits() &&
4489          (EltSizeInBits * 2) == OpVT.getScalarSizeInBits() &&
4490          "Unexpected PACK operand types");
4491   assert((EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) &&
4492          "Unexpected PACK result type");
4493 
4494   // Rely on vector shuffles for vXi64 -> vXi32 packing.
4495   if (EltSizeInBits == 32) {
4496     SmallVector<int> PackMask;
4497     int Offset = PackHiHalf ? 1 : 0;
4498     int NumElts = VT.getVectorNumElements();
4499     for (int I = 0; I != NumElts; I += 4) {
4500       PackMask.push_back(I + Offset);
4501       PackMask.push_back(I + Offset + 2);
4502       PackMask.push_back(I + Offset + NumElts);
4503       PackMask.push_back(I + Offset + NumElts + 2);
4504     }
4505     return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS),
4506                                 DAG.getBitcast(VT, RHS), PackMask);
4507   }
4508 
4509   // See if we already have sufficient leading bits for PACKSS/PACKUS.
4510   if (!PackHiHalf) {
4511     if (UsePackUS &&
4512         DAG.computeKnownBits(LHS).countMaxActiveBits() <= EltSizeInBits &&
4513         DAG.computeKnownBits(RHS).countMaxActiveBits() <= EltSizeInBits)
4514       return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
4515 
4516     if (DAG.ComputeMaxSignificantBits(LHS) <= EltSizeInBits &&
4517         DAG.ComputeMaxSignificantBits(RHS) <= EltSizeInBits)
4518       return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
4519   }
4520 
4521   // Fallback to sign/zero extending the requested half and pack.
4522   SDValue Amt = DAG.getTargetConstant(EltSizeInBits, dl, MVT::i8);
4523   if (UsePackUS) {
4524     if (PackHiHalf) {
4525       LHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, LHS, Amt);
4526       RHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, RHS, Amt);
4527     } else {
4528       SDValue Mask = DAG.getConstant((1ULL << EltSizeInBits) - 1, dl, OpVT);
4529       LHS = DAG.getNode(ISD::AND, dl, OpVT, LHS, Mask);
4530       RHS = DAG.getNode(ISD::AND, dl, OpVT, RHS, Mask);
4531     };
4532     return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
4533   };
4534 
4535   if (!PackHiHalf) {
4536     LHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, LHS, Amt);
4537     RHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, RHS, Amt);
4538   }
4539   LHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, LHS, Amt);
4540   RHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, RHS, Amt);
4541   return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
4542 }
4543 
4544 /// Return a vector_shuffle of the specified vector of zero or undef vector.
4545 /// This produces a shuffle where the low element of V2 is swizzled into the
4546 /// zero/undef vector, landing at element Idx.
4547 /// This produces a shuffle mask like 4,1,2,3 (idx=0) or  0,1,2,4 (idx=3).
4548 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
4549                                            bool IsZero,
4550                                            const X86Subtarget &Subtarget,
4551                                            SelectionDAG &DAG) {
4552   MVT VT = V2.getSimpleValueType();
4553   SDValue V1 = IsZero
4554     ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
4555   int NumElems = VT.getVectorNumElements();
4556   SmallVector<int, 16> MaskVec(NumElems);
4557   for (int i = 0; i != NumElems; ++i)
4558     // If this is the insertion idx, put the low elt of V2 here.
4559     MaskVec[i] = (i == Idx) ? NumElems : i;
4560   return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec);
4561 }
4562 
4563 static ConstantPoolSDNode *getTargetConstantPoolFromBasePtr(SDValue Ptr) {
4564   if (Ptr.getOpcode() == X86ISD::Wrapper ||
4565       Ptr.getOpcode() == X86ISD::WrapperRIP)
4566     Ptr = Ptr.getOperand(0);
4567   return dyn_cast<ConstantPoolSDNode>(Ptr);
4568 }
4569 
4570 static const Constant *getTargetConstantFromBasePtr(SDValue Ptr) {
4571   ConstantPoolSDNode *CNode = getTargetConstantPoolFromBasePtr(Ptr);
4572   if (!CNode || CNode->isMachineConstantPoolEntry() || CNode->getOffset() != 0)
4573     return nullptr;
4574   return CNode->getConstVal();
4575 }
4576 
4577 static const Constant *getTargetConstantFromNode(LoadSDNode *Load) {
4578   if (!Load || !ISD::isNormalLoad(Load))
4579     return nullptr;
4580   return getTargetConstantFromBasePtr(Load->getBasePtr());
4581 }
4582 
4583 static const Constant *getTargetConstantFromNode(SDValue Op) {
4584   Op = peekThroughBitcasts(Op);
4585   return getTargetConstantFromNode(dyn_cast<LoadSDNode>(Op));
4586 }
4587 
4588 const Constant *
4589 X86TargetLowering::getTargetConstantFromLoad(LoadSDNode *LD) const {
4590   assert(LD && "Unexpected null LoadSDNode");
4591   return getTargetConstantFromNode(LD);
4592 }
4593 
4594 // Extract raw constant bits from constant pools.
4595 static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
4596                                           APInt &UndefElts,
4597                                           SmallVectorImpl<APInt> &EltBits,
4598                                           bool AllowWholeUndefs = true,
4599                                           bool AllowPartialUndefs = true) {
4600   assert(EltBits.empty() && "Expected an empty EltBits vector");
4601 
4602   Op = peekThroughBitcasts(Op);
4603 
4604   EVT VT = Op.getValueType();
4605   unsigned SizeInBits = VT.getSizeInBits();
4606   assert((SizeInBits % EltSizeInBits) == 0 && "Can't split constant!");
4607   unsigned NumElts = SizeInBits / EltSizeInBits;
4608 
4609   // Bitcast a source array of element bits to the target size.
4610   auto CastBitData = [&](APInt &UndefSrcElts, ArrayRef<APInt> SrcEltBits) {
4611     unsigned NumSrcElts = UndefSrcElts.getBitWidth();
4612     unsigned SrcEltSizeInBits = SrcEltBits[0].getBitWidth();
4613     assert((NumSrcElts * SrcEltSizeInBits) == SizeInBits &&
4614            "Constant bit sizes don't match");
4615 
4616     // Don't split if we don't allow undef bits.
4617     bool AllowUndefs = AllowWholeUndefs || AllowPartialUndefs;
4618     if (UndefSrcElts.getBoolValue() && !AllowUndefs)
4619       return false;
4620 
4621     // If we're already the right size, don't bother bitcasting.
4622     if (NumSrcElts == NumElts) {
4623       UndefElts = UndefSrcElts;
4624       EltBits.assign(SrcEltBits.begin(), SrcEltBits.end());
4625       return true;
4626     }
4627 
4628     // Extract all the undef/constant element data and pack into single bitsets.
4629     APInt UndefBits(SizeInBits, 0);
4630     APInt MaskBits(SizeInBits, 0);
4631 
4632     for (unsigned i = 0; i != NumSrcElts; ++i) {
4633       unsigned BitOffset = i * SrcEltSizeInBits;
4634       if (UndefSrcElts[i])
4635         UndefBits.setBits(BitOffset, BitOffset + SrcEltSizeInBits);
4636       MaskBits.insertBits(SrcEltBits[i], BitOffset);
4637     }
4638 
4639     // Split the undef/constant single bitset data into the target elements.
4640     UndefElts = APInt(NumElts, 0);
4641     EltBits.resize(NumElts, APInt(EltSizeInBits, 0));
4642 
4643     for (unsigned i = 0; i != NumElts; ++i) {
4644       unsigned BitOffset = i * EltSizeInBits;
4645       APInt UndefEltBits = UndefBits.extractBits(EltSizeInBits, BitOffset);
4646 
4647       // Only treat an element as UNDEF if all bits are UNDEF.
4648       if (UndefEltBits.isAllOnes()) {
4649         if (!AllowWholeUndefs)
4650           return false;
4651         UndefElts.setBit(i);
4652         continue;
4653       }
4654 
4655       // If only some bits are UNDEF then treat them as zero (or bail if not
4656       // supported).
4657       if (UndefEltBits.getBoolValue() && !AllowPartialUndefs)
4658         return false;
4659 
4660       EltBits[i] = MaskBits.extractBits(EltSizeInBits, BitOffset);
4661     }
4662     return true;
4663   };
4664 
4665   // Collect constant bits and insert into mask/undef bit masks.
4666   auto CollectConstantBits = [](const Constant *Cst, APInt &Mask, APInt &Undefs,
4667                                 unsigned UndefBitIndex) {
4668     if (!Cst)
4669       return false;
4670     if (isa<UndefValue>(Cst)) {
4671       Undefs.setBit(UndefBitIndex);
4672       return true;
4673     }
4674     if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
4675       Mask = CInt->getValue();
4676       return true;
4677     }
4678     if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
4679       Mask = CFP->getValueAPF().bitcastToAPInt();
4680       return true;
4681     }
4682     if (auto *CDS = dyn_cast<ConstantDataSequential>(Cst)) {
4683       Type *Ty = CDS->getType();
4684       Mask = APInt::getZero(Ty->getPrimitiveSizeInBits());
4685       Type *EltTy = CDS->getElementType();
4686       bool IsInteger = EltTy->isIntegerTy();
4687       bool IsFP =
4688           EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
4689       if (!IsInteger && !IsFP)
4690         return false;
4691       unsigned EltBits = EltTy->getPrimitiveSizeInBits();
4692       for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I)
4693         if (IsInteger)
4694           Mask.insertBits(CDS->getElementAsAPInt(I), I * EltBits);
4695         else
4696           Mask.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(),
4697                           I * EltBits);
4698       return true;
4699     }
4700     return false;
4701   };
4702 
4703   // Handle UNDEFs.
4704   if (Op.isUndef()) {
4705     APInt UndefSrcElts = APInt::getAllOnes(NumElts);
4706     SmallVector<APInt, 64> SrcEltBits(NumElts, APInt(EltSizeInBits, 0));
4707     return CastBitData(UndefSrcElts, SrcEltBits);
4708   }
4709 
4710   // Extract scalar constant bits.
4711   if (auto *Cst = dyn_cast<ConstantSDNode>(Op)) {
4712     APInt UndefSrcElts = APInt::getZero(1);
4713     SmallVector<APInt, 64> SrcEltBits(1, Cst->getAPIntValue());
4714     return CastBitData(UndefSrcElts, SrcEltBits);
4715   }
4716   if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
4717     APInt UndefSrcElts = APInt::getZero(1);
4718     APInt RawBits = Cst->getValueAPF().bitcastToAPInt();
4719     SmallVector<APInt, 64> SrcEltBits(1, RawBits);
4720     return CastBitData(UndefSrcElts, SrcEltBits);
4721   }
4722 
4723   // Extract constant bits from build vector.
4724   if (auto *BV = dyn_cast<BuildVectorSDNode>(Op)) {
4725     BitVector Undefs;
4726     SmallVector<APInt> SrcEltBits;
4727     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4728     if (BV->getConstantRawBits(true, SrcEltSizeInBits, SrcEltBits, Undefs)) {
4729       APInt UndefSrcElts = APInt::getZero(SrcEltBits.size());
4730       for (unsigned I = 0, E = SrcEltBits.size(); I != E; ++I)
4731         if (Undefs[I])
4732           UndefSrcElts.setBit(I);
4733       return CastBitData(UndefSrcElts, SrcEltBits);
4734     }
4735   }
4736 
4737   // Extract constant bits from constant pool vector.
4738   if (auto *Cst = getTargetConstantFromNode(Op)) {
4739     Type *CstTy = Cst->getType();
4740     unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
4741     if (!CstTy->isVectorTy() || (CstSizeInBits % SizeInBits) != 0)
4742       return false;
4743 
4744     unsigned SrcEltSizeInBits = CstTy->getScalarSizeInBits();
4745     unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4746     if ((SizeInBits % SrcEltSizeInBits) != 0)
4747       return false;
4748 
4749     APInt UndefSrcElts(NumSrcElts, 0);
4750     SmallVector<APInt, 64> SrcEltBits(NumSrcElts, APInt(SrcEltSizeInBits, 0));
4751     for (unsigned i = 0; i != NumSrcElts; ++i)
4752       if (!CollectConstantBits(Cst->getAggregateElement(i), SrcEltBits[i],
4753                                UndefSrcElts, i))
4754         return false;
4755 
4756     return CastBitData(UndefSrcElts, SrcEltBits);
4757   }
4758 
4759   // Extract constant bits from a broadcasted constant pool scalar.
4760   if (Op.getOpcode() == X86ISD::VBROADCAST_LOAD &&
4761       EltSizeInBits <= VT.getScalarSizeInBits()) {
4762     auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
4763     if (MemIntr->getMemoryVT().getStoreSizeInBits() != VT.getScalarSizeInBits())
4764       return false;
4765 
4766     SDValue Ptr = MemIntr->getBasePtr();
4767     if (const Constant *C = getTargetConstantFromBasePtr(Ptr)) {
4768       unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4769       unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4770 
4771       APInt UndefSrcElts(NumSrcElts, 0);
4772       SmallVector<APInt, 64> SrcEltBits(1, APInt(SrcEltSizeInBits, 0));
4773       if (CollectConstantBits(C, SrcEltBits[0], UndefSrcElts, 0)) {
4774         if (UndefSrcElts[0])
4775           UndefSrcElts.setBits(0, NumSrcElts);
4776         if (SrcEltBits[0].getBitWidth() != SrcEltSizeInBits)
4777           SrcEltBits[0] = SrcEltBits[0].trunc(SrcEltSizeInBits);
4778         SrcEltBits.append(NumSrcElts - 1, SrcEltBits[0]);
4779         return CastBitData(UndefSrcElts, SrcEltBits);
4780       }
4781     }
4782   }
4783 
4784   // Extract constant bits from a subvector broadcast.
4785   if (Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) {
4786     auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
4787     SDValue Ptr = MemIntr->getBasePtr();
4788     // The source constant may be larger than the subvector broadcast,
4789     // ensure we extract the correct subvector constants.
4790     if (const Constant *Cst = getTargetConstantFromBasePtr(Ptr)) {
4791       Type *CstTy = Cst->getType();
4792       unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
4793       unsigned SubVecSizeInBits = MemIntr->getMemoryVT().getStoreSizeInBits();
4794       if (!CstTy->isVectorTy() || (CstSizeInBits % SubVecSizeInBits) != 0 ||
4795           (SizeInBits % SubVecSizeInBits) != 0)
4796         return false;
4797       unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
4798       unsigned NumSubElts = SubVecSizeInBits / CstEltSizeInBits;
4799       unsigned NumSubVecs = SizeInBits / SubVecSizeInBits;
4800       APInt UndefSubElts(NumSubElts, 0);
4801       SmallVector<APInt, 64> SubEltBits(NumSubElts * NumSubVecs,
4802                                         APInt(CstEltSizeInBits, 0));
4803       for (unsigned i = 0; i != NumSubElts; ++i) {
4804         if (!CollectConstantBits(Cst->getAggregateElement(i), SubEltBits[i],
4805                                  UndefSubElts, i))
4806           return false;
4807         for (unsigned j = 1; j != NumSubVecs; ++j)
4808           SubEltBits[i + (j * NumSubElts)] = SubEltBits[i];
4809       }
4810       UndefSubElts = APInt::getSplat(NumSubVecs * UndefSubElts.getBitWidth(),
4811                                      UndefSubElts);
4812       return CastBitData(UndefSubElts, SubEltBits);
4813     }
4814   }
4815 
4816   // Extract a rematerialized scalar constant insertion.
4817   if (Op.getOpcode() == X86ISD::VZEXT_MOVL &&
4818       Op.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR &&
4819       isa<ConstantSDNode>(Op.getOperand(0).getOperand(0))) {
4820     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4821     unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4822 
4823     APInt UndefSrcElts(NumSrcElts, 0);
4824     SmallVector<APInt, 64> SrcEltBits;
4825     const APInt &C = Op.getOperand(0).getConstantOperandAPInt(0);
4826     SrcEltBits.push_back(C.zextOrTrunc(SrcEltSizeInBits));
4827     SrcEltBits.append(NumSrcElts - 1, APInt(SrcEltSizeInBits, 0));
4828     return CastBitData(UndefSrcElts, SrcEltBits);
4829   }
4830 
4831   // Insert constant bits from a base and sub vector sources.
4832   if (Op.getOpcode() == ISD::INSERT_SUBVECTOR) {
4833     // If bitcasts to larger elements we might lose track of undefs - don't
4834     // allow any to be safe.
4835     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4836     bool AllowUndefs = EltSizeInBits >= SrcEltSizeInBits;
4837 
4838     APInt UndefSrcElts, UndefSubElts;
4839     SmallVector<APInt, 32> EltSrcBits, EltSubBits;
4840     if (getTargetConstantBitsFromNode(Op.getOperand(1), SrcEltSizeInBits,
4841                                       UndefSubElts, EltSubBits,
4842                                       AllowWholeUndefs && AllowUndefs,
4843                                       AllowPartialUndefs && AllowUndefs) &&
4844         getTargetConstantBitsFromNode(Op.getOperand(0), SrcEltSizeInBits,
4845                                       UndefSrcElts, EltSrcBits,
4846                                       AllowWholeUndefs && AllowUndefs,
4847                                       AllowPartialUndefs && AllowUndefs)) {
4848       unsigned BaseIdx = Op.getConstantOperandVal(2);
4849       UndefSrcElts.insertBits(UndefSubElts, BaseIdx);
4850       for (unsigned i = 0, e = EltSubBits.size(); i != e; ++i)
4851         EltSrcBits[BaseIdx + i] = EltSubBits[i];
4852       return CastBitData(UndefSrcElts, EltSrcBits);
4853     }
4854   }
4855 
4856   // Extract constant bits from a subvector's source.
4857   if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
4858     // TODO - support extract_subvector through bitcasts.
4859     if (EltSizeInBits != VT.getScalarSizeInBits())
4860       return false;
4861 
4862     if (getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
4863                                       UndefElts, EltBits, AllowWholeUndefs,
4864                                       AllowPartialUndefs)) {
4865       EVT SrcVT = Op.getOperand(0).getValueType();
4866       unsigned NumSrcElts = SrcVT.getVectorNumElements();
4867       unsigned NumSubElts = VT.getVectorNumElements();
4868       unsigned BaseIdx = Op.getConstantOperandVal(1);
4869       UndefElts = UndefElts.extractBits(NumSubElts, BaseIdx);
4870       if ((BaseIdx + NumSubElts) != NumSrcElts)
4871         EltBits.erase(EltBits.begin() + BaseIdx + NumSubElts, EltBits.end());
4872       if (BaseIdx != 0)
4873         EltBits.erase(EltBits.begin(), EltBits.begin() + BaseIdx);
4874       return true;
4875     }
4876   }
4877 
4878   // Extract constant bits from shuffle node sources.
4879   if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Op)) {
4880     // TODO - support shuffle through bitcasts.
4881     if (EltSizeInBits != VT.getScalarSizeInBits())
4882       return false;
4883 
4884     ArrayRef<int> Mask = SVN->getMask();
4885     if ((!AllowWholeUndefs || !AllowPartialUndefs) &&
4886         llvm::any_of(Mask, [](int M) { return M < 0; }))
4887       return false;
4888 
4889     APInt UndefElts0, UndefElts1;
4890     SmallVector<APInt, 32> EltBits0, EltBits1;
4891     if (isAnyInRange(Mask, 0, NumElts) &&
4892         !getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
4893                                        UndefElts0, EltBits0, AllowWholeUndefs,
4894                                        AllowPartialUndefs))
4895       return false;
4896     if (isAnyInRange(Mask, NumElts, 2 * NumElts) &&
4897         !getTargetConstantBitsFromNode(Op.getOperand(1), EltSizeInBits,
4898                                        UndefElts1, EltBits1, AllowWholeUndefs,
4899                                        AllowPartialUndefs))
4900       return false;
4901 
4902     UndefElts = APInt::getZero(NumElts);
4903     for (int i = 0; i != (int)NumElts; ++i) {
4904       int M = Mask[i];
4905       if (M < 0) {
4906         UndefElts.setBit(i);
4907         EltBits.push_back(APInt::getZero(EltSizeInBits));
4908       } else if (M < (int)NumElts) {
4909         if (UndefElts0[M])
4910           UndefElts.setBit(i);
4911         EltBits.push_back(EltBits0[M]);
4912       } else {
4913         if (UndefElts1[M - NumElts])
4914           UndefElts.setBit(i);
4915         EltBits.push_back(EltBits1[M - NumElts]);
4916       }
4917     }
4918     return true;
4919   }
4920 
4921   return false;
4922 }
4923 
4924 namespace llvm {
4925 namespace X86 {
4926 bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs) {
4927   APInt UndefElts;
4928   SmallVector<APInt, 16> EltBits;
4929   if (getTargetConstantBitsFromNode(Op, Op.getScalarValueSizeInBits(),
4930                                     UndefElts, EltBits, true,
4931                                     AllowPartialUndefs)) {
4932     int SplatIndex = -1;
4933     for (int i = 0, e = EltBits.size(); i != e; ++i) {
4934       if (UndefElts[i])
4935         continue;
4936       if (0 <= SplatIndex && EltBits[i] != EltBits[SplatIndex]) {
4937         SplatIndex = -1;
4938         break;
4939       }
4940       SplatIndex = i;
4941     }
4942     if (0 <= SplatIndex) {
4943       SplatVal = EltBits[SplatIndex];
4944       return true;
4945     }
4946   }
4947 
4948   return false;
4949 }
4950 } // namespace X86
4951 } // namespace llvm
4952 
4953 static bool getTargetShuffleMaskIndices(SDValue MaskNode,
4954                                         unsigned MaskEltSizeInBits,
4955                                         SmallVectorImpl<uint64_t> &RawMask,
4956                                         APInt &UndefElts) {
4957   // Extract the raw target constant bits.
4958   SmallVector<APInt, 64> EltBits;
4959   if (!getTargetConstantBitsFromNode(MaskNode, MaskEltSizeInBits, UndefElts,
4960                                      EltBits, /* AllowWholeUndefs */ true,
4961                                      /* AllowPartialUndefs */ false))
4962     return false;
4963 
4964   // Insert the extracted elements into the mask.
4965   for (const APInt &Elt : EltBits)
4966     RawMask.push_back(Elt.getZExtValue());
4967 
4968   return true;
4969 }
4970 
4971 // Match not(xor X, -1) -> X.
4972 // Match not(pcmpgt(C, X)) -> pcmpgt(X, C - 1).
4973 // Match not(extract_subvector(xor X, -1)) -> extract_subvector(X).
4974 // Match not(concat_vectors(xor X, -1, xor Y, -1)) -> concat_vectors(X, Y).
4975 static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
4976   V = peekThroughBitcasts(V);
4977   if (V.getOpcode() == ISD::XOR &&
4978       (ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()) ||
4979        isAllOnesConstant(V.getOperand(1))))
4980     return V.getOperand(0);
4981   if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
4982       (isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) {
4983     if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
4984       Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
4985       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), V.getValueType(),
4986                          Not, V.getOperand(1));
4987     }
4988   }
4989   if (V.getOpcode() == X86ISD::PCMPGT &&
4990       !ISD::isBuildVectorAllZeros(V.getOperand(0).getNode()) &&
4991       !ISD::isBuildVectorAllOnes(V.getOperand(0).getNode()) &&
4992       V.getOperand(0).hasOneUse()) {
4993     APInt UndefElts;
4994     SmallVector<APInt> EltBits;
4995     if (getTargetConstantBitsFromNode(V.getOperand(0),
4996                                       V.getScalarValueSizeInBits(), UndefElts,
4997                                       EltBits)) {
4998       // Don't fold min_signed_value -> (min_signed_value - 1)
4999       bool MinSigned = false;
5000       for (APInt &Elt : EltBits) {
5001         MinSigned |= Elt.isMinSignedValue();
5002         Elt -= 1;
5003       }
5004       if (!MinSigned) {
5005         SDLoc DL(V);
5006         MVT VT = V.getSimpleValueType();
5007         return DAG.getNode(X86ISD::PCMPGT, DL, VT, V.getOperand(1),
5008                            getConstVector(EltBits, UndefElts, VT, DAG, DL));
5009       }
5010     }
5011   }
5012   SmallVector<SDValue, 2> CatOps;
5013   if (collectConcatOps(V.getNode(), CatOps, DAG)) {
5014     for (SDValue &CatOp : CatOps) {
5015       SDValue NotCat = IsNOT(CatOp, DAG);
5016       if (!NotCat) return SDValue();
5017       CatOp = DAG.getBitcast(CatOp.getValueType(), NotCat);
5018     }
5019     return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(V), V.getValueType(), CatOps);
5020   }
5021   return SDValue();
5022 }
5023 
5024 /// Create a shuffle mask that matches the PACKSS/PACKUS truncation.
5025 /// A multi-stage pack shuffle mask is created by specifying NumStages > 1.
5026 /// Note: This ignores saturation, so inputs must be checked first.
5027 static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
5028                                   bool Unary, unsigned NumStages = 1) {
5029   assert(Mask.empty() && "Expected an empty shuffle mask vector");
5030   unsigned NumElts = VT.getVectorNumElements();
5031   unsigned NumLanes = VT.getSizeInBits() / 128;
5032   unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits();
5033   unsigned Offset = Unary ? 0 : NumElts;
5034   unsigned Repetitions = 1u << (NumStages - 1);
5035   unsigned Increment = 1u << NumStages;
5036   assert((NumEltsPerLane >> NumStages) > 0 && "Illegal packing compaction");
5037 
5038   for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
5039     for (unsigned Stage = 0; Stage != Repetitions; ++Stage) {
5040       for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
5041         Mask.push_back(Elt + (Lane * NumEltsPerLane));
5042       for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
5043         Mask.push_back(Elt + (Lane * NumEltsPerLane) + Offset);
5044     }
5045   }
5046 }
5047 
5048 // Split the demanded elts of a PACKSS/PACKUS node between its operands.
5049 static void getPackDemandedElts(EVT VT, const APInt &DemandedElts,
5050                                 APInt &DemandedLHS, APInt &DemandedRHS) {
5051   int NumLanes = VT.getSizeInBits() / 128;
5052   int NumElts = DemandedElts.getBitWidth();
5053   int NumInnerElts = NumElts / 2;
5054   int NumEltsPerLane = NumElts / NumLanes;
5055   int NumInnerEltsPerLane = NumInnerElts / NumLanes;
5056 
5057   DemandedLHS = APInt::getZero(NumInnerElts);
5058   DemandedRHS = APInt::getZero(NumInnerElts);
5059 
5060   // Map DemandedElts to the packed operands.
5061   for (int Lane = 0; Lane != NumLanes; ++Lane) {
5062     for (int Elt = 0; Elt != NumInnerEltsPerLane; ++Elt) {
5063       int OuterIdx = (Lane * NumEltsPerLane) + Elt;
5064       int InnerIdx = (Lane * NumInnerEltsPerLane) + Elt;
5065       if (DemandedElts[OuterIdx])
5066         DemandedLHS.setBit(InnerIdx);
5067       if (DemandedElts[OuterIdx + NumInnerEltsPerLane])
5068         DemandedRHS.setBit(InnerIdx);
5069     }
5070   }
5071 }
5072 
5073 // Split the demanded elts of a HADD/HSUB node between its operands.
5074 static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts,
5075                                  APInt &DemandedLHS, APInt &DemandedRHS) {
5076   int NumLanes = VT.getSizeInBits() / 128;
5077   int NumElts = DemandedElts.getBitWidth();
5078   int NumEltsPerLane = NumElts / NumLanes;
5079   int HalfEltsPerLane = NumEltsPerLane / 2;
5080 
5081   DemandedLHS = APInt::getZero(NumElts);
5082   DemandedRHS = APInt::getZero(NumElts);
5083 
5084   // Map DemandedElts to the horizontal operands.
5085   for (int Idx = 0; Idx != NumElts; ++Idx) {
5086     if (!DemandedElts[Idx])
5087       continue;
5088     int LaneIdx = (Idx / NumEltsPerLane) * NumEltsPerLane;
5089     int LocalIdx = Idx % NumEltsPerLane;
5090     if (LocalIdx < HalfEltsPerLane) {
5091       DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 0);
5092       DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 1);
5093     } else {
5094       LocalIdx -= HalfEltsPerLane;
5095       DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 0);
5096       DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 1);
5097     }
5098   }
5099 }
5100 
5101 /// Calculates the shuffle mask corresponding to the target-specific opcode.
5102 /// If the mask could be calculated, returns it in \p Mask, returns the shuffle
5103 /// operands in \p Ops, and returns true.
5104 /// Sets \p IsUnary to true if only one source is used. Note that this will set
5105 /// IsUnary for shuffles which use a single input multiple times, and in those
5106 /// cases it will adjust the mask to only have indices within that single input.
5107 /// It is an error to call this with non-empty Mask/Ops vectors.
5108 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,
5109                                  SmallVectorImpl<SDValue> &Ops,
5110                                  SmallVectorImpl<int> &Mask, bool &IsUnary) {
5111   unsigned NumElems = VT.getVectorNumElements();
5112   unsigned MaskEltSize = VT.getScalarSizeInBits();
5113   SmallVector<uint64_t, 32> RawMask;
5114   APInt RawUndefs;
5115   uint64_t ImmN;
5116 
5117   assert(Mask.empty() && "getTargetShuffleMask expects an empty Mask vector");
5118   assert(Ops.empty() && "getTargetShuffleMask expects an empty Ops vector");
5119 
5120   IsUnary = false;
5121   bool IsFakeUnary = false;
5122   switch (N->getOpcode()) {
5123   case X86ISD::BLENDI:
5124     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5125     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5126     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5127     DecodeBLENDMask(NumElems, ImmN, Mask);
5128     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5129     break;
5130   case X86ISD::SHUFP:
5131     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5132     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5133     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5134     DecodeSHUFPMask(NumElems, MaskEltSize, ImmN, Mask);
5135     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5136     break;
5137   case X86ISD::INSERTPS:
5138     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5139     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5140     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5141     DecodeINSERTPSMask(ImmN, Mask);
5142     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5143     break;
5144   case X86ISD::EXTRQI:
5145     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5146     if (isa<ConstantSDNode>(N->getOperand(1)) &&
5147         isa<ConstantSDNode>(N->getOperand(2))) {
5148       int BitLen = N->getConstantOperandVal(1);
5149       int BitIdx = N->getConstantOperandVal(2);
5150       DecodeEXTRQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
5151       IsUnary = true;
5152     }
5153     break;
5154   case X86ISD::INSERTQI:
5155     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5156     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5157     if (isa<ConstantSDNode>(N->getOperand(2)) &&
5158         isa<ConstantSDNode>(N->getOperand(3))) {
5159       int BitLen = N->getConstantOperandVal(2);
5160       int BitIdx = N->getConstantOperandVal(3);
5161       DecodeINSERTQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
5162       IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5163     }
5164     break;
5165   case X86ISD::UNPCKH:
5166     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5167     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5168     DecodeUNPCKHMask(NumElems, MaskEltSize, Mask);
5169     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5170     break;
5171   case X86ISD::UNPCKL:
5172     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5173     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5174     DecodeUNPCKLMask(NumElems, MaskEltSize, Mask);
5175     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5176     break;
5177   case X86ISD::MOVHLPS:
5178     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5179     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5180     DecodeMOVHLPSMask(NumElems, Mask);
5181     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5182     break;
5183   case X86ISD::MOVLHPS:
5184     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5185     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5186     DecodeMOVLHPSMask(NumElems, Mask);
5187     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5188     break;
5189   case X86ISD::VALIGN:
5190     assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
5191            "Only 32-bit and 64-bit elements are supported!");
5192     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5193     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5194     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5195     DecodeVALIGNMask(NumElems, ImmN, Mask);
5196     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5197     Ops.push_back(N->getOperand(1));
5198     Ops.push_back(N->getOperand(0));
5199     break;
5200   case X86ISD::PALIGNR:
5201     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5202     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5203     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5204     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5205     DecodePALIGNRMask(NumElems, ImmN, Mask);
5206     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5207     Ops.push_back(N->getOperand(1));
5208     Ops.push_back(N->getOperand(0));
5209     break;
5210   case X86ISD::VSHLDQ:
5211     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5212     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5213     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5214     DecodePSLLDQMask(NumElems, ImmN, Mask);
5215     IsUnary = true;
5216     break;
5217   case X86ISD::VSRLDQ:
5218     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5219     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5220     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5221     DecodePSRLDQMask(NumElems, ImmN, Mask);
5222     IsUnary = true;
5223     break;
5224   case X86ISD::PSHUFD:
5225   case X86ISD::VPERMILPI:
5226     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5227     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5228     DecodePSHUFMask(NumElems, MaskEltSize, ImmN, Mask);
5229     IsUnary = true;
5230     break;
5231   case X86ISD::PSHUFHW:
5232     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5233     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5234     DecodePSHUFHWMask(NumElems, ImmN, Mask);
5235     IsUnary = true;
5236     break;
5237   case X86ISD::PSHUFLW:
5238     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5239     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5240     DecodePSHUFLWMask(NumElems, ImmN, Mask);
5241     IsUnary = true;
5242     break;
5243   case X86ISD::VZEXT_MOVL:
5244     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5245     DecodeZeroMoveLowMask(NumElems, Mask);
5246     IsUnary = true;
5247     break;
5248   case X86ISD::VBROADCAST:
5249     // We only decode broadcasts of same-sized vectors, peeking through to
5250     // extracted subvectors is likely to cause hasOneUse issues with
5251     // SimplifyDemandedBits etc.
5252     if (N->getOperand(0).getValueType() == VT) {
5253       DecodeVectorBroadcast(NumElems, Mask);
5254       IsUnary = true;
5255       break;
5256     }
5257     return false;
5258   case X86ISD::VPERMILPV: {
5259     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5260     IsUnary = true;
5261     SDValue MaskNode = N->getOperand(1);
5262     if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5263                                     RawUndefs)) {
5264       DecodeVPERMILPMask(NumElems, MaskEltSize, RawMask, RawUndefs, Mask);
5265       break;
5266     }
5267     return false;
5268   }
5269   case X86ISD::PSHUFB: {
5270     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5271     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5272     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5273     IsUnary = true;
5274     SDValue MaskNode = N->getOperand(1);
5275     if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
5276       DecodePSHUFBMask(RawMask, RawUndefs, Mask);
5277       break;
5278     }
5279     return false;
5280   }
5281   case X86ISD::VPERMI:
5282     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5283     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5284     DecodeVPERMMask(NumElems, ImmN, Mask);
5285     IsUnary = true;
5286     break;
5287   case X86ISD::MOVSS:
5288   case X86ISD::MOVSD:
5289   case X86ISD::MOVSH:
5290     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5291     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5292     DecodeScalarMoveMask(NumElems, /* IsLoad */ false, Mask);
5293     break;
5294   case X86ISD::VPERM2X128:
5295     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5296     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5297     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5298     DecodeVPERM2X128Mask(NumElems, ImmN, Mask);
5299     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5300     break;
5301   case X86ISD::SHUF128:
5302     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5303     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5304     ImmN = N->getConstantOperandVal(N->getNumOperands() - 1);
5305     decodeVSHUF64x2FamilyMask(NumElems, MaskEltSize, ImmN, Mask);
5306     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5307     break;
5308   case X86ISD::MOVSLDUP:
5309     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5310     DecodeMOVSLDUPMask(NumElems, Mask);
5311     IsUnary = true;
5312     break;
5313   case X86ISD::MOVSHDUP:
5314     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5315     DecodeMOVSHDUPMask(NumElems, Mask);
5316     IsUnary = true;
5317     break;
5318   case X86ISD::MOVDDUP:
5319     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5320     DecodeMOVDDUPMask(NumElems, Mask);
5321     IsUnary = true;
5322     break;
5323   case X86ISD::VPERMIL2: {
5324     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5325     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5326     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5327     SDValue MaskNode = N->getOperand(2);
5328     SDValue CtrlNode = N->getOperand(3);
5329     if (ConstantSDNode *CtrlOp = dyn_cast<ConstantSDNode>(CtrlNode)) {
5330       unsigned CtrlImm = CtrlOp->getZExtValue();
5331       if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5332                                       RawUndefs)) {
5333         DecodeVPERMIL2PMask(NumElems, MaskEltSize, CtrlImm, RawMask, RawUndefs,
5334                             Mask);
5335         break;
5336       }
5337     }
5338     return false;
5339   }
5340   case X86ISD::VPPERM: {
5341     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5342     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5343     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5344     SDValue MaskNode = N->getOperand(2);
5345     if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
5346       DecodeVPPERMMask(RawMask, RawUndefs, Mask);
5347       break;
5348     }
5349     return false;
5350   }
5351   case X86ISD::VPERMV: {
5352     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");
5353     IsUnary = true;
5354     // Unlike most shuffle nodes, VPERMV's mask operand is operand 0.
5355     Ops.push_back(N->getOperand(1));
5356     SDValue MaskNode = N->getOperand(0);
5357     if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5358                                     RawUndefs)) {
5359       DecodeVPERMVMask(RawMask, RawUndefs, Mask);
5360       break;
5361     }
5362     return false;
5363   }
5364   case X86ISD::VPERMV3: {
5365     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");
5366     assert(N->getOperand(2).getValueType() == VT && "Unexpected value type");
5367     IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(2);
5368     // Unlike most shuffle nodes, VPERMV3's mask operand is the middle one.
5369     Ops.push_back(N->getOperand(0));
5370     Ops.push_back(N->getOperand(2));
5371     SDValue MaskNode = N->getOperand(1);
5372     if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5373                                     RawUndefs)) {
5374       DecodeVPERMV3Mask(RawMask, RawUndefs, Mask);
5375       break;
5376     }
5377     return false;
5378   }
5379   default: llvm_unreachable("unknown target shuffle node");
5380   }
5381 
5382   // Empty mask indicates the decode failed.
5383   if (Mask.empty())
5384     return false;
5385 
5386   // Check if we're getting a shuffle mask with zero'd elements.
5387   if (!AllowSentinelZero && isAnyZero(Mask))
5388     return false;
5389 
5390   // If we have a fake unary shuffle, the shuffle mask is spread across two
5391   // inputs that are actually the same node. Re-map the mask to always point
5392   // into the first input.
5393   if (IsFakeUnary)
5394     for (int &M : Mask)
5395       if (M >= (int)Mask.size())
5396         M -= Mask.size();
5397 
5398   // If we didn't already add operands in the opcode-specific code, default to
5399   // adding 1 or 2 operands starting at 0.
5400   if (Ops.empty()) {
5401     Ops.push_back(N->getOperand(0));
5402     if (!IsUnary || IsFakeUnary)
5403       Ops.push_back(N->getOperand(1));
5404   }
5405 
5406   return true;
5407 }
5408 
5409 // Wrapper for getTargetShuffleMask with InUnary;
5410 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,
5411                                  SmallVectorImpl<SDValue> &Ops,
5412                                  SmallVectorImpl<int> &Mask) {
5413   bool IsUnary;
5414   return getTargetShuffleMask(N, VT, AllowSentinelZero, Ops, Mask, IsUnary);
5415 }
5416 
5417 /// Compute whether each element of a shuffle is zeroable.
5418 ///
5419 /// A "zeroable" vector shuffle element is one which can be lowered to zero.
5420 /// Either it is an undef element in the shuffle mask, the element of the input
5421 /// referenced is undef, or the element of the input referenced is known to be
5422 /// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
5423 /// as many lanes with this technique as possible to simplify the remaining
5424 /// shuffle.
5425 static void computeZeroableShuffleElements(ArrayRef<int> Mask,
5426                                            SDValue V1, SDValue V2,
5427                                            APInt &KnownUndef, APInt &KnownZero) {
5428   int Size = Mask.size();
5429   KnownUndef = KnownZero = APInt::getZero(Size);
5430 
5431   V1 = peekThroughBitcasts(V1);
5432   V2 = peekThroughBitcasts(V2);
5433 
5434   bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
5435   bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
5436 
5437   int VectorSizeInBits = V1.getValueSizeInBits();
5438   int ScalarSizeInBits = VectorSizeInBits / Size;
5439   assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
5440 
5441   for (int i = 0; i < Size; ++i) {
5442     int M = Mask[i];
5443     // Handle the easy cases.
5444     if (M < 0) {
5445       KnownUndef.setBit(i);
5446       continue;
5447     }
5448     if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
5449       KnownZero.setBit(i);
5450       continue;
5451     }
5452 
5453     // Determine shuffle input and normalize the mask.
5454     SDValue V = M < Size ? V1 : V2;
5455     M %= Size;
5456 
5457     // Currently we can only search BUILD_VECTOR for UNDEF/ZERO elements.
5458     if (V.getOpcode() != ISD::BUILD_VECTOR)
5459       continue;
5460 
5461     // If the BUILD_VECTOR has fewer elements then the bitcasted portion of
5462     // the (larger) source element must be UNDEF/ZERO.
5463     if ((Size % V.getNumOperands()) == 0) {
5464       int Scale = Size / V->getNumOperands();
5465       SDValue Op = V.getOperand(M / Scale);
5466       if (Op.isUndef())
5467         KnownUndef.setBit(i);
5468       if (X86::isZeroNode(Op))
5469         KnownZero.setBit(i);
5470       else if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) {
5471         APInt Val = Cst->getAPIntValue();
5472         Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
5473         if (Val == 0)
5474           KnownZero.setBit(i);
5475       } else if (ConstantFPSDNode *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
5476         APInt Val = Cst->getValueAPF().bitcastToAPInt();
5477         Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
5478         if (Val == 0)
5479           KnownZero.setBit(i);
5480       }
5481       continue;
5482     }
5483 
5484     // If the BUILD_VECTOR has more elements then all the (smaller) source
5485     // elements must be UNDEF or ZERO.
5486     if ((V.getNumOperands() % Size) == 0) {
5487       int Scale = V->getNumOperands() / Size;
5488       bool AllUndef = true;
5489       bool AllZero = true;
5490       for (int j = 0; j < Scale; ++j) {
5491         SDValue Op = V.getOperand((M * Scale) + j);
5492         AllUndef &= Op.isUndef();
5493         AllZero &= X86::isZeroNode(Op);
5494       }
5495       if (AllUndef)
5496         KnownUndef.setBit(i);
5497       if (AllZero)
5498         KnownZero.setBit(i);
5499       continue;
5500     }
5501   }
5502 }
5503 
5504 /// Decode a target shuffle mask and inputs and see if any values are
5505 /// known to be undef or zero from their inputs.
5506 /// Returns true if the target shuffle mask was decoded.
5507 /// FIXME: Merge this with computeZeroableShuffleElements?
5508 static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
5509                                          SmallVectorImpl<SDValue> &Ops,
5510                                          APInt &KnownUndef, APInt &KnownZero) {
5511   bool IsUnary;
5512   if (!isTargetShuffle(N.getOpcode()))
5513     return false;
5514 
5515   MVT VT = N.getSimpleValueType();
5516   if (!getTargetShuffleMask(N.getNode(), VT, true, Ops, Mask, IsUnary))
5517     return false;
5518 
5519   int Size = Mask.size();
5520   SDValue V1 = Ops[0];
5521   SDValue V2 = IsUnary ? V1 : Ops[1];
5522   KnownUndef = KnownZero = APInt::getZero(Size);
5523 
5524   V1 = peekThroughBitcasts(V1);
5525   V2 = peekThroughBitcasts(V2);
5526 
5527   assert((VT.getSizeInBits() % Size) == 0 &&
5528          "Illegal split of shuffle value type");
5529   unsigned EltSizeInBits = VT.getSizeInBits() / Size;
5530 
5531   // Extract known constant input data.
5532   APInt UndefSrcElts[2];
5533   SmallVector<APInt, 32> SrcEltBits[2];
5534   bool IsSrcConstant[2] = {
5535       getTargetConstantBitsFromNode(V1, EltSizeInBits, UndefSrcElts[0],
5536                                     SrcEltBits[0], true, false),
5537       getTargetConstantBitsFromNode(V2, EltSizeInBits, UndefSrcElts[1],
5538                                     SrcEltBits[1], true, false)};
5539 
5540   for (int i = 0; i < Size; ++i) {
5541     int M = Mask[i];
5542 
5543     // Already decoded as SM_SentinelZero / SM_SentinelUndef.
5544     if (M < 0) {
5545       assert(isUndefOrZero(M) && "Unknown shuffle sentinel value!");
5546       if (SM_SentinelUndef == M)
5547         KnownUndef.setBit(i);
5548       if (SM_SentinelZero == M)
5549         KnownZero.setBit(i);
5550       continue;
5551     }
5552 
5553     // Determine shuffle input and normalize the mask.
5554     unsigned SrcIdx = M / Size;
5555     SDValue V = M < Size ? V1 : V2;
5556     M %= Size;
5557 
5558     // We are referencing an UNDEF input.
5559     if (V.isUndef()) {
5560       KnownUndef.setBit(i);
5561       continue;
5562     }
5563 
5564     // SCALAR_TO_VECTOR - only the first element is defined, and the rest UNDEF.
5565     // TODO: We currently only set UNDEF for integer types - floats use the same
5566     // registers as vectors and many of the scalar folded loads rely on the
5567     // SCALAR_TO_VECTOR pattern.
5568     if (V.getOpcode() == ISD::SCALAR_TO_VECTOR &&
5569         (Size % V.getValueType().getVectorNumElements()) == 0) {
5570       int Scale = Size / V.getValueType().getVectorNumElements();
5571       int Idx = M / Scale;
5572       if (Idx != 0 && !VT.isFloatingPoint())
5573         KnownUndef.setBit(i);
5574       else if (Idx == 0 && X86::isZeroNode(V.getOperand(0)))
5575         KnownZero.setBit(i);
5576       continue;
5577     }
5578 
5579     // INSERT_SUBVECTOR - to widen vectors we often insert them into UNDEF
5580     // base vectors.
5581     if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
5582       SDValue Vec = V.getOperand(0);
5583       int NumVecElts = Vec.getValueType().getVectorNumElements();
5584       if (Vec.isUndef() && Size == NumVecElts) {
5585         int Idx = V.getConstantOperandVal(2);
5586         int NumSubElts = V.getOperand(1).getValueType().getVectorNumElements();
5587         if (M < Idx || (Idx + NumSubElts) <= M)
5588           KnownUndef.setBit(i);
5589       }
5590       continue;
5591     }
5592 
5593     // Attempt to extract from the source's constant bits.
5594     if (IsSrcConstant[SrcIdx]) {
5595       if (UndefSrcElts[SrcIdx][M])
5596         KnownUndef.setBit(i);
5597       else if (SrcEltBits[SrcIdx][M] == 0)
5598         KnownZero.setBit(i);
5599     }
5600   }
5601 
5602   assert(VT.getVectorNumElements() == (unsigned)Size &&
5603          "Different mask size from vector size!");
5604   return true;
5605 }
5606 
5607 // Replace target shuffle mask elements with known undef/zero sentinels.
5608 static void resolveTargetShuffleFromZeroables(SmallVectorImpl<int> &Mask,
5609                                               const APInt &KnownUndef,
5610                                               const APInt &KnownZero,
5611                                               bool ResolveKnownZeros= true) {
5612   unsigned NumElts = Mask.size();
5613   assert(KnownUndef.getBitWidth() == NumElts &&
5614          KnownZero.getBitWidth() == NumElts && "Shuffle mask size mismatch");
5615 
5616   for (unsigned i = 0; i != NumElts; ++i) {
5617     if (KnownUndef[i])
5618       Mask[i] = SM_SentinelUndef;
5619     else if (ResolveKnownZeros && KnownZero[i])
5620       Mask[i] = SM_SentinelZero;
5621   }
5622 }
5623 
5624 // Extract target shuffle mask sentinel elements to known undef/zero bitmasks.
5625 static void resolveZeroablesFromTargetShuffle(const SmallVectorImpl<int> &Mask,
5626                                               APInt &KnownUndef,
5627                                               APInt &KnownZero) {
5628   unsigned NumElts = Mask.size();
5629   KnownUndef = KnownZero = APInt::getZero(NumElts);
5630 
5631   for (unsigned i = 0; i != NumElts; ++i) {
5632     int M = Mask[i];
5633     if (SM_SentinelUndef == M)
5634       KnownUndef.setBit(i);
5635     if (SM_SentinelZero == M)
5636       KnownZero.setBit(i);
5637   }
5638 }
5639 
5640 // Attempt to create a shuffle mask from a VSELECT/BLENDV condition mask.
5641 static bool createShuffleMaskFromVSELECT(SmallVectorImpl<int> &Mask,
5642                                          SDValue Cond, bool IsBLENDV = false) {
5643   EVT CondVT = Cond.getValueType();
5644   unsigned EltSizeInBits = CondVT.getScalarSizeInBits();
5645   unsigned NumElts = CondVT.getVectorNumElements();
5646 
5647   APInt UndefElts;
5648   SmallVector<APInt, 32> EltBits;
5649   if (!getTargetConstantBitsFromNode(Cond, EltSizeInBits, UndefElts, EltBits,
5650                                      true, false))
5651     return false;
5652 
5653   Mask.resize(NumElts, SM_SentinelUndef);
5654 
5655   for (int i = 0; i != (int)NumElts; ++i) {
5656     Mask[i] = i;
5657     // Arbitrarily choose from the 2nd operand if the select condition element
5658     // is undef.
5659     // TODO: Can we do better by matching patterns such as even/odd?
5660     if (UndefElts[i] || (!IsBLENDV && EltBits[i].isZero()) ||
5661         (IsBLENDV && EltBits[i].isNonNegative()))
5662       Mask[i] += NumElts;
5663   }
5664 
5665   return true;
5666 }
5667 
5668 // Forward declaration (for getFauxShuffleMask recursive check).
5669 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
5670                                    SmallVectorImpl<SDValue> &Inputs,
5671                                    SmallVectorImpl<int> &Mask,
5672                                    const SelectionDAG &DAG, unsigned Depth,
5673                                    bool ResolveKnownElts);
5674 
5675 // Attempt to decode ops that could be represented as a shuffle mask.
5676 // The decoded shuffle mask may contain a different number of elements to the
5677 // destination value type.
5678 // TODO: Merge into getTargetShuffleInputs()
5679 static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
5680                                SmallVectorImpl<int> &Mask,
5681                                SmallVectorImpl<SDValue> &Ops,
5682                                const SelectionDAG &DAG, unsigned Depth,
5683                                bool ResolveKnownElts) {
5684   Mask.clear();
5685   Ops.clear();
5686 
5687   MVT VT = N.getSimpleValueType();
5688   unsigned NumElts = VT.getVectorNumElements();
5689   unsigned NumSizeInBits = VT.getSizeInBits();
5690   unsigned NumBitsPerElt = VT.getScalarSizeInBits();
5691   if ((NumBitsPerElt % 8) != 0 || (NumSizeInBits % 8) != 0)
5692     return false;
5693   assert(NumElts == DemandedElts.getBitWidth() && "Unexpected vector size");
5694   unsigned NumSizeInBytes = NumSizeInBits / 8;
5695   unsigned NumBytesPerElt = NumBitsPerElt / 8;
5696 
5697   unsigned Opcode = N.getOpcode();
5698   switch (Opcode) {
5699   case ISD::VECTOR_SHUFFLE: {
5700     // Don't treat ISD::VECTOR_SHUFFLE as a target shuffle so decode it here.
5701     ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(N)->getMask();
5702     if (isUndefOrInRange(ShuffleMask, 0, 2 * NumElts)) {
5703       Mask.append(ShuffleMask.begin(), ShuffleMask.end());
5704       Ops.push_back(N.getOperand(0));
5705       Ops.push_back(N.getOperand(1));
5706       return true;
5707     }
5708     return false;
5709   }
5710   case ISD::AND:
5711   case X86ISD::ANDNP: {
5712     // Attempt to decode as a per-byte mask.
5713     APInt UndefElts;
5714     SmallVector<APInt, 32> EltBits;
5715     SDValue N0 = N.getOperand(0);
5716     SDValue N1 = N.getOperand(1);
5717     bool IsAndN = (X86ISD::ANDNP == Opcode);
5718     uint64_t ZeroMask = IsAndN ? 255 : 0;
5719     if (!getTargetConstantBitsFromNode(IsAndN ? N0 : N1, 8, UndefElts, EltBits))
5720       return false;
5721     // We can't assume an undef src element gives an undef dst - the other src
5722     // might be zero.
5723     if (!UndefElts.isZero())
5724       return false;
5725     for (int i = 0, e = (int)EltBits.size(); i != e; ++i) {
5726       const APInt &ByteBits = EltBits[i];
5727       if (ByteBits != 0 && ByteBits != 255)
5728         return false;
5729       Mask.push_back(ByteBits == ZeroMask ? SM_SentinelZero : i);
5730     }
5731     Ops.push_back(IsAndN ? N1 : N0);
5732     return true;
5733   }
5734   case ISD::OR: {
5735     // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other
5736     // is a valid shuffle index.
5737     SDValue N0 = peekThroughBitcasts(N.getOperand(0));
5738     SDValue N1 = peekThroughBitcasts(N.getOperand(1));
5739     if (!N0.getValueType().isVector() || !N1.getValueType().isVector())
5740       return false;
5741 
5742     SmallVector<int, 64> SrcMask0, SrcMask1;
5743     SmallVector<SDValue, 2> SrcInputs0, SrcInputs1;
5744     APInt Demand0 = APInt::getAllOnes(N0.getValueType().getVectorNumElements());
5745     APInt Demand1 = APInt::getAllOnes(N1.getValueType().getVectorNumElements());
5746     if (!getTargetShuffleInputs(N0, Demand0, SrcInputs0, SrcMask0, DAG,
5747                                 Depth + 1, true) ||
5748         !getTargetShuffleInputs(N1, Demand1, SrcInputs1, SrcMask1, DAG,
5749                                 Depth + 1, true))
5750       return false;
5751 
5752     size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
5753     SmallVector<int, 64> Mask0, Mask1;
5754     narrowShuffleMaskElts(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
5755     narrowShuffleMaskElts(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
5756     for (int i = 0; i != (int)MaskSize; ++i) {
5757       // NOTE: Don't handle SM_SentinelUndef, as we can end up in infinite
5758       // loops converting between OR and BLEND shuffles due to
5759       // canWidenShuffleElements merging away undef elements, meaning we
5760       // fail to recognise the OR as the undef element isn't known zero.
5761       if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero)
5762         Mask.push_back(SM_SentinelZero);
5763       else if (Mask1[i] == SM_SentinelZero)
5764         Mask.push_back(i);
5765       else if (Mask0[i] == SM_SentinelZero)
5766         Mask.push_back(i + MaskSize);
5767       else
5768         return false;
5769     }
5770     Ops.push_back(N0);
5771     Ops.push_back(N1);
5772     return true;
5773   }
5774   case ISD::INSERT_SUBVECTOR: {
5775     SDValue Src = N.getOperand(0);
5776     SDValue Sub = N.getOperand(1);
5777     EVT SubVT = Sub.getValueType();
5778     unsigned NumSubElts = SubVT.getVectorNumElements();
5779     if (!N->isOnlyUserOf(Sub.getNode()))
5780       return false;
5781     SDValue SubBC = peekThroughBitcasts(Sub);
5782     uint64_t InsertIdx = N.getConstantOperandVal(2);
5783     // Handle INSERT_SUBVECTOR(SRC0, EXTRACT_SUBVECTOR(SRC1)).
5784     if (SubBC.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5785         SubBC.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
5786       uint64_t ExtractIdx = SubBC.getConstantOperandVal(1);
5787       SDValue SubBCSrc = SubBC.getOperand(0);
5788       unsigned NumSubSrcBCElts = SubBCSrc.getValueType().getVectorNumElements();
5789       unsigned MaxElts = std::max(NumElts, NumSubSrcBCElts);
5790       assert((MaxElts % NumElts) == 0 && (MaxElts % NumSubSrcBCElts) == 0 &&
5791              "Subvector valuetype mismatch");
5792       InsertIdx *= (MaxElts / NumElts);
5793       ExtractIdx *= (MaxElts / NumSubSrcBCElts);
5794       NumSubElts *= (MaxElts / NumElts);
5795       bool SrcIsUndef = Src.isUndef();
5796       for (int i = 0; i != (int)MaxElts; ++i)
5797         Mask.push_back(SrcIsUndef ? SM_SentinelUndef : i);
5798       for (int i = 0; i != (int)NumSubElts; ++i)
5799         Mask[InsertIdx + i] = (SrcIsUndef ? 0 : MaxElts) + ExtractIdx + i;
5800       if (!SrcIsUndef)
5801         Ops.push_back(Src);
5802       Ops.push_back(SubBCSrc);
5803       return true;
5804     }
5805     // Handle INSERT_SUBVECTOR(SRC0, SHUFFLE(SRC1)).
5806     SmallVector<int, 64> SubMask;
5807     SmallVector<SDValue, 2> SubInputs;
5808     SDValue SubSrc = peekThroughOneUseBitcasts(Sub);
5809     EVT SubSrcVT = SubSrc.getValueType();
5810     if (!SubSrcVT.isVector())
5811       return false;
5812 
5813     APInt SubDemand = APInt::getAllOnes(SubSrcVT.getVectorNumElements());
5814     if (!getTargetShuffleInputs(SubSrc, SubDemand, SubInputs, SubMask, DAG,
5815                                 Depth + 1, ResolveKnownElts))
5816       return false;
5817 
5818     // Subvector shuffle inputs must not be larger than the subvector.
5819     if (llvm::any_of(SubInputs, [SubVT](SDValue SubInput) {
5820           return SubVT.getFixedSizeInBits() <
5821                  SubInput.getValueSizeInBits().getFixedValue();
5822         }))
5823       return false;
5824 
5825     if (SubMask.size() != NumSubElts) {
5826       assert(((SubMask.size() % NumSubElts) == 0 ||
5827               (NumSubElts % SubMask.size()) == 0) && "Illegal submask scale");
5828       if ((NumSubElts % SubMask.size()) == 0) {
5829         int Scale = NumSubElts / SubMask.size();
5830         SmallVector<int,64> ScaledSubMask;
5831         narrowShuffleMaskElts(Scale, SubMask, ScaledSubMask);
5832         SubMask = ScaledSubMask;
5833       } else {
5834         int Scale = SubMask.size() / NumSubElts;
5835         NumSubElts = SubMask.size();
5836         NumElts *= Scale;
5837         InsertIdx *= Scale;
5838       }
5839     }
5840     Ops.push_back(Src);
5841     Ops.append(SubInputs.begin(), SubInputs.end());
5842     if (ISD::isBuildVectorAllZeros(Src.getNode()))
5843       Mask.append(NumElts, SM_SentinelZero);
5844     else
5845       for (int i = 0; i != (int)NumElts; ++i)
5846         Mask.push_back(i);
5847     for (int i = 0; i != (int)NumSubElts; ++i) {
5848       int M = SubMask[i];
5849       if (0 <= M) {
5850         int InputIdx = M / NumSubElts;
5851         M = (NumElts * (1 + InputIdx)) + (M % NumSubElts);
5852       }
5853       Mask[i + InsertIdx] = M;
5854     }
5855     return true;
5856   }
5857   case X86ISD::PINSRB:
5858   case X86ISD::PINSRW:
5859   case ISD::SCALAR_TO_VECTOR:
5860   case ISD::INSERT_VECTOR_ELT: {
5861     // Match against a insert_vector_elt/scalar_to_vector of an extract from a
5862     // vector, for matching src/dst vector types.
5863     SDValue Scl = N.getOperand(Opcode == ISD::SCALAR_TO_VECTOR ? 0 : 1);
5864 
5865     unsigned DstIdx = 0;
5866     if (Opcode != ISD::SCALAR_TO_VECTOR) {
5867       // Check we have an in-range constant insertion index.
5868       if (!isa<ConstantSDNode>(N.getOperand(2)) ||
5869           N.getConstantOperandAPInt(2).uge(NumElts))
5870         return false;
5871       DstIdx = N.getConstantOperandVal(2);
5872 
5873       // Attempt to recognise an INSERT*(VEC, 0, DstIdx) shuffle pattern.
5874       if (X86::isZeroNode(Scl)) {
5875         Ops.push_back(N.getOperand(0));
5876         for (unsigned i = 0; i != NumElts; ++i)
5877           Mask.push_back(i == DstIdx ? SM_SentinelZero : (int)i);
5878         return true;
5879       }
5880     }
5881 
5882     // Peek through trunc/aext/zext.
5883     // TODO: aext shouldn't require SM_SentinelZero padding.
5884     // TODO: handle shift of scalars.
5885     unsigned MinBitsPerElt = Scl.getScalarValueSizeInBits();
5886     while (Scl.getOpcode() == ISD::TRUNCATE ||
5887            Scl.getOpcode() == ISD::ANY_EXTEND ||
5888            Scl.getOpcode() == ISD::ZERO_EXTEND) {
5889       Scl = Scl.getOperand(0);
5890       MinBitsPerElt =
5891           std::min<unsigned>(MinBitsPerElt, Scl.getScalarValueSizeInBits());
5892     }
5893     if ((MinBitsPerElt % 8) != 0)
5894       return false;
5895 
5896     // Attempt to find the source vector the scalar was extracted from.
5897     SDValue SrcExtract;
5898     if ((Scl.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5899          Scl.getOpcode() == X86ISD::PEXTRW ||
5900          Scl.getOpcode() == X86ISD::PEXTRB) &&
5901         Scl.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
5902       SrcExtract = Scl;
5903     }
5904     if (!SrcExtract || !isa<ConstantSDNode>(SrcExtract.getOperand(1)))
5905       return false;
5906 
5907     SDValue SrcVec = SrcExtract.getOperand(0);
5908     EVT SrcVT = SrcVec.getValueType();
5909     if (!SrcVT.getScalarType().isByteSized())
5910       return false;
5911     unsigned SrcIdx = SrcExtract.getConstantOperandVal(1);
5912     unsigned SrcByte = SrcIdx * (SrcVT.getScalarSizeInBits() / 8);
5913     unsigned DstByte = DstIdx * NumBytesPerElt;
5914     MinBitsPerElt =
5915         std::min<unsigned>(MinBitsPerElt, SrcVT.getScalarSizeInBits());
5916 
5917     // Create 'identity' byte level shuffle mask and then add inserted bytes.
5918     if (Opcode == ISD::SCALAR_TO_VECTOR) {
5919       Ops.push_back(SrcVec);
5920       Mask.append(NumSizeInBytes, SM_SentinelUndef);
5921     } else {
5922       Ops.push_back(SrcVec);
5923       Ops.push_back(N.getOperand(0));
5924       for (int i = 0; i != (int)NumSizeInBytes; ++i)
5925         Mask.push_back(NumSizeInBytes + i);
5926     }
5927 
5928     unsigned MinBytesPerElts = MinBitsPerElt / 8;
5929     MinBytesPerElts = std::min(MinBytesPerElts, NumBytesPerElt);
5930     for (unsigned i = 0; i != MinBytesPerElts; ++i)
5931       Mask[DstByte + i] = SrcByte + i;
5932     for (unsigned i = MinBytesPerElts; i < NumBytesPerElt; ++i)
5933       Mask[DstByte + i] = SM_SentinelZero;
5934     return true;
5935   }
5936   case X86ISD::PACKSS:
5937   case X86ISD::PACKUS: {
5938     SDValue N0 = N.getOperand(0);
5939     SDValue N1 = N.getOperand(1);
5940     assert(N0.getValueType().getVectorNumElements() == (NumElts / 2) &&
5941            N1.getValueType().getVectorNumElements() == (NumElts / 2) &&
5942            "Unexpected input value type");
5943 
5944     APInt EltsLHS, EltsRHS;
5945     getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS);
5946 
5947     // If we know input saturation won't happen (or we don't care for particular
5948     // lanes), we can treat this as a truncation shuffle.
5949     bool Offset0 = false, Offset1 = false;
5950     if (Opcode == X86ISD::PACKSS) {
5951       if ((!(N0.isUndef() || EltsLHS.isZero()) &&
5952            DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) ||
5953           (!(N1.isUndef() || EltsRHS.isZero()) &&
5954            DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt))
5955         return false;
5956       // We can't easily fold ASHR into a shuffle, but if it was feeding a
5957       // PACKSS then it was likely being used for sign-extension for a
5958       // truncation, so just peek through and adjust the mask accordingly.
5959       if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) &&
5960           N0.getConstantOperandAPInt(1) == NumBitsPerElt) {
5961         Offset0 = true;
5962         N0 = N0.getOperand(0);
5963       }
5964       if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) &&
5965           N1.getConstantOperandAPInt(1) == NumBitsPerElt) {
5966         Offset1 = true;
5967         N1 = N1.getOperand(0);
5968       }
5969     } else {
5970       APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt);
5971       if ((!(N0.isUndef() || EltsLHS.isZero()) &&
5972            !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) ||
5973           (!(N1.isUndef() || EltsRHS.isZero()) &&
5974            !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1)))
5975         return false;
5976     }
5977 
5978     bool IsUnary = (N0 == N1);
5979 
5980     Ops.push_back(N0);
5981     if (!IsUnary)
5982       Ops.push_back(N1);
5983 
5984     createPackShuffleMask(VT, Mask, IsUnary);
5985 
5986     if (Offset0 || Offset1) {
5987       for (int &M : Mask)
5988         if ((Offset0 && isInRange(M, 0, NumElts)) ||
5989             (Offset1 && isInRange(M, NumElts, 2 * NumElts)))
5990           ++M;
5991     }
5992     return true;
5993   }
5994   case ISD::VSELECT:
5995   case X86ISD::BLENDV: {
5996     SDValue Cond = N.getOperand(0);
5997     if (createShuffleMaskFromVSELECT(Mask, Cond, Opcode == X86ISD::BLENDV)) {
5998       Ops.push_back(N.getOperand(1));
5999       Ops.push_back(N.getOperand(2));
6000       return true;
6001     }
6002     return false;
6003   }
6004   case X86ISD::VTRUNC: {
6005     SDValue Src = N.getOperand(0);
6006     EVT SrcVT = Src.getValueType();
6007     // Truncated source must be a simple vector.
6008     if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6009         (SrcVT.getScalarSizeInBits() % 8) != 0)
6010       return false;
6011     unsigned NumSrcElts = SrcVT.getVectorNumElements();
6012     unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
6013     unsigned Scale = NumBitsPerSrcElt / NumBitsPerElt;
6014     assert((NumBitsPerSrcElt % NumBitsPerElt) == 0 && "Illegal truncation");
6015     for (unsigned i = 0; i != NumSrcElts; ++i)
6016       Mask.push_back(i * Scale);
6017     Mask.append(NumElts - NumSrcElts, SM_SentinelZero);
6018     Ops.push_back(Src);
6019     return true;
6020   }
6021   case X86ISD::VSHLI:
6022   case X86ISD::VSRLI: {
6023     uint64_t ShiftVal = N.getConstantOperandVal(1);
6024     // Out of range bit shifts are guaranteed to be zero.
6025     if (NumBitsPerElt <= ShiftVal) {
6026       Mask.append(NumElts, SM_SentinelZero);
6027       return true;
6028     }
6029 
6030     // We can only decode 'whole byte' bit shifts as shuffles.
6031     if ((ShiftVal % 8) != 0)
6032       break;
6033 
6034     uint64_t ByteShift = ShiftVal / 8;
6035     Ops.push_back(N.getOperand(0));
6036 
6037     // Clear mask to all zeros and insert the shifted byte indices.
6038     Mask.append(NumSizeInBytes, SM_SentinelZero);
6039 
6040     if (X86ISD::VSHLI == Opcode) {
6041       for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt)
6042         for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
6043           Mask[i + j] = i + j - ByteShift;
6044     } else {
6045       for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt)
6046         for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
6047           Mask[i + j - ByteShift] = i + j;
6048     }
6049     return true;
6050   }
6051   case X86ISD::VROTLI:
6052   case X86ISD::VROTRI: {
6053     // We can only decode 'whole byte' bit rotates as shuffles.
6054     uint64_t RotateVal = N.getConstantOperandAPInt(1).urem(NumBitsPerElt);
6055     if ((RotateVal % 8) != 0)
6056       return false;
6057     Ops.push_back(N.getOperand(0));
6058     int Offset = RotateVal / 8;
6059     Offset = (X86ISD::VROTLI == Opcode ? NumBytesPerElt - Offset : Offset);
6060     for (int i = 0; i != (int)NumElts; ++i) {
6061       int BaseIdx = i * NumBytesPerElt;
6062       for (int j = 0; j != (int)NumBytesPerElt; ++j) {
6063         Mask.push_back(BaseIdx + ((Offset + j) % NumBytesPerElt));
6064       }
6065     }
6066     return true;
6067   }
6068   case X86ISD::VBROADCAST: {
6069     SDValue Src = N.getOperand(0);
6070     if (!Src.getSimpleValueType().isVector()) {
6071       if (Src.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6072           !isNullConstant(Src.getOperand(1)) ||
6073           Src.getOperand(0).getValueType().getScalarType() !=
6074               VT.getScalarType())
6075         return false;
6076       Src = Src.getOperand(0);
6077     }
6078     Ops.push_back(Src);
6079     Mask.append(NumElts, 0);
6080     return true;
6081   }
6082   case ISD::SIGN_EXTEND_VECTOR_INREG: {
6083     SDValue Src = N.getOperand(0);
6084     EVT SrcVT = Src.getValueType();
6085     unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
6086 
6087     // Extended source must be a simple vector.
6088     if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6089         (NumBitsPerSrcElt % 8) != 0)
6090       return false;
6091 
6092     // We can only handle all-signbits extensions.
6093     APInt DemandedSrcElts =
6094         DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
6095     if (DAG.ComputeNumSignBits(Src, DemandedSrcElts) != NumBitsPerSrcElt)
6096       return false;
6097 
6098     assert((NumBitsPerElt % NumBitsPerSrcElt) == 0 && "Unexpected extension");
6099     unsigned Scale = NumBitsPerElt / NumBitsPerSrcElt;
6100     for (unsigned I = 0; I != NumElts; ++I)
6101       Mask.append(Scale, I);
6102     Ops.push_back(Src);
6103     return true;
6104   }
6105   case ISD::ZERO_EXTEND:
6106   case ISD::ANY_EXTEND:
6107   case ISD::ZERO_EXTEND_VECTOR_INREG:
6108   case ISD::ANY_EXTEND_VECTOR_INREG: {
6109     SDValue Src = N.getOperand(0);
6110     EVT SrcVT = Src.getValueType();
6111 
6112     // Extended source must be a simple vector.
6113     if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6114         (SrcVT.getScalarSizeInBits() % 8) != 0)
6115       return false;
6116 
6117     bool IsAnyExtend =
6118         (ISD::ANY_EXTEND == Opcode || ISD::ANY_EXTEND_VECTOR_INREG == Opcode);
6119     DecodeZeroExtendMask(SrcVT.getScalarSizeInBits(), NumBitsPerElt, NumElts,
6120                          IsAnyExtend, Mask);
6121     Ops.push_back(Src);
6122     return true;
6123   }
6124   }
6125 
6126   return false;
6127 }
6128 
6129 /// Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
6130 static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
6131                                               SmallVectorImpl<int> &Mask) {
6132   int MaskWidth = Mask.size();
6133   SmallVector<SDValue, 16> UsedInputs;
6134   for (int i = 0, e = Inputs.size(); i < e; ++i) {
6135     int lo = UsedInputs.size() * MaskWidth;
6136     int hi = lo + MaskWidth;
6137 
6138     // Strip UNDEF input usage.
6139     if (Inputs[i].isUndef())
6140       for (int &M : Mask)
6141         if ((lo <= M) && (M < hi))
6142           M = SM_SentinelUndef;
6143 
6144     // Check for unused inputs.
6145     if (none_of(Mask, [lo, hi](int i) { return (lo <= i) && (i < hi); })) {
6146       for (int &M : Mask)
6147         if (lo <= M)
6148           M -= MaskWidth;
6149       continue;
6150     }
6151 
6152     // Check for repeated inputs.
6153     bool IsRepeat = false;
6154     for (int j = 0, ue = UsedInputs.size(); j != ue; ++j) {
6155       if (UsedInputs[j] != Inputs[i])
6156         continue;
6157       for (int &M : Mask)
6158         if (lo <= M)
6159           M = (M < hi) ? ((M - lo) + (j * MaskWidth)) : (M - MaskWidth);
6160       IsRepeat = true;
6161       break;
6162     }
6163     if (IsRepeat)
6164       continue;
6165 
6166     UsedInputs.push_back(Inputs[i]);
6167   }
6168   Inputs = UsedInputs;
6169 }
6170 
6171 /// Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs
6172 /// and then sets the SM_SentinelUndef and SM_SentinelZero values.
6173 /// Returns true if the target shuffle mask was decoded.
6174 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
6175                                    SmallVectorImpl<SDValue> &Inputs,
6176                                    SmallVectorImpl<int> &Mask,
6177                                    APInt &KnownUndef, APInt &KnownZero,
6178                                    const SelectionDAG &DAG, unsigned Depth,
6179                                    bool ResolveKnownElts) {
6180   if (Depth >= SelectionDAG::MaxRecursionDepth)
6181     return false; // Limit search depth.
6182 
6183   EVT VT = Op.getValueType();
6184   if (!VT.isSimple() || !VT.isVector())
6185     return false;
6186 
6187   if (getTargetShuffleAndZeroables(Op, Mask, Inputs, KnownUndef, KnownZero)) {
6188     if (ResolveKnownElts)
6189       resolveTargetShuffleFromZeroables(Mask, KnownUndef, KnownZero);
6190     return true;
6191   }
6192   if (getFauxShuffleMask(Op, DemandedElts, Mask, Inputs, DAG, Depth,
6193                          ResolveKnownElts)) {
6194     resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
6195     return true;
6196   }
6197   return false;
6198 }
6199 
6200 static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
6201                                    SmallVectorImpl<SDValue> &Inputs,
6202                                    SmallVectorImpl<int> &Mask,
6203                                    const SelectionDAG &DAG, unsigned Depth,
6204                                    bool ResolveKnownElts) {
6205   APInt KnownUndef, KnownZero;
6206   return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, KnownUndef,
6207                                 KnownZero, DAG, Depth, ResolveKnownElts);
6208 }
6209 
6210 static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
6211                                    SmallVectorImpl<int> &Mask,
6212                                    const SelectionDAG &DAG, unsigned Depth = 0,
6213                                    bool ResolveKnownElts = true) {
6214   EVT VT = Op.getValueType();
6215   if (!VT.isSimple() || !VT.isVector())
6216     return false;
6217 
6218   unsigned NumElts = Op.getValueType().getVectorNumElements();
6219   APInt DemandedElts = APInt::getAllOnes(NumElts);
6220   return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, DAG, Depth,
6221                                 ResolveKnownElts);
6222 }
6223 
6224 // Attempt to create a scalar/subvector broadcast from the base MemSDNode.
6225 static SDValue getBROADCAST_LOAD(unsigned Opcode, const SDLoc &DL, EVT VT,
6226                                  EVT MemVT, MemSDNode *Mem, unsigned Offset,
6227                                  SelectionDAG &DAG) {
6228   assert((Opcode == X86ISD::VBROADCAST_LOAD ||
6229           Opcode == X86ISD::SUBV_BROADCAST_LOAD) &&
6230          "Unknown broadcast load type");
6231 
6232   // Ensure this is a simple (non-atomic, non-voltile), temporal read memop.
6233   if (!Mem || !Mem->readMem() || !Mem->isSimple() || Mem->isNonTemporal())
6234     return SDValue();
6235 
6236   SDValue Ptr = DAG.getMemBasePlusOffset(Mem->getBasePtr(),
6237                                          TypeSize::getFixed(Offset), DL);
6238   SDVTList Tys = DAG.getVTList(VT, MVT::Other);
6239   SDValue Ops[] = {Mem->getChain(), Ptr};
6240   SDValue BcstLd = DAG.getMemIntrinsicNode(
6241       Opcode, DL, Tys, Ops, MemVT,
6242       DAG.getMachineFunction().getMachineMemOperand(
6243           Mem->getMemOperand(), Offset, MemVT.getStoreSize()));
6244   DAG.makeEquivalentMemoryOrdering(SDValue(Mem, 1), BcstLd.getValue(1));
6245   return BcstLd;
6246 }
6247 
6248 /// Returns the scalar element that will make up the i'th
6249 /// element of the result of the vector shuffle.
6250 static SDValue getShuffleScalarElt(SDValue Op, unsigned Index,
6251                                    SelectionDAG &DAG, unsigned Depth) {
6252   if (Depth >= SelectionDAG::MaxRecursionDepth)
6253     return SDValue(); // Limit search depth.
6254 
6255   EVT VT = Op.getValueType();
6256   unsigned Opcode = Op.getOpcode();
6257   unsigned NumElems = VT.getVectorNumElements();
6258 
6259   // Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
6260   if (auto *SV = dyn_cast<ShuffleVectorSDNode>(Op)) {
6261     int Elt = SV->getMaskElt(Index);
6262 
6263     if (Elt < 0)
6264       return DAG.getUNDEF(VT.getVectorElementType());
6265 
6266     SDValue Src = (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1);
6267     return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
6268   }
6269 
6270   // Recurse into target specific vector shuffles to find scalars.
6271   if (isTargetShuffle(Opcode)) {
6272     MVT ShufVT = VT.getSimpleVT();
6273     MVT ShufSVT = ShufVT.getVectorElementType();
6274     int NumElems = (int)ShufVT.getVectorNumElements();
6275     SmallVector<int, 16> ShuffleMask;
6276     SmallVector<SDValue, 16> ShuffleOps;
6277     if (!getTargetShuffleMask(Op.getNode(), ShufVT, true, ShuffleOps,
6278                               ShuffleMask))
6279       return SDValue();
6280 
6281     int Elt = ShuffleMask[Index];
6282     if (Elt == SM_SentinelZero)
6283       return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(Op), ShufSVT)
6284                                  : DAG.getConstantFP(+0.0, SDLoc(Op), ShufSVT);
6285     if (Elt == SM_SentinelUndef)
6286       return DAG.getUNDEF(ShufSVT);
6287 
6288     assert(0 <= Elt && Elt < (2 * NumElems) && "Shuffle index out of range");
6289     SDValue Src = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1];
6290     return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
6291   }
6292 
6293   // Recurse into insert_subvector base/sub vector to find scalars.
6294   if (Opcode == ISD::INSERT_SUBVECTOR) {
6295     SDValue Vec = Op.getOperand(0);
6296     SDValue Sub = Op.getOperand(1);
6297     uint64_t SubIdx = Op.getConstantOperandVal(2);
6298     unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
6299 
6300     if (SubIdx <= Index && Index < (SubIdx + NumSubElts))
6301       return getShuffleScalarElt(Sub, Index - SubIdx, DAG, Depth + 1);
6302     return getShuffleScalarElt(Vec, Index, DAG, Depth + 1);
6303   }
6304 
6305   // Recurse into concat_vectors sub vector to find scalars.
6306   if (Opcode == ISD::CONCAT_VECTORS) {
6307     EVT SubVT = Op.getOperand(0).getValueType();
6308     unsigned NumSubElts = SubVT.getVectorNumElements();
6309     uint64_t SubIdx = Index / NumSubElts;
6310     uint64_t SubElt = Index % NumSubElts;
6311     return getShuffleScalarElt(Op.getOperand(SubIdx), SubElt, DAG, Depth + 1);
6312   }
6313 
6314   // Recurse into extract_subvector src vector to find scalars.
6315   if (Opcode == ISD::EXTRACT_SUBVECTOR) {
6316     SDValue Src = Op.getOperand(0);
6317     uint64_t SrcIdx = Op.getConstantOperandVal(1);
6318     return getShuffleScalarElt(Src, Index + SrcIdx, DAG, Depth + 1);
6319   }
6320 
6321   // We only peek through bitcasts of the same vector width.
6322   if (Opcode == ISD::BITCAST) {
6323     SDValue Src = Op.getOperand(0);
6324     EVT SrcVT = Src.getValueType();
6325     if (SrcVT.isVector() && SrcVT.getVectorNumElements() == NumElems)
6326       return getShuffleScalarElt(Src, Index, DAG, Depth + 1);
6327     return SDValue();
6328   }
6329 
6330   // Actual nodes that may contain scalar elements
6331 
6332   // For insert_vector_elt - either return the index matching scalar or recurse
6333   // into the base vector.
6334   if (Opcode == ISD::INSERT_VECTOR_ELT &&
6335       isa<ConstantSDNode>(Op.getOperand(2))) {
6336     if (Op.getConstantOperandAPInt(2) == Index)
6337       return Op.getOperand(1);
6338     return getShuffleScalarElt(Op.getOperand(0), Index, DAG, Depth + 1);
6339   }
6340 
6341   if (Opcode == ISD::SCALAR_TO_VECTOR)
6342     return (Index == 0) ? Op.getOperand(0)
6343                         : DAG.getUNDEF(VT.getVectorElementType());
6344 
6345   if (Opcode == ISD::BUILD_VECTOR)
6346     return Op.getOperand(Index);
6347 
6348   return SDValue();
6349 }
6350 
6351 // Use PINSRB/PINSRW/PINSRD to create a build vector.
6352 static SDValue LowerBuildVectorAsInsert(SDValue Op, const APInt &NonZeroMask,
6353                                         unsigned NumNonZero, unsigned NumZero,
6354                                         SelectionDAG &DAG,
6355                                         const X86Subtarget &Subtarget) {
6356   MVT VT = Op.getSimpleValueType();
6357   unsigned NumElts = VT.getVectorNumElements();
6358   assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) ||
6359           ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) &&
6360          "Illegal vector insertion");
6361 
6362   SDLoc dl(Op);
6363   SDValue V;
6364   bool First = true;
6365 
6366   for (unsigned i = 0; i < NumElts; ++i) {
6367     bool IsNonZero = NonZeroMask[i];
6368     if (!IsNonZero)
6369       continue;
6370 
6371     // If the build vector contains zeros or our first insertion is not the
6372     // first index then insert into zero vector to break any register
6373     // dependency else use SCALAR_TO_VECTOR.
6374     if (First) {
6375       First = false;
6376       if (NumZero || 0 != i)
6377         V = getZeroVector(VT, Subtarget, DAG, dl);
6378       else {
6379         assert(0 == i && "Expected insertion into zero-index");
6380         V = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
6381         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V);
6382         V = DAG.getBitcast(VT, V);
6383         continue;
6384       }
6385     }
6386     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V, Op.getOperand(i),
6387                     DAG.getIntPtrConstant(i, dl));
6388   }
6389 
6390   return V;
6391 }
6392 
6393 /// Custom lower build_vector of v16i8.
6394 static SDValue LowerBuildVectorv16i8(SDValue Op, const APInt &NonZeroMask,
6395                                      unsigned NumNonZero, unsigned NumZero,
6396                                      SelectionDAG &DAG,
6397                                      const X86Subtarget &Subtarget) {
6398   if (NumNonZero > 8 && !Subtarget.hasSSE41())
6399     return SDValue();
6400 
6401   // SSE4.1 - use PINSRB to insert each byte directly.
6402   if (Subtarget.hasSSE41())
6403     return LowerBuildVectorAsInsert(Op, NonZeroMask, NumNonZero, NumZero, DAG,
6404                                     Subtarget);
6405 
6406   SDLoc dl(Op);
6407   SDValue V;
6408 
6409   // Pre-SSE4.1 - merge byte pairs and insert with PINSRW.
6410   // If both the lowest 16-bits are non-zero, then convert to MOVD.
6411   if (!NonZeroMask.extractBits(2, 0).isZero() &&
6412       !NonZeroMask.extractBits(2, 2).isZero()) {
6413     for (unsigned I = 0; I != 4; ++I) {
6414       if (!NonZeroMask[I])
6415         continue;
6416       SDValue Elt = DAG.getZExtOrTrunc(Op.getOperand(I), dl, MVT::i32);
6417       if (I != 0)
6418         Elt = DAG.getNode(ISD::SHL, dl, MVT::i32, Elt,
6419                           DAG.getConstant(I * 8, dl, MVT::i8));
6420       V = V ? DAG.getNode(ISD::OR, dl, MVT::i32, V, Elt) : Elt;
6421     }
6422     assert(V && "Failed to fold v16i8 vector to zero");
6423     V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V);
6424     V = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, V);
6425     V = DAG.getBitcast(MVT::v8i16, V);
6426   }
6427   for (unsigned i = V ? 4 : 0; i < 16; i += 2) {
6428     bool ThisIsNonZero = NonZeroMask[i];
6429     bool NextIsNonZero = NonZeroMask[i + 1];
6430     if (!ThisIsNonZero && !NextIsNonZero)
6431       continue;
6432 
6433     SDValue Elt;
6434     if (ThisIsNonZero) {
6435       if (NumZero || NextIsNonZero)
6436         Elt = DAG.getZExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
6437       else
6438         Elt = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
6439     }
6440 
6441     if (NextIsNonZero) {
6442       SDValue NextElt = Op.getOperand(i + 1);
6443       if (i == 0 && NumZero)
6444         NextElt = DAG.getZExtOrTrunc(NextElt, dl, MVT::i32);
6445       else
6446         NextElt = DAG.getAnyExtOrTrunc(NextElt, dl, MVT::i32);
6447       NextElt = DAG.getNode(ISD::SHL, dl, MVT::i32, NextElt,
6448                             DAG.getConstant(8, dl, MVT::i8));
6449       if (ThisIsNonZero)
6450         Elt = DAG.getNode(ISD::OR, dl, MVT::i32, NextElt, Elt);
6451       else
6452         Elt = NextElt;
6453     }
6454 
6455     // If our first insertion is not the first index or zeros are needed, then
6456     // insert into zero vector. Otherwise, use SCALAR_TO_VECTOR (leaves high
6457     // elements undefined).
6458     if (!V) {
6459       if (i != 0 || NumZero)
6460         V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
6461       else {
6462         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Elt);
6463         V = DAG.getBitcast(MVT::v8i16, V);
6464         continue;
6465       }
6466     }
6467     Elt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Elt);
6468     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, Elt,
6469                     DAG.getIntPtrConstant(i / 2, dl));
6470   }
6471 
6472   return DAG.getBitcast(MVT::v16i8, V);
6473 }
6474 
6475 /// Custom lower build_vector of v8i16.
6476 static SDValue LowerBuildVectorv8i16(SDValue Op, const APInt &NonZeroMask,
6477                                      unsigned NumNonZero, unsigned NumZero,
6478                                      SelectionDAG &DAG,
6479                                      const X86Subtarget &Subtarget) {
6480   if (NumNonZero > 4 && !Subtarget.hasSSE41())
6481     return SDValue();
6482 
6483   // Use PINSRW to insert each byte directly.
6484   return LowerBuildVectorAsInsert(Op, NonZeroMask, NumNonZero, NumZero, DAG,
6485                                   Subtarget);
6486 }
6487 
6488 /// Custom lower build_vector of v4i32 or v4f32.
6489 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
6490                                      const X86Subtarget &Subtarget) {
6491   // If this is a splat of a pair of elements, use MOVDDUP (unless the target
6492   // has XOP; in that case defer lowering to potentially use VPERMIL2PS).
6493   // Because we're creating a less complicated build vector here, we may enable
6494   // further folding of the MOVDDUP via shuffle transforms.
6495   if (Subtarget.hasSSE3() && !Subtarget.hasXOP() &&
6496       Op.getOperand(0) == Op.getOperand(2) &&
6497       Op.getOperand(1) == Op.getOperand(3) &&
6498       Op.getOperand(0) != Op.getOperand(1)) {
6499     SDLoc DL(Op);
6500     MVT VT = Op.getSimpleValueType();
6501     MVT EltVT = VT.getVectorElementType();
6502     // Create a new build vector with the first 2 elements followed by undef
6503     // padding, bitcast to v2f64, duplicate, and bitcast back.
6504     SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
6505                        DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
6506     SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
6507     SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
6508     return DAG.getBitcast(VT, Dup);
6509   }
6510 
6511   // Find all zeroable elements.
6512   std::bitset<4> Zeroable, Undefs;
6513   for (int i = 0; i < 4; ++i) {
6514     SDValue Elt = Op.getOperand(i);
6515     Undefs[i] = Elt.isUndef();
6516     Zeroable[i] = (Elt.isUndef() || X86::isZeroNode(Elt));
6517   }
6518   assert(Zeroable.size() - Zeroable.count() > 1 &&
6519          "We expect at least two non-zero elements!");
6520 
6521   // We only know how to deal with build_vector nodes where elements are either
6522   // zeroable or extract_vector_elt with constant index.
6523   SDValue FirstNonZero;
6524   unsigned FirstNonZeroIdx;
6525   for (unsigned i = 0; i < 4; ++i) {
6526     if (Zeroable[i])
6527       continue;
6528     SDValue Elt = Op.getOperand(i);
6529     if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6530         !isa<ConstantSDNode>(Elt.getOperand(1)))
6531       return SDValue();
6532     // Make sure that this node is extracting from a 128-bit vector.
6533     MVT VT = Elt.getOperand(0).getSimpleValueType();
6534     if (!VT.is128BitVector())
6535       return SDValue();
6536     if (!FirstNonZero.getNode()) {
6537       FirstNonZero = Elt;
6538       FirstNonZeroIdx = i;
6539     }
6540   }
6541 
6542   assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!");
6543   SDValue V1 = FirstNonZero.getOperand(0);
6544   MVT VT = V1.getSimpleValueType();
6545 
6546   // See if this build_vector can be lowered as a blend with zero.
6547   SDValue Elt;
6548   unsigned EltMaskIdx, EltIdx;
6549   int Mask[4];
6550   for (EltIdx = 0; EltIdx < 4; ++EltIdx) {
6551     if (Zeroable[EltIdx]) {
6552       // The zero vector will be on the right hand side.
6553       Mask[EltIdx] = EltIdx+4;
6554       continue;
6555     }
6556 
6557     Elt = Op->getOperand(EltIdx);
6558     // By construction, Elt is a EXTRACT_VECTOR_ELT with constant index.
6559     EltMaskIdx = Elt.getConstantOperandVal(1);
6560     if (Elt.getOperand(0) != V1 || EltMaskIdx != EltIdx)
6561       break;
6562     Mask[EltIdx] = EltIdx;
6563   }
6564 
6565   if (EltIdx == 4) {
6566     // Let the shuffle legalizer deal with blend operations.
6567     SDValue VZeroOrUndef = (Zeroable == Undefs)
6568                                ? DAG.getUNDEF(VT)
6569                                : getZeroVector(VT, Subtarget, DAG, SDLoc(Op));
6570     if (V1.getSimpleValueType() != VT)
6571       V1 = DAG.getBitcast(VT, V1);
6572     return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZeroOrUndef, Mask);
6573   }
6574 
6575   // See if we can lower this build_vector to a INSERTPS.
6576   if (!Subtarget.hasSSE41())
6577     return SDValue();
6578 
6579   SDValue V2 = Elt.getOperand(0);
6580   if (Elt == FirstNonZero && EltIdx == FirstNonZeroIdx)
6581     V1 = SDValue();
6582 
6583   bool CanFold = true;
6584   for (unsigned i = EltIdx + 1; i < 4 && CanFold; ++i) {
6585     if (Zeroable[i])
6586       continue;
6587 
6588     SDValue Current = Op->getOperand(i);
6589     SDValue SrcVector = Current->getOperand(0);
6590     if (!V1.getNode())
6591       V1 = SrcVector;
6592     CanFold = (SrcVector == V1) && (Current.getConstantOperandAPInt(1) == i);
6593   }
6594 
6595   if (!CanFold)
6596     return SDValue();
6597 
6598   assert(V1.getNode() && "Expected at least two non-zero elements!");
6599   if (V1.getSimpleValueType() != MVT::v4f32)
6600     V1 = DAG.getBitcast(MVT::v4f32, V1);
6601   if (V2.getSimpleValueType() != MVT::v4f32)
6602     V2 = DAG.getBitcast(MVT::v4f32, V2);
6603 
6604   // Ok, we can emit an INSERTPS instruction.
6605   unsigned ZMask = Zeroable.to_ulong();
6606 
6607   unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask;
6608   assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
6609   SDLoc DL(Op);
6610   SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
6611                                DAG.getIntPtrConstant(InsertPSMask, DL, true));
6612   return DAG.getBitcast(VT, Result);
6613 }
6614 
6615 /// Return a vector logical shift node.
6616 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
6617                          SelectionDAG &DAG, const TargetLowering &TLI,
6618                          const SDLoc &dl) {
6619   assert(VT.is128BitVector() && "Unknown type for VShift");
6620   MVT ShVT = MVT::v16i8;
6621   unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
6622   SrcOp = DAG.getBitcast(ShVT, SrcOp);
6623   assert(NumBits % 8 == 0 && "Only support byte sized shifts");
6624   SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
6625   return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
6626 }
6627 
6628 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
6629                                       SelectionDAG &DAG) {
6630 
6631   // Check if the scalar load can be widened into a vector load. And if
6632   // the address is "base + cst" see if the cst can be "absorbed" into
6633   // the shuffle mask.
6634   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) {
6635     SDValue Ptr = LD->getBasePtr();
6636     if (!ISD::isNormalLoad(LD) || !LD->isSimple())
6637       return SDValue();
6638     EVT PVT = LD->getValueType(0);
6639     if (PVT != MVT::i32 && PVT != MVT::f32)
6640       return SDValue();
6641 
6642     int FI = -1;
6643     int64_t Offset = 0;
6644     if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) {
6645       FI = FINode->getIndex();
6646       Offset = 0;
6647     } else if (DAG.isBaseWithConstantOffset(Ptr) &&
6648                isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
6649       FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6650       Offset = Ptr.getConstantOperandVal(1);
6651       Ptr = Ptr.getOperand(0);
6652     } else {
6653       return SDValue();
6654     }
6655 
6656     // FIXME: 256-bit vector instructions don't require a strict alignment,
6657     // improve this code to support it better.
6658     Align RequiredAlign(VT.getSizeInBits() / 8);
6659     SDValue Chain = LD->getChain();
6660     // Make sure the stack object alignment is at least 16 or 32.
6661     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6662     MaybeAlign InferredAlign = DAG.InferPtrAlign(Ptr);
6663     if (!InferredAlign || *InferredAlign < RequiredAlign) {
6664       if (MFI.isFixedObjectIndex(FI)) {
6665         // Can't change the alignment. FIXME: It's possible to compute
6666         // the exact stack offset and reference FI + adjust offset instead.
6667         // If someone *really* cares about this. That's the way to implement it.
6668         return SDValue();
6669       } else {
6670         MFI.setObjectAlignment(FI, RequiredAlign);
6671       }
6672     }
6673 
6674     // (Offset % 16 or 32) must be multiple of 4. Then address is then
6675     // Ptr + (Offset & ~15).
6676     if (Offset < 0)
6677       return SDValue();
6678     if ((Offset % RequiredAlign.value()) & 3)
6679       return SDValue();
6680     int64_t StartOffset = Offset & ~int64_t(RequiredAlign.value() - 1);
6681     if (StartOffset) {
6682       SDLoc DL(Ptr);
6683       Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
6684                         DAG.getConstant(StartOffset, DL, Ptr.getValueType()));
6685     }
6686 
6687     int EltNo = (Offset - StartOffset) >> 2;
6688     unsigned NumElems = VT.getVectorNumElements();
6689 
6690     EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
6691     SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
6692                              LD->getPointerInfo().getWithOffset(StartOffset));
6693 
6694     SmallVector<int, 8> Mask(NumElems, EltNo);
6695 
6696     return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), Mask);
6697   }
6698 
6699   return SDValue();
6700 }
6701 
6702 // Recurse to find a LoadSDNode source and the accumulated ByteOffest.
6703 static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
6704   if (ISD::isNON_EXTLoad(Elt.getNode())) {
6705     auto *BaseLd = cast<LoadSDNode>(Elt);
6706     if (!BaseLd->isSimple())
6707       return false;
6708     Ld = BaseLd;
6709     ByteOffset = 0;
6710     return true;
6711   }
6712 
6713   switch (Elt.getOpcode()) {
6714   case ISD::BITCAST:
6715   case ISD::TRUNCATE:
6716   case ISD::SCALAR_TO_VECTOR:
6717     return findEltLoadSrc(Elt.getOperand(0), Ld, ByteOffset);
6718   case ISD::SRL:
6719     if (auto *AmtC = dyn_cast<ConstantSDNode>(Elt.getOperand(1))) {
6720       uint64_t Amt = AmtC->getZExtValue();
6721       if ((Amt % 8) == 0 && findEltLoadSrc(Elt.getOperand(0), Ld, ByteOffset)) {
6722         ByteOffset += Amt / 8;
6723         return true;
6724       }
6725     }
6726     break;
6727   case ISD::EXTRACT_VECTOR_ELT:
6728     if (auto *IdxC = dyn_cast<ConstantSDNode>(Elt.getOperand(1))) {
6729       SDValue Src = Elt.getOperand(0);
6730       unsigned SrcSizeInBits = Src.getScalarValueSizeInBits();
6731       unsigned DstSizeInBits = Elt.getScalarValueSizeInBits();
6732       if (DstSizeInBits == SrcSizeInBits && (SrcSizeInBits % 8) == 0 &&
6733           findEltLoadSrc(Src, Ld, ByteOffset)) {
6734         uint64_t Idx = IdxC->getZExtValue();
6735         ByteOffset += Idx * (SrcSizeInBits / 8);
6736         return true;
6737       }
6738     }
6739     break;
6740   }
6741 
6742   return false;
6743 }
6744 
6745 /// Given the initializing elements 'Elts' of a vector of type 'VT', see if the
6746 /// elements can be replaced by a single large load which has the same value as
6747 /// a build_vector or insert_subvector whose loaded operands are 'Elts'.
6748 ///
6749 /// Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
6750 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
6751                                         const SDLoc &DL, SelectionDAG &DAG,
6752                                         const X86Subtarget &Subtarget,
6753                                         bool IsAfterLegalize) {
6754   if ((VT.getScalarSizeInBits() % 8) != 0)
6755     return SDValue();
6756 
6757   unsigned NumElems = Elts.size();
6758 
6759   int LastLoadedElt = -1;
6760   APInt LoadMask = APInt::getZero(NumElems);
6761   APInt ZeroMask = APInt::getZero(NumElems);
6762   APInt UndefMask = APInt::getZero(NumElems);
6763 
6764   SmallVector<LoadSDNode*, 8> Loads(NumElems, nullptr);
6765   SmallVector<int64_t, 8> ByteOffsets(NumElems, 0);
6766 
6767   // For each element in the initializer, see if we've found a load, zero or an
6768   // undef.
6769   for (unsigned i = 0; i < NumElems; ++i) {
6770     SDValue Elt = peekThroughBitcasts(Elts[i]);
6771     if (!Elt.getNode())
6772       return SDValue();
6773     if (Elt.isUndef()) {
6774       UndefMask.setBit(i);
6775       continue;
6776     }
6777     if (X86::isZeroNode(Elt) || ISD::isBuildVectorAllZeros(Elt.getNode())) {
6778       ZeroMask.setBit(i);
6779       continue;
6780     }
6781 
6782     // Each loaded element must be the correct fractional portion of the
6783     // requested vector load.
6784     unsigned EltSizeInBits = Elt.getValueSizeInBits();
6785     if ((NumElems * EltSizeInBits) != VT.getSizeInBits())
6786       return SDValue();
6787 
6788     if (!findEltLoadSrc(Elt, Loads[i], ByteOffsets[i]) || ByteOffsets[i] < 0)
6789       return SDValue();
6790     unsigned LoadSizeInBits = Loads[i]->getValueSizeInBits(0);
6791     if (((ByteOffsets[i] * 8) + EltSizeInBits) > LoadSizeInBits)
6792       return SDValue();
6793 
6794     LoadMask.setBit(i);
6795     LastLoadedElt = i;
6796   }
6797   assert((ZeroMask.popcount() + UndefMask.popcount() + LoadMask.popcount()) ==
6798              NumElems &&
6799          "Incomplete element masks");
6800 
6801   // Handle Special Cases - all undef or undef/zero.
6802   if (UndefMask.popcount() == NumElems)
6803     return DAG.getUNDEF(VT);
6804   if ((ZeroMask.popcount() + UndefMask.popcount()) == NumElems)
6805     return VT.isInteger() ? DAG.getConstant(0, DL, VT)
6806                           : DAG.getConstantFP(0.0, DL, VT);
6807 
6808   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6809   int FirstLoadedElt = LoadMask.countr_zero();
6810   SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
6811   EVT EltBaseVT = EltBase.getValueType();
6812   assert(EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits() &&
6813          "Register/Memory size mismatch");
6814   LoadSDNode *LDBase = Loads[FirstLoadedElt];
6815   assert(LDBase && "Did not find base load for merging consecutive loads");
6816   unsigned BaseSizeInBits = EltBaseVT.getStoreSizeInBits();
6817   unsigned BaseSizeInBytes = BaseSizeInBits / 8;
6818   int NumLoadedElts = (1 + LastLoadedElt - FirstLoadedElt);
6819   int LoadSizeInBits = NumLoadedElts * BaseSizeInBits;
6820   assert((BaseSizeInBits % 8) == 0 && "Sub-byte element loads detected");
6821 
6822   // TODO: Support offsetting the base load.
6823   if (ByteOffsets[FirstLoadedElt] != 0)
6824     return SDValue();
6825 
6826   // Check to see if the element's load is consecutive to the base load
6827   // or offset from a previous (already checked) load.
6828   auto CheckConsecutiveLoad = [&](LoadSDNode *Base, int EltIdx) {
6829     LoadSDNode *Ld = Loads[EltIdx];
6830     int64_t ByteOffset = ByteOffsets[EltIdx];
6831     if (ByteOffset && (ByteOffset % BaseSizeInBytes) == 0) {
6832       int64_t BaseIdx = EltIdx - (ByteOffset / BaseSizeInBytes);
6833       return (0 <= BaseIdx && BaseIdx < (int)NumElems && LoadMask[BaseIdx] &&
6834               Loads[BaseIdx] == Ld && ByteOffsets[BaseIdx] == 0);
6835     }
6836     return DAG.areNonVolatileConsecutiveLoads(Ld, Base, BaseSizeInBytes,
6837                                               EltIdx - FirstLoadedElt);
6838   };
6839 
6840   // Consecutive loads can contain UNDEFS but not ZERO elements.
6841   // Consecutive loads with UNDEFs and ZEROs elements require a
6842   // an additional shuffle stage to clear the ZERO elements.
6843   bool IsConsecutiveLoad = true;
6844   bool IsConsecutiveLoadWithZeros = true;
6845   for (int i = FirstLoadedElt + 1; i <= LastLoadedElt; ++i) {
6846     if (LoadMask[i]) {
6847       if (!CheckConsecutiveLoad(LDBase, i)) {
6848         IsConsecutiveLoad = false;
6849         IsConsecutiveLoadWithZeros = false;
6850         break;
6851       }
6852     } else if (ZeroMask[i]) {
6853       IsConsecutiveLoad = false;
6854     }
6855   }
6856 
6857   auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
6858     auto MMOFlags = LDBase->getMemOperand()->getFlags();
6859     assert(LDBase->isSimple() &&
6860            "Cannot merge volatile or atomic loads.");
6861     SDValue NewLd =
6862         DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
6863                     LDBase->getPointerInfo(), LDBase->getOriginalAlign(),
6864                     MMOFlags);
6865     for (auto *LD : Loads)
6866       if (LD)
6867         DAG.makeEquivalentMemoryOrdering(LD, NewLd);
6868     return NewLd;
6869   };
6870 
6871   // Check if the base load is entirely dereferenceable.
6872   bool IsDereferenceable = LDBase->getPointerInfo().isDereferenceable(
6873       VT.getSizeInBits() / 8, *DAG.getContext(), DAG.getDataLayout());
6874 
6875   // LOAD - all consecutive load/undefs (must start/end with a load or be
6876   // entirely dereferenceable). If we have found an entire vector of loads and
6877   // undefs, then return a large load of the entire vector width starting at the
6878   // base pointer. If the vector contains zeros, then attempt to shuffle those
6879   // elements.
6880   if (FirstLoadedElt == 0 &&
6881       (NumLoadedElts == (int)NumElems || IsDereferenceable) &&
6882       (IsConsecutiveLoad || IsConsecutiveLoadWithZeros)) {
6883     if (IsAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
6884       return SDValue();
6885 
6886     // Don't create 256-bit non-temporal aligned loads without AVX2 as these
6887     // will lower to regular temporal loads and use the cache.
6888     if (LDBase->isNonTemporal() && LDBase->getAlign() >= Align(32) &&
6889         VT.is256BitVector() && !Subtarget.hasInt256())
6890       return SDValue();
6891 
6892     if (NumElems == 1)
6893       return DAG.getBitcast(VT, Elts[FirstLoadedElt]);
6894 
6895     if (!ZeroMask)
6896       return CreateLoad(VT, LDBase);
6897 
6898     // IsConsecutiveLoadWithZeros - we need to create a shuffle of the loaded
6899     // vector and a zero vector to clear out the zero elements.
6900     if (!IsAfterLegalize && VT.isVector()) {
6901       unsigned NumMaskElts = VT.getVectorNumElements();
6902       if ((NumMaskElts % NumElems) == 0) {
6903         unsigned Scale = NumMaskElts / NumElems;
6904         SmallVector<int, 4> ClearMask(NumMaskElts, -1);
6905         for (unsigned i = 0; i < NumElems; ++i) {
6906           if (UndefMask[i])
6907             continue;
6908           int Offset = ZeroMask[i] ? NumMaskElts : 0;
6909           for (unsigned j = 0; j != Scale; ++j)
6910             ClearMask[(i * Scale) + j] = (i * Scale) + j + Offset;
6911         }
6912         SDValue V = CreateLoad(VT, LDBase);
6913         SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
6914                                    : DAG.getConstantFP(0.0, DL, VT);
6915         return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
6916       }
6917     }
6918   }
6919 
6920   // If the upper half of a ymm/zmm load is undef then just load the lower half.
6921   if (VT.is256BitVector() || VT.is512BitVector()) {
6922     unsigned HalfNumElems = NumElems / 2;
6923     if (UndefMask.extractBits(HalfNumElems, HalfNumElems).isAllOnes()) {
6924       EVT HalfVT =
6925           EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems);
6926       SDValue HalfLD =
6927           EltsFromConsecutiveLoads(HalfVT, Elts.drop_back(HalfNumElems), DL,
6928                                    DAG, Subtarget, IsAfterLegalize);
6929       if (HalfLD)
6930         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
6931                            HalfLD, DAG.getIntPtrConstant(0, DL));
6932     }
6933   }
6934 
6935   // VZEXT_LOAD - consecutive 32/64-bit load/undefs followed by zeros/undefs.
6936   if (IsConsecutiveLoad && FirstLoadedElt == 0 &&
6937       ((LoadSizeInBits == 16 && Subtarget.hasFP16()) || LoadSizeInBits == 32 ||
6938        LoadSizeInBits == 64) &&
6939       ((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))) {
6940     MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSizeInBits)
6941                                       : MVT::getIntegerVT(LoadSizeInBits);
6942     MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSizeInBits);
6943     // Allow v4f32 on SSE1 only targets.
6944     // FIXME: Add more isel patterns so we can just use VT directly.
6945     if (!Subtarget.hasSSE2() && VT == MVT::v4f32)
6946       VecVT = MVT::v4f32;
6947     if (TLI.isTypeLegal(VecVT)) {
6948       SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
6949       SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
6950       SDValue ResNode = DAG.getMemIntrinsicNode(
6951           X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(),
6952           LDBase->getOriginalAlign(), MachineMemOperand::MOLoad);
6953       for (auto *LD : Loads)
6954         if (LD)
6955           DAG.makeEquivalentMemoryOrdering(LD, ResNode);
6956       return DAG.getBitcast(VT, ResNode);
6957     }
6958   }
6959 
6960   // BROADCAST - match the smallest possible repetition pattern, load that
6961   // scalar/subvector element and then broadcast to the entire vector.
6962   if (ZeroMask.isZero() && isPowerOf2_32(NumElems) && Subtarget.hasAVX() &&
6963       (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) {
6964     for (unsigned SubElems = 1; SubElems < NumElems; SubElems *= 2) {
6965       unsigned RepeatSize = SubElems * BaseSizeInBits;
6966       unsigned ScalarSize = std::min(RepeatSize, 64u);
6967       if (!Subtarget.hasAVX2() && ScalarSize < 32)
6968         continue;
6969 
6970       // Don't attempt a 1:N subvector broadcast - it should be caught by
6971       // combineConcatVectorOps, else will cause infinite loops.
6972       if (RepeatSize > ScalarSize && SubElems == 1)
6973         continue;
6974 
6975       bool Match = true;
6976       SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT));
6977       for (unsigned i = 0; i != NumElems && Match; ++i) {
6978         if (!LoadMask[i])
6979           continue;
6980         SDValue Elt = peekThroughBitcasts(Elts[i]);
6981         if (RepeatedLoads[i % SubElems].isUndef())
6982           RepeatedLoads[i % SubElems] = Elt;
6983         else
6984           Match &= (RepeatedLoads[i % SubElems] == Elt);
6985       }
6986 
6987       // We must have loads at both ends of the repetition.
6988       Match &= !RepeatedLoads.front().isUndef();
6989       Match &= !RepeatedLoads.back().isUndef();
6990       if (!Match)
6991         continue;
6992 
6993       EVT RepeatVT =
6994           VT.isInteger() && (RepeatSize != 64 || TLI.isTypeLegal(MVT::i64))
6995               ? EVT::getIntegerVT(*DAG.getContext(), ScalarSize)
6996               : EVT::getFloatingPointVT(ScalarSize);
6997       if (RepeatSize > ScalarSize)
6998         RepeatVT = EVT::getVectorVT(*DAG.getContext(), RepeatVT,
6999                                     RepeatSize / ScalarSize);
7000       EVT BroadcastVT =
7001           EVT::getVectorVT(*DAG.getContext(), RepeatVT.getScalarType(),
7002                            VT.getSizeInBits() / ScalarSize);
7003       if (TLI.isTypeLegal(BroadcastVT)) {
7004         if (SDValue RepeatLoad = EltsFromConsecutiveLoads(
7005                 RepeatVT, RepeatedLoads, DL, DAG, Subtarget, IsAfterLegalize)) {
7006           SDValue Broadcast = RepeatLoad;
7007           if (RepeatSize > ScalarSize) {
7008             while (Broadcast.getValueSizeInBits() < VT.getSizeInBits())
7009               Broadcast = concatSubVectors(Broadcast, Broadcast, DAG, DL);
7010           } else {
7011             if (!Subtarget.hasAVX2() &&
7012                 !X86::mayFoldLoadIntoBroadcastFromMem(
7013                     RepeatLoad, RepeatVT.getScalarType().getSimpleVT(),
7014                     Subtarget,
7015                     /*AssumeSingleUse=*/true))
7016               return SDValue();
7017             Broadcast =
7018                 DAG.getNode(X86ISD::VBROADCAST, DL, BroadcastVT, RepeatLoad);
7019           }
7020           return DAG.getBitcast(VT, Broadcast);
7021         }
7022       }
7023     }
7024   }
7025 
7026   return SDValue();
7027 }
7028 
7029 // Combine a vector ops (shuffles etc.) that is equal to build_vector load1,
7030 // load2, load3, load4, <0, 1, 2, 3> into a vector load if the load addresses
7031 // are consecutive, non-overlapping, and in the right order.
7032 static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL,
7033                                          SelectionDAG &DAG,
7034                                          const X86Subtarget &Subtarget,
7035                                          bool IsAfterLegalize) {
7036   SmallVector<SDValue, 64> Elts;
7037   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
7038     if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) {
7039       Elts.push_back(Elt);
7040       continue;
7041     }
7042     return SDValue();
7043   }
7044   assert(Elts.size() == VT.getVectorNumElements());
7045   return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget,
7046                                   IsAfterLegalize);
7047 }
7048 
7049 static Constant *getConstantVector(MVT VT, ArrayRef<APInt> Bits,
7050                                    const APInt &Undefs, LLVMContext &C) {
7051   unsigned ScalarSize = VT.getScalarSizeInBits();
7052   Type *Ty = EVT(VT.getScalarType()).getTypeForEVT(C);
7053 
7054   auto getConstantScalar = [&](const APInt &Val) -> Constant * {
7055     if (VT.isFloatingPoint()) {
7056       if (ScalarSize == 16)
7057         return ConstantFP::get(C, APFloat(APFloat::IEEEhalf(), Val));
7058       if (ScalarSize == 32)
7059         return ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val));
7060       assert(ScalarSize == 64 && "Unsupported floating point scalar size");
7061       return ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val));
7062     }
7063     return Constant::getIntegerValue(Ty, Val);
7064   };
7065 
7066   SmallVector<Constant *, 32> ConstantVec;
7067   for (unsigned I = 0, E = Bits.size(); I != E; ++I)
7068     ConstantVec.push_back(Undefs[I] ? UndefValue::get(Ty)
7069                                     : getConstantScalar(Bits[I]));
7070 
7071   return ConstantVector::get(ArrayRef<Constant *>(ConstantVec));
7072 }
7073 
7074 static Constant *getConstantVector(MVT VT, const APInt &SplatValue,
7075                                    unsigned SplatBitSize, LLVMContext &C) {
7076   unsigned ScalarSize = VT.getScalarSizeInBits();
7077 
7078   auto getConstantScalar = [&](const APInt &Val) -> Constant * {
7079     if (VT.isFloatingPoint()) {
7080       if (ScalarSize == 16)
7081         return ConstantFP::get(C, APFloat(APFloat::IEEEhalf(), Val));
7082       if (ScalarSize == 32)
7083         return ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val));
7084       assert(ScalarSize == 64 && "Unsupported floating point scalar size");
7085       return ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val));
7086     }
7087     return Constant::getIntegerValue(Type::getIntNTy(C, ScalarSize), Val);
7088   };
7089 
7090   if (ScalarSize == SplatBitSize)
7091     return getConstantScalar(SplatValue);
7092 
7093   unsigned NumElm = SplatBitSize / ScalarSize;
7094   SmallVector<Constant *, 32> ConstantVec;
7095   for (unsigned I = 0; I != NumElm; ++I) {
7096     APInt Val = SplatValue.extractBits(ScalarSize, ScalarSize * I);
7097     ConstantVec.push_back(getConstantScalar(Val));
7098   }
7099   return ConstantVector::get(ArrayRef<Constant *>(ConstantVec));
7100 }
7101 
7102 static bool isFoldableUseOfShuffle(SDNode *N) {
7103   for (auto *U : N->uses()) {
7104     unsigned Opc = U->getOpcode();
7105     // VPERMV/VPERMV3 shuffles can never fold their index operands.
7106     if (Opc == X86ISD::VPERMV && U->getOperand(0).getNode() == N)
7107       return false;
7108     if (Opc == X86ISD::VPERMV3 && U->getOperand(1).getNode() == N)
7109       return false;
7110     if (isTargetShuffle(Opc))
7111       return true;
7112     if (Opc == ISD::BITCAST) // Ignore bitcasts
7113       return isFoldableUseOfShuffle(U);
7114     if (N->hasOneUse()) {
7115       // TODO, there may be some general way to know if a SDNode can
7116       // be folded. We now only know whether an MI is foldable.
7117       if (Opc == X86ISD::VPDPBUSD && U->getOperand(2).getNode() != N)
7118         return false;
7119       return true;
7120     }
7121   }
7122   return false;
7123 }
7124 
7125 /// Attempt to use the vbroadcast instruction to generate a splat value
7126 /// from a splat BUILD_VECTOR which uses:
7127 ///  a. A single scalar load, or a constant.
7128 ///  b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
7129 ///
7130 /// The VBROADCAST node is returned when a pattern is found,
7131 /// or SDValue() otherwise.
7132 static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
7133                                            const X86Subtarget &Subtarget,
7134                                            SelectionDAG &DAG) {
7135   // VBROADCAST requires AVX.
7136   // TODO: Splats could be generated for non-AVX CPUs using SSE
7137   // instructions, but there's less potential gain for only 128-bit vectors.
7138   if (!Subtarget.hasAVX())
7139     return SDValue();
7140 
7141   MVT VT = BVOp->getSimpleValueType(0);
7142   unsigned NumElts = VT.getVectorNumElements();
7143   SDLoc dl(BVOp);
7144 
7145   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
7146          "Unsupported vector type for broadcast.");
7147 
7148   // See if the build vector is a repeating sequence of scalars (inc. splat).
7149   SDValue Ld;
7150   BitVector UndefElements;
7151   SmallVector<SDValue, 16> Sequence;
7152   if (BVOp->getRepeatedSequence(Sequence, &UndefElements)) {
7153     assert((NumElts % Sequence.size()) == 0 && "Sequence doesn't fit.");
7154     if (Sequence.size() == 1)
7155       Ld = Sequence[0];
7156   }
7157 
7158   // Attempt to use VBROADCASTM
7159   // From this pattern:
7160   // a. t0 = (zext_i64 (bitcast_i8 v2i1 X))
7161   // b. t1 = (build_vector t0 t0)
7162   //
7163   // Create (VBROADCASTM v2i1 X)
7164   if (!Sequence.empty() && Subtarget.hasCDI()) {
7165     // If not a splat, are the upper sequence values zeroable?
7166     unsigned SeqLen = Sequence.size();
7167     bool UpperZeroOrUndef =
7168         SeqLen == 1 ||
7169         llvm::all_of(ArrayRef(Sequence).drop_front(), [](SDValue V) {
7170           return !V || V.isUndef() || isNullConstant(V);
7171         });
7172     SDValue Op0 = Sequence[0];
7173     if (UpperZeroOrUndef && ((Op0.getOpcode() == ISD::BITCAST) ||
7174                              (Op0.getOpcode() == ISD::ZERO_EXTEND &&
7175                               Op0.getOperand(0).getOpcode() == ISD::BITCAST))) {
7176       SDValue BOperand = Op0.getOpcode() == ISD::BITCAST
7177                              ? Op0.getOperand(0)
7178                              : Op0.getOperand(0).getOperand(0);
7179       MVT MaskVT = BOperand.getSimpleValueType();
7180       MVT EltType = MVT::getIntegerVT(VT.getScalarSizeInBits() * SeqLen);
7181       if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) ||  // for broadcastmb2q
7182           (EltType == MVT::i32 && MaskVT == MVT::v16i1)) { // for broadcastmw2d
7183         MVT BcstVT = MVT::getVectorVT(EltType, NumElts / SeqLen);
7184         if (!VT.is512BitVector() && !Subtarget.hasVLX()) {
7185           unsigned Scale = 512 / VT.getSizeInBits();
7186           BcstVT = MVT::getVectorVT(EltType, Scale * (NumElts / SeqLen));
7187         }
7188         SDValue Bcst = DAG.getNode(X86ISD::VBROADCASTM, dl, BcstVT, BOperand);
7189         if (BcstVT.getSizeInBits() != VT.getSizeInBits())
7190           Bcst = extractSubVector(Bcst, 0, DAG, dl, VT.getSizeInBits());
7191         return DAG.getBitcast(VT, Bcst);
7192       }
7193     }
7194   }
7195 
7196   unsigned NumUndefElts = UndefElements.count();
7197   if (!Ld || (NumElts - NumUndefElts) <= 1) {
7198     APInt SplatValue, Undef;
7199     unsigned SplatBitSize;
7200     bool HasUndef;
7201     // Check if this is a repeated constant pattern suitable for broadcasting.
7202     if (BVOp->isConstantSplat(SplatValue, Undef, SplatBitSize, HasUndef) &&
7203         SplatBitSize > VT.getScalarSizeInBits() &&
7204         SplatBitSize < VT.getSizeInBits()) {
7205       // Avoid replacing with broadcast when it's a use of a shuffle
7206       // instruction to preserve the present custom lowering of shuffles.
7207       if (isFoldableUseOfShuffle(BVOp))
7208         return SDValue();
7209       // replace BUILD_VECTOR with broadcast of the repeated constants.
7210       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7211       LLVMContext *Ctx = DAG.getContext();
7212       MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
7213       if (SplatBitSize == 32 || SplatBitSize == 64 ||
7214           (SplatBitSize < 32 && Subtarget.hasAVX2())) {
7215         // Load the constant scalar/subvector and broadcast it.
7216         MVT CVT = MVT::getIntegerVT(SplatBitSize);
7217         Constant *C = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx);
7218         SDValue CP = DAG.getConstantPool(C, PVT);
7219         unsigned Repeat = VT.getSizeInBits() / SplatBitSize;
7220 
7221         Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
7222         SDVTList Tys = DAG.getVTList(MVT::getVectorVT(CVT, Repeat), MVT::Other);
7223         SDValue Ops[] = {DAG.getEntryNode(), CP};
7224         MachinePointerInfo MPI =
7225             MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7226         SDValue Brdcst =
7227             DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT,
7228                                     MPI, Alignment, MachineMemOperand::MOLoad);
7229         return DAG.getBitcast(VT, Brdcst);
7230       }
7231       if (SplatBitSize > 64) {
7232         // Load the vector of constants and broadcast it.
7233         Constant *VecC = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx);
7234         SDValue VCP = DAG.getConstantPool(VecC, PVT);
7235         unsigned NumElm = SplatBitSize / VT.getScalarSizeInBits();
7236         MVT VVT = MVT::getVectorVT(VT.getScalarType(), NumElm);
7237         Align Alignment = cast<ConstantPoolSDNode>(VCP)->getAlign();
7238         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7239         SDValue Ops[] = {DAG.getEntryNode(), VCP};
7240         MachinePointerInfo MPI =
7241             MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7242         return DAG.getMemIntrinsicNode(X86ISD::SUBV_BROADCAST_LOAD, dl, Tys,
7243                                        Ops, VVT, MPI, Alignment,
7244                                        MachineMemOperand::MOLoad);
7245       }
7246     }
7247 
7248     // If we are moving a scalar into a vector (Ld must be set and all elements
7249     // but 1 are undef) and that operation is not obviously supported by
7250     // vmovd/vmovq/vmovss/vmovsd, then keep trying to form a broadcast.
7251     // That's better than general shuffling and may eliminate a load to GPR and
7252     // move from scalar to vector register.
7253     if (!Ld || NumElts - NumUndefElts != 1)
7254       return SDValue();
7255     unsigned ScalarSize = Ld.getValueSizeInBits();
7256     if (!(UndefElements[0] || (ScalarSize != 32 && ScalarSize != 64)))
7257       return SDValue();
7258   }
7259 
7260   bool ConstSplatVal =
7261       (Ld.getOpcode() == ISD::Constant || Ld.getOpcode() == ISD::ConstantFP);
7262   bool IsLoad = ISD::isNormalLoad(Ld.getNode());
7263 
7264   // TODO: Handle broadcasts of non-constant sequences.
7265 
7266   // Make sure that all of the users of a non-constant load are from the
7267   // BUILD_VECTOR node.
7268   // FIXME: Is the use count needed for non-constant, non-load case?
7269   if (!ConstSplatVal && !IsLoad && !BVOp->isOnlyUserOf(Ld.getNode()))
7270     return SDValue();
7271 
7272   unsigned ScalarSize = Ld.getValueSizeInBits();
7273   bool IsGE256 = (VT.getSizeInBits() >= 256);
7274 
7275   // When optimizing for size, generate up to 5 extra bytes for a broadcast
7276   // instruction to save 8 or more bytes of constant pool data.
7277   // TODO: If multiple splats are generated to load the same constant,
7278   // it may be detrimental to overall size. There needs to be a way to detect
7279   // that condition to know if this is truly a size win.
7280   bool OptForSize = DAG.shouldOptForSize();
7281 
7282   // Handle broadcasting a single constant scalar from the constant pool
7283   // into a vector.
7284   // On Sandybridge (no AVX2), it is still better to load a constant vector
7285   // from the constant pool and not to broadcast it from a scalar.
7286   // But override that restriction when optimizing for size.
7287   // TODO: Check if splatting is recommended for other AVX-capable CPUs.
7288   if (ConstSplatVal && (Subtarget.hasAVX2() || OptForSize)) {
7289     EVT CVT = Ld.getValueType();
7290     assert(!CVT.isVector() && "Must not broadcast a vector type");
7291 
7292     // Splat f16, f32, i32, v4f64, v4i64 in all cases with AVX2.
7293     // For size optimization, also splat v2f64 and v2i64, and for size opt
7294     // with AVX2, also splat i8 and i16.
7295     // With pattern matching, the VBROADCAST node may become a VMOVDDUP.
7296     if (ScalarSize == 32 ||
7297         (ScalarSize == 64 && (IsGE256 || Subtarget.hasVLX())) ||
7298         CVT == MVT::f16 ||
7299         (OptForSize && (ScalarSize == 64 || Subtarget.hasAVX2()))) {
7300       const Constant *C = nullptr;
7301       if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Ld))
7302         C = CI->getConstantIntValue();
7303       else if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Ld))
7304         C = CF->getConstantFPValue();
7305 
7306       assert(C && "Invalid constant type");
7307 
7308       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7309       SDValue CP =
7310           DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout()));
7311       Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
7312 
7313       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7314       SDValue Ops[] = {DAG.getEntryNode(), CP};
7315       MachinePointerInfo MPI =
7316           MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7317       return DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT,
7318                                      MPI, Alignment, MachineMemOperand::MOLoad);
7319     }
7320   }
7321 
7322   // Handle AVX2 in-register broadcasts.
7323   if (!IsLoad && Subtarget.hasInt256() &&
7324       (ScalarSize == 32 || (IsGE256 && ScalarSize == 64)))
7325     return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7326 
7327   // The scalar source must be a normal load.
7328   if (!IsLoad)
7329     return SDValue();
7330 
7331   // Make sure the non-chain result is only used by this build vector.
7332   if (!Ld->hasNUsesOfValue(NumElts - NumUndefElts, 0))
7333     return SDValue();
7334 
7335   if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
7336       (Subtarget.hasVLX() && ScalarSize == 64)) {
7337     auto *LN = cast<LoadSDNode>(Ld);
7338     SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7339     SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
7340     SDValue BCast =
7341         DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
7342                                 LN->getMemoryVT(), LN->getMemOperand());
7343     DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7344     return BCast;
7345   }
7346 
7347   // The integer check is needed for the 64-bit into 128-bit so it doesn't match
7348   // double since there is no vbroadcastsd xmm
7349   if (Subtarget.hasInt256() && Ld.getValueType().isInteger() &&
7350       (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64)) {
7351     auto *LN = cast<LoadSDNode>(Ld);
7352     SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7353     SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
7354     SDValue BCast =
7355         DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
7356                                 LN->getMemoryVT(), LN->getMemOperand());
7357     DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7358     return BCast;
7359   }
7360 
7361   if (ScalarSize == 16 && Subtarget.hasFP16() && IsGE256)
7362     return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7363 
7364   // Unsupported broadcast.
7365   return SDValue();
7366 }
7367 
7368 /// For an EXTRACT_VECTOR_ELT with a constant index return the real
7369 /// underlying vector and index.
7370 ///
7371 /// Modifies \p ExtractedFromVec to the real vector and returns the real
7372 /// index.
7373 static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
7374                                          SDValue ExtIdx) {
7375   int Idx = ExtIdx->getAsZExtVal();
7376   if (!isa<ShuffleVectorSDNode>(ExtractedFromVec))
7377     return Idx;
7378 
7379   // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already
7380   // lowered this:
7381   //   (extract_vector_elt (v8f32 %1), Constant<6>)
7382   // to:
7383   //   (extract_vector_elt (vector_shuffle<2,u,u,u>
7384   //                           (extract_subvector (v8f32 %0), Constant<4>),
7385   //                           undef)
7386   //                       Constant<0>)
7387   // In this case the vector is the extract_subvector expression and the index
7388   // is 2, as specified by the shuffle.
7389   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(ExtractedFromVec);
7390   SDValue ShuffleVec = SVOp->getOperand(0);
7391   MVT ShuffleVecVT = ShuffleVec.getSimpleValueType();
7392   assert(ShuffleVecVT.getVectorElementType() ==
7393          ExtractedFromVec.getSimpleValueType().getVectorElementType());
7394 
7395   int ShuffleIdx = SVOp->getMaskElt(Idx);
7396   if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) {
7397     ExtractedFromVec = ShuffleVec;
7398     return ShuffleIdx;
7399   }
7400   return Idx;
7401 }
7402 
7403 static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
7404   MVT VT = Op.getSimpleValueType();
7405 
7406   // Skip if insert_vec_elt is not supported.
7407   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7408   if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
7409     return SDValue();
7410 
7411   SDLoc DL(Op);
7412   unsigned NumElems = Op.getNumOperands();
7413 
7414   SDValue VecIn1;
7415   SDValue VecIn2;
7416   SmallVector<unsigned, 4> InsertIndices;
7417   SmallVector<int, 8> Mask(NumElems, -1);
7418 
7419   for (unsigned i = 0; i != NumElems; ++i) {
7420     unsigned Opc = Op.getOperand(i).getOpcode();
7421 
7422     if (Opc == ISD::UNDEF)
7423       continue;
7424 
7425     if (Opc != ISD::EXTRACT_VECTOR_ELT) {
7426       // Quit if more than 1 elements need inserting.
7427       if (InsertIndices.size() > 1)
7428         return SDValue();
7429 
7430       InsertIndices.push_back(i);
7431       continue;
7432     }
7433 
7434     SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
7435     SDValue ExtIdx = Op.getOperand(i).getOperand(1);
7436 
7437     // Quit if non-constant index.
7438     if (!isa<ConstantSDNode>(ExtIdx))
7439       return SDValue();
7440     int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx);
7441 
7442     // Quit if extracted from vector of different type.
7443     if (ExtractedFromVec.getValueType() != VT)
7444       return SDValue();
7445 
7446     if (!VecIn1.getNode())
7447       VecIn1 = ExtractedFromVec;
7448     else if (VecIn1 != ExtractedFromVec) {
7449       if (!VecIn2.getNode())
7450         VecIn2 = ExtractedFromVec;
7451       else if (VecIn2 != ExtractedFromVec)
7452         // Quit if more than 2 vectors to shuffle
7453         return SDValue();
7454     }
7455 
7456     if (ExtractedFromVec == VecIn1)
7457       Mask[i] = Idx;
7458     else if (ExtractedFromVec == VecIn2)
7459       Mask[i] = Idx + NumElems;
7460   }
7461 
7462   if (!VecIn1.getNode())
7463     return SDValue();
7464 
7465   VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
7466   SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
7467 
7468   for (unsigned Idx : InsertIndices)
7469     NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
7470                      DAG.getIntPtrConstant(Idx, DL));
7471 
7472   return NV;
7473 }
7474 
7475 // Lower BUILD_VECTOR operation for v8bf16, v16bf16 and v32bf16 types.
7476 static SDValue LowerBUILD_VECTORvXbf16(SDValue Op, SelectionDAG &DAG,
7477                                        const X86Subtarget &Subtarget) {
7478   MVT VT = Op.getSimpleValueType();
7479   MVT IVT =
7480       VT.changeVectorElementType(Subtarget.hasFP16() ? MVT::f16 : MVT::i16);
7481   SmallVector<SDValue, 16> NewOps;
7482   for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I)
7483     NewOps.push_back(DAG.getBitcast(Subtarget.hasFP16() ? MVT::f16 : MVT::i16,
7484                                     Op.getOperand(I)));
7485   SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps);
7486   return DAG.getBitcast(VT, Res);
7487 }
7488 
7489 // Lower BUILD_VECTOR operation for v8i1 and v16i1 types.
7490 static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
7491                                      const X86Subtarget &Subtarget) {
7492 
7493   MVT VT = Op.getSimpleValueType();
7494   assert((VT.getVectorElementType() == MVT::i1) &&
7495          "Unexpected type in LowerBUILD_VECTORvXi1!");
7496 
7497   SDLoc dl(Op);
7498   if (ISD::isBuildVectorAllZeros(Op.getNode()) ||
7499       ISD::isBuildVectorAllOnes(Op.getNode()))
7500     return Op;
7501 
7502   uint64_t Immediate = 0;
7503   SmallVector<unsigned, 16> NonConstIdx;
7504   bool IsSplat = true;
7505   bool HasConstElts = false;
7506   int SplatIdx = -1;
7507   for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
7508     SDValue In = Op.getOperand(idx);
7509     if (In.isUndef())
7510       continue;
7511     if (auto *InC = dyn_cast<ConstantSDNode>(In)) {
7512       Immediate |= (InC->getZExtValue() & 0x1) << idx;
7513       HasConstElts = true;
7514     } else {
7515       NonConstIdx.push_back(idx);
7516     }
7517     if (SplatIdx < 0)
7518       SplatIdx = idx;
7519     else if (In != Op.getOperand(SplatIdx))
7520       IsSplat = false;
7521   }
7522 
7523   // for splat use " (select i1 splat_elt, all-ones, all-zeroes)"
7524   if (IsSplat) {
7525     // The build_vector allows the scalar element to be larger than the vector
7526     // element type. We need to mask it to use as a condition unless we know
7527     // the upper bits are zero.
7528     // FIXME: Use computeKnownBits instead of checking specific opcode?
7529     SDValue Cond = Op.getOperand(SplatIdx);
7530     assert(Cond.getValueType() == MVT::i8 && "Unexpected VT!");
7531     if (Cond.getOpcode() != ISD::SETCC)
7532       Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond,
7533                          DAG.getConstant(1, dl, MVT::i8));
7534 
7535     // Perform the select in the scalar domain so we can use cmov.
7536     if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
7537       SDValue Select = DAG.getSelect(dl, MVT::i32, Cond,
7538                                      DAG.getAllOnesConstant(dl, MVT::i32),
7539                                      DAG.getConstant(0, dl, MVT::i32));
7540       Select = DAG.getBitcast(MVT::v32i1, Select);
7541       return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Select, Select);
7542     } else {
7543       MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
7544       SDValue Select = DAG.getSelect(dl, ImmVT, Cond,
7545                                      DAG.getAllOnesConstant(dl, ImmVT),
7546                                      DAG.getConstant(0, dl, ImmVT));
7547       MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
7548       Select = DAG.getBitcast(VecVT, Select);
7549       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select,
7550                          DAG.getIntPtrConstant(0, dl));
7551     }
7552   }
7553 
7554   // insert elements one by one
7555   SDValue DstVec;
7556   if (HasConstElts) {
7557     if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
7558       SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
7559       SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
7560       ImmL = DAG.getBitcast(MVT::v32i1, ImmL);
7561       ImmH = DAG.getBitcast(MVT::v32i1, ImmH);
7562       DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH);
7563     } else {
7564       MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
7565       SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
7566       MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
7567       DstVec = DAG.getBitcast(VecVT, Imm);
7568       DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec,
7569                            DAG.getIntPtrConstant(0, dl));
7570     }
7571   } else
7572     DstVec = DAG.getUNDEF(VT);
7573 
7574   for (unsigned InsertIdx : NonConstIdx) {
7575     DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
7576                          Op.getOperand(InsertIdx),
7577                          DAG.getIntPtrConstant(InsertIdx, dl));
7578   }
7579   return DstVec;
7580 }
7581 
7582 LLVM_ATTRIBUTE_UNUSED static bool isHorizOp(unsigned Opcode) {
7583   switch (Opcode) {
7584   case X86ISD::PACKSS:
7585   case X86ISD::PACKUS:
7586   case X86ISD::FHADD:
7587   case X86ISD::FHSUB:
7588   case X86ISD::HADD:
7589   case X86ISD::HSUB:
7590     return true;
7591   }
7592   return false;
7593 }
7594 
7595 /// This is a helper function of LowerToHorizontalOp().
7596 /// This function checks that the build_vector \p N in input implements a
7597 /// 128-bit partial horizontal operation on a 256-bit vector, but that operation
7598 /// may not match the layout of an x86 256-bit horizontal instruction.
7599 /// In other words, if this returns true, then some extraction/insertion will
7600 /// be required to produce a valid horizontal instruction.
7601 ///
7602 /// Parameter \p Opcode defines the kind of horizontal operation to match.
7603 /// For example, if \p Opcode is equal to ISD::ADD, then this function
7604 /// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
7605 /// is equal to ISD::SUB, then this function checks if this is a horizontal
7606 /// arithmetic sub.
7607 ///
7608 /// This function only analyzes elements of \p N whose indices are
7609 /// in range [BaseIdx, LastIdx).
7610 ///
7611 /// TODO: This function was originally used to match both real and fake partial
7612 /// horizontal operations, but the index-matching logic is incorrect for that.
7613 /// See the corrected implementation in isHopBuildVector(). Can we reduce this
7614 /// code because it is only used for partial h-op matching now?
7615 static bool isHorizontalBinOpPart(const BuildVectorSDNode *N, unsigned Opcode,
7616                                   SelectionDAG &DAG,
7617                                   unsigned BaseIdx, unsigned LastIdx,
7618                                   SDValue &V0, SDValue &V1) {
7619   EVT VT = N->getValueType(0);
7620   assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops");
7621   assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!");
7622   assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&
7623          "Invalid Vector in input!");
7624 
7625   bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD);
7626   bool CanFold = true;
7627   unsigned ExpectedVExtractIdx = BaseIdx;
7628   unsigned NumElts = LastIdx - BaseIdx;
7629   V0 = DAG.getUNDEF(VT);
7630   V1 = DAG.getUNDEF(VT);
7631 
7632   // Check if N implements a horizontal binop.
7633   for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) {
7634     SDValue Op = N->getOperand(i + BaseIdx);
7635 
7636     // Skip UNDEFs.
7637     if (Op->isUndef()) {
7638       // Update the expected vector extract index.
7639       if (i * 2 == NumElts)
7640         ExpectedVExtractIdx = BaseIdx;
7641       ExpectedVExtractIdx += 2;
7642       continue;
7643     }
7644 
7645     CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
7646 
7647     if (!CanFold)
7648       break;
7649 
7650     SDValue Op0 = Op.getOperand(0);
7651     SDValue Op1 = Op.getOperand(1);
7652 
7653     // Try to match the following pattern:
7654     // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
7655     CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7656         Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7657         Op0.getOperand(0) == Op1.getOperand(0) &&
7658         isa<ConstantSDNode>(Op0.getOperand(1)) &&
7659         isa<ConstantSDNode>(Op1.getOperand(1)));
7660     if (!CanFold)
7661       break;
7662 
7663     unsigned I0 = Op0.getConstantOperandVal(1);
7664     unsigned I1 = Op1.getConstantOperandVal(1);
7665 
7666     if (i * 2 < NumElts) {
7667       if (V0.isUndef()) {
7668         V0 = Op0.getOperand(0);
7669         if (V0.getValueType() != VT)
7670           return false;
7671       }
7672     } else {
7673       if (V1.isUndef()) {
7674         V1 = Op0.getOperand(0);
7675         if (V1.getValueType() != VT)
7676           return false;
7677       }
7678       if (i * 2 == NumElts)
7679         ExpectedVExtractIdx = BaseIdx;
7680     }
7681 
7682     SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
7683     if (I0 == ExpectedVExtractIdx)
7684       CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
7685     else if (IsCommutable && I1 == ExpectedVExtractIdx) {
7686       // Try to match the following dag sequence:
7687       // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
7688       CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
7689     } else
7690       CanFold = false;
7691 
7692     ExpectedVExtractIdx += 2;
7693   }
7694 
7695   return CanFold;
7696 }
7697 
7698 /// Emit a sequence of two 128-bit horizontal add/sub followed by
7699 /// a concat_vector.
7700 ///
7701 /// This is a helper function of LowerToHorizontalOp().
7702 /// This function expects two 256-bit vectors called V0 and V1.
7703 /// At first, each vector is split into two separate 128-bit vectors.
7704 /// Then, the resulting 128-bit vectors are used to implement two
7705 /// horizontal binary operations.
7706 ///
7707 /// The kind of horizontal binary operation is defined by \p X86Opcode.
7708 ///
7709 /// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to
7710 /// the two new horizontal binop.
7711 /// When Mode is set, the first horizontal binop dag node would take as input
7712 /// the lower 128-bit of V0 and the upper 128-bit of V0. The second
7713 /// horizontal binop dag node would take as input the lower 128-bit of V1
7714 /// and the upper 128-bit of V1.
7715 ///   Example:
7716 ///     HADD V0_LO, V0_HI
7717 ///     HADD V1_LO, V1_HI
7718 ///
7719 /// Otherwise, the first horizontal binop dag node takes as input the lower
7720 /// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
7721 /// dag node takes the upper 128-bit of V0 and the upper 128-bit of V1.
7722 ///   Example:
7723 ///     HADD V0_LO, V1_LO
7724 ///     HADD V0_HI, V1_HI
7725 ///
7726 /// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower
7727 /// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to
7728 /// the upper 128-bits of the result.
7729 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
7730                                      const SDLoc &DL, SelectionDAG &DAG,
7731                                      unsigned X86Opcode, bool Mode,
7732                                      bool isUndefLO, bool isUndefHI) {
7733   MVT VT = V0.getSimpleValueType();
7734   assert(VT.is256BitVector() && VT == V1.getSimpleValueType() &&
7735          "Invalid nodes in input!");
7736 
7737   unsigned NumElts = VT.getVectorNumElements();
7738   SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
7739   SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
7740   SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
7741   SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
7742   MVT NewVT = V0_LO.getSimpleValueType();
7743 
7744   SDValue LO = DAG.getUNDEF(NewVT);
7745   SDValue HI = DAG.getUNDEF(NewVT);
7746 
7747   if (Mode) {
7748     // Don't emit a horizontal binop if the result is expected to be UNDEF.
7749     if (!isUndefLO && !V0->isUndef())
7750       LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
7751     if (!isUndefHI && !V1->isUndef())
7752       HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
7753   } else {
7754     // Don't emit a horizontal binop if the result is expected to be UNDEF.
7755     if (!isUndefLO && (!V0_LO->isUndef() || !V1_LO->isUndef()))
7756       LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
7757 
7758     if (!isUndefHI && (!V0_HI->isUndef() || !V1_HI->isUndef()))
7759       HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
7760   }
7761 
7762   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
7763 }
7764 
7765 /// Returns true iff \p BV builds a vector with the result equivalent to
7766 /// the result of ADDSUB/SUBADD operation.
7767 /// If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1
7768 /// (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters
7769 /// \p Opnd0 and \p Opnd1.
7770 static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
7771                              const X86Subtarget &Subtarget, SelectionDAG &DAG,
7772                              SDValue &Opnd0, SDValue &Opnd1,
7773                              unsigned &NumExtracts,
7774                              bool &IsSubAdd) {
7775 
7776   MVT VT = BV->getSimpleValueType(0);
7777   if (!Subtarget.hasSSE3() || !VT.isFloatingPoint())
7778     return false;
7779 
7780   unsigned NumElts = VT.getVectorNumElements();
7781   SDValue InVec0 = DAG.getUNDEF(VT);
7782   SDValue InVec1 = DAG.getUNDEF(VT);
7783 
7784   NumExtracts = 0;
7785 
7786   // Odd-numbered elements in the input build vector are obtained from
7787   // adding/subtracting two integer/float elements.
7788   // Even-numbered elements in the input build vector are obtained from
7789   // subtracting/adding two integer/float elements.
7790   unsigned Opc[2] = {0, 0};
7791   for (unsigned i = 0, e = NumElts; i != e; ++i) {
7792     SDValue Op = BV->getOperand(i);
7793 
7794     // Skip 'undef' values.
7795     unsigned Opcode = Op.getOpcode();
7796     if (Opcode == ISD::UNDEF)
7797       continue;
7798 
7799     // Early exit if we found an unexpected opcode.
7800     if (Opcode != ISD::FADD && Opcode != ISD::FSUB)
7801       return false;
7802 
7803     SDValue Op0 = Op.getOperand(0);
7804     SDValue Op1 = Op.getOperand(1);
7805 
7806     // Try to match the following pattern:
7807     // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i))
7808     // Early exit if we cannot match that sequence.
7809     if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7810         Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7811         !isa<ConstantSDNode>(Op0.getOperand(1)) ||
7812         Op0.getOperand(1) != Op1.getOperand(1))
7813       return false;
7814 
7815     unsigned I0 = Op0.getConstantOperandVal(1);
7816     if (I0 != i)
7817       return false;
7818 
7819     // We found a valid add/sub node, make sure its the same opcode as previous
7820     // elements for this parity.
7821     if (Opc[i % 2] != 0 && Opc[i % 2] != Opcode)
7822       return false;
7823     Opc[i % 2] = Opcode;
7824 
7825     // Update InVec0 and InVec1.
7826     if (InVec0.isUndef()) {
7827       InVec0 = Op0.getOperand(0);
7828       if (InVec0.getSimpleValueType() != VT)
7829         return false;
7830     }
7831     if (InVec1.isUndef()) {
7832       InVec1 = Op1.getOperand(0);
7833       if (InVec1.getSimpleValueType() != VT)
7834         return false;
7835     }
7836 
7837     // Make sure that operands in input to each add/sub node always
7838     // come from a same pair of vectors.
7839     if (InVec0 != Op0.getOperand(0)) {
7840       if (Opcode == ISD::FSUB)
7841         return false;
7842 
7843       // FADD is commutable. Try to commute the operands
7844       // and then test again.
7845       std::swap(Op0, Op1);
7846       if (InVec0 != Op0.getOperand(0))
7847         return false;
7848     }
7849 
7850     if (InVec1 != Op1.getOperand(0))
7851       return false;
7852 
7853     // Increment the number of extractions done.
7854     ++NumExtracts;
7855   }
7856 
7857   // Ensure we have found an opcode for both parities and that they are
7858   // different. Don't try to fold this build_vector into an ADDSUB/SUBADD if the
7859   // inputs are undef.
7860   if (!Opc[0] || !Opc[1] || Opc[0] == Opc[1] ||
7861       InVec0.isUndef() || InVec1.isUndef())
7862     return false;
7863 
7864   IsSubAdd = Opc[0] == ISD::FADD;
7865 
7866   Opnd0 = InVec0;
7867   Opnd1 = InVec1;
7868   return true;
7869 }
7870 
7871 /// Returns true if is possible to fold MUL and an idiom that has already been
7872 /// recognized as ADDSUB/SUBADD(\p Opnd0, \p Opnd1) into
7873 /// FMADDSUB/FMSUBADD(x, y, \p Opnd1). If (and only if) true is returned, the
7874 /// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1, \p Opnd2.
7875 ///
7876 /// Prior to calling this function it should be known that there is some
7877 /// SDNode that potentially can be replaced with an X86ISD::ADDSUB operation
7878 /// using \p Opnd0 and \p Opnd1 as operands. Also, this method is called
7879 /// before replacement of such SDNode with ADDSUB operation. Thus the number
7880 /// of \p Opnd0 uses is expected to be equal to 2.
7881 /// For example, this function may be called for the following IR:
7882 ///    %AB = fmul fast <2 x double> %A, %B
7883 ///    %Sub = fsub fast <2 x double> %AB, %C
7884 ///    %Add = fadd fast <2 x double> %AB, %C
7885 ///    %Addsub = shufflevector <2 x double> %Sub, <2 x double> %Add,
7886 ///                            <2 x i32> <i32 0, i32 3>
7887 /// There is a def for %Addsub here, which potentially can be replaced by
7888 /// X86ISD::ADDSUB operation:
7889 ///    %Addsub = X86ISD::ADDSUB %AB, %C
7890 /// and such ADDSUB can further be replaced with FMADDSUB:
7891 ///    %Addsub = FMADDSUB %A, %B, %C.
7892 ///
7893 /// The main reason why this method is called before the replacement of the
7894 /// recognized ADDSUB idiom with ADDSUB operation is that such replacement
7895 /// is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit
7896 /// FMADDSUB is.
7897 static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget,
7898                                  SelectionDAG &DAG,
7899                                  SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
7900                                  unsigned ExpectedUses) {
7901   if (Opnd0.getOpcode() != ISD::FMUL ||
7902       !Opnd0->hasNUsesOfValue(ExpectedUses, 0) || !Subtarget.hasAnyFMA())
7903     return false;
7904 
7905   // FIXME: These checks must match the similar ones in
7906   // DAGCombiner::visitFADDForFMACombine. It would be good to have one
7907   // function that would answer if it is Ok to fuse MUL + ADD to FMADD
7908   // or MUL + ADDSUB to FMADDSUB.
7909   const TargetOptions &Options = DAG.getTarget().Options;
7910   bool AllowFusion =
7911       (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath);
7912   if (!AllowFusion)
7913     return false;
7914 
7915   Opnd2 = Opnd1;
7916   Opnd1 = Opnd0.getOperand(1);
7917   Opnd0 = Opnd0.getOperand(0);
7918 
7919   return true;
7920 }
7921 
7922 /// Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or
7923 /// 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or
7924 /// X86ISD::FMSUBADD node.
7925 static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
7926                                        const X86Subtarget &Subtarget,
7927                                        SelectionDAG &DAG) {
7928   SDValue Opnd0, Opnd1;
7929   unsigned NumExtracts;
7930   bool IsSubAdd;
7931   if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts,
7932                         IsSubAdd))
7933     return SDValue();
7934 
7935   MVT VT = BV->getSimpleValueType(0);
7936   SDLoc DL(BV);
7937 
7938   // Try to generate X86ISD::FMADDSUB node here.
7939   SDValue Opnd2;
7940   if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) {
7941     unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
7942     return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
7943   }
7944 
7945   // We only support ADDSUB.
7946   if (IsSubAdd)
7947     return SDValue();
7948 
7949   // There are no known X86 targets with 512-bit ADDSUB instructions!
7950   // Convert to blend(fsub,fadd).
7951   if (VT.is512BitVector()) {
7952     SmallVector<int> Mask;
7953     for (int I = 0, E = VT.getVectorNumElements(); I != E; I += 2) {
7954         Mask.push_back(I);
7955         Mask.push_back(I + E + 1);
7956     }
7957     SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1);
7958     SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1);
7959     return DAG.getVectorShuffle(VT, DL, Sub, Add, Mask);
7960   }
7961 
7962   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
7963 }
7964 
7965 static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG,
7966                              unsigned &HOpcode, SDValue &V0, SDValue &V1) {
7967   // Initialize outputs to known values.
7968   MVT VT = BV->getSimpleValueType(0);
7969   HOpcode = ISD::DELETED_NODE;
7970   V0 = DAG.getUNDEF(VT);
7971   V1 = DAG.getUNDEF(VT);
7972 
7973   // x86 256-bit horizontal ops are defined in a non-obvious way. Each 128-bit
7974   // half of the result is calculated independently from the 128-bit halves of
7975   // the inputs, so that makes the index-checking logic below more complicated.
7976   unsigned NumElts = VT.getVectorNumElements();
7977   unsigned GenericOpcode = ISD::DELETED_NODE;
7978   unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1;
7979   unsigned NumEltsIn128Bits = NumElts / Num128BitChunks;
7980   unsigned NumEltsIn64Bits = NumEltsIn128Bits / 2;
7981   for (unsigned i = 0; i != Num128BitChunks; ++i) {
7982     for (unsigned j = 0; j != NumEltsIn128Bits; ++j) {
7983       // Ignore undef elements.
7984       SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
7985       if (Op.isUndef())
7986         continue;
7987 
7988       // If there's an opcode mismatch, we're done.
7989       if (HOpcode != ISD::DELETED_NODE && Op.getOpcode() != GenericOpcode)
7990         return false;
7991 
7992       // Initialize horizontal opcode.
7993       if (HOpcode == ISD::DELETED_NODE) {
7994         GenericOpcode = Op.getOpcode();
7995         switch (GenericOpcode) {
7996         case ISD::ADD: HOpcode = X86ISD::HADD; break;
7997         case ISD::SUB: HOpcode = X86ISD::HSUB; break;
7998         case ISD::FADD: HOpcode = X86ISD::FHADD; break;
7999         case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
8000         default: return false;
8001         }
8002       }
8003 
8004       SDValue Op0 = Op.getOperand(0);
8005       SDValue Op1 = Op.getOperand(1);
8006       if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8007           Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8008           Op0.getOperand(0) != Op1.getOperand(0) ||
8009           !isa<ConstantSDNode>(Op0.getOperand(1)) ||
8010           !isa<ConstantSDNode>(Op1.getOperand(1)) || !Op.hasOneUse())
8011         return false;
8012 
8013       // The source vector is chosen based on which 64-bit half of the
8014       // destination vector is being calculated.
8015       if (j < NumEltsIn64Bits) {
8016         if (V0.isUndef())
8017           V0 = Op0.getOperand(0);
8018       } else {
8019         if (V1.isUndef())
8020           V1 = Op0.getOperand(0);
8021       }
8022 
8023       SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
8024       if (SourceVec != Op0.getOperand(0))
8025         return false;
8026 
8027       // op (extract_vector_elt A, I), (extract_vector_elt A, I+1)
8028       unsigned ExtIndex0 = Op0.getConstantOperandVal(1);
8029       unsigned ExtIndex1 = Op1.getConstantOperandVal(1);
8030       unsigned ExpectedIndex = i * NumEltsIn128Bits +
8031                                (j % NumEltsIn64Bits) * 2;
8032       if (ExpectedIndex == ExtIndex0 && ExtIndex1 == ExtIndex0 + 1)
8033         continue;
8034 
8035       // If this is not a commutative op, this does not match.
8036       if (GenericOpcode != ISD::ADD && GenericOpcode != ISD::FADD)
8037         return false;
8038 
8039       // Addition is commutative, so try swapping the extract indexes.
8040       // op (extract_vector_elt A, I+1), (extract_vector_elt A, I)
8041       if (ExpectedIndex == ExtIndex1 && ExtIndex0 == ExtIndex1 + 1)
8042         continue;
8043 
8044       // Extract indexes do not match horizontal requirement.
8045       return false;
8046     }
8047   }
8048   // We matched. Opcode and operands are returned by reference as arguments.
8049   return true;
8050 }
8051 
8052 static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
8053                                     SelectionDAG &DAG, unsigned HOpcode,
8054                                     SDValue V0, SDValue V1) {
8055   // If either input vector is not the same size as the build vector,
8056   // extract/insert the low bits to the correct size.
8057   // This is free (examples: zmm --> xmm, xmm --> ymm).
8058   MVT VT = BV->getSimpleValueType(0);
8059   unsigned Width = VT.getSizeInBits();
8060   if (V0.getValueSizeInBits() > Width)
8061     V0 = extractSubVector(V0, 0, DAG, SDLoc(BV), Width);
8062   else if (V0.getValueSizeInBits() < Width)
8063     V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, SDLoc(BV), Width);
8064 
8065   if (V1.getValueSizeInBits() > Width)
8066     V1 = extractSubVector(V1, 0, DAG, SDLoc(BV), Width);
8067   else if (V1.getValueSizeInBits() < Width)
8068     V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, SDLoc(BV), Width);
8069 
8070   unsigned NumElts = VT.getVectorNumElements();
8071   APInt DemandedElts = APInt::getAllOnes(NumElts);
8072   for (unsigned i = 0; i != NumElts; ++i)
8073     if (BV->getOperand(i).isUndef())
8074       DemandedElts.clearBit(i);
8075 
8076   // If we don't need the upper xmm, then perform as a xmm hop.
8077   unsigned HalfNumElts = NumElts / 2;
8078   if (VT.is256BitVector() && DemandedElts.lshr(HalfNumElts) == 0) {
8079     MVT HalfVT = VT.getHalfNumVectorElementsVT();
8080     V0 = extractSubVector(V0, 0, DAG, SDLoc(BV), 128);
8081     V1 = extractSubVector(V1, 0, DAG, SDLoc(BV), 128);
8082     SDValue Half = DAG.getNode(HOpcode, SDLoc(BV), HalfVT, V0, V1);
8083     return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, SDLoc(BV), 256);
8084   }
8085 
8086   return DAG.getNode(HOpcode, SDLoc(BV), VT, V0, V1);
8087 }
8088 
8089 /// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
8090 static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
8091                                    const X86Subtarget &Subtarget,
8092                                    SelectionDAG &DAG) {
8093   // We need at least 2 non-undef elements to make this worthwhile by default.
8094   unsigned NumNonUndefs =
8095       count_if(BV->op_values(), [](SDValue V) { return !V.isUndef(); });
8096   if (NumNonUndefs < 2)
8097     return SDValue();
8098 
8099   // There are 4 sets of horizontal math operations distinguished by type:
8100   // int/FP at 128-bit/256-bit. Each type was introduced with a different
8101   // subtarget feature. Try to match those "native" patterns first.
8102   MVT VT = BV->getSimpleValueType(0);
8103   if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) ||
8104       ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) ||
8105       ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) ||
8106       ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) {
8107     unsigned HOpcode;
8108     SDValue V0, V1;
8109     if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
8110       return getHopForBuildVector(BV, DAG, HOpcode, V0, V1);
8111   }
8112 
8113   // Try harder to match 256-bit ops by using extract/concat.
8114   if (!Subtarget.hasAVX() || !VT.is256BitVector())
8115     return SDValue();
8116 
8117   // Count the number of UNDEF operands in the build_vector in input.
8118   unsigned NumElts = VT.getVectorNumElements();
8119   unsigned Half = NumElts / 2;
8120   unsigned NumUndefsLO = 0;
8121   unsigned NumUndefsHI = 0;
8122   for (unsigned i = 0, e = Half; i != e; ++i)
8123     if (BV->getOperand(i)->isUndef())
8124       NumUndefsLO++;
8125 
8126   for (unsigned i = Half, e = NumElts; i != e; ++i)
8127     if (BV->getOperand(i)->isUndef())
8128       NumUndefsHI++;
8129 
8130   SDLoc DL(BV);
8131   SDValue InVec0, InVec1;
8132   if (VT == MVT::v8i32 || VT == MVT::v16i16) {
8133     SDValue InVec2, InVec3;
8134     unsigned X86Opcode;
8135     bool CanFold = true;
8136 
8137     if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) &&
8138         isHorizontalBinOpPart(BV, ISD::ADD, DAG, Half, NumElts, InVec2,
8139                               InVec3) &&
8140         ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
8141         ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
8142       X86Opcode = X86ISD::HADD;
8143     else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, Half, InVec0,
8144                                    InVec1) &&
8145              isHorizontalBinOpPart(BV, ISD::SUB, DAG, Half, NumElts, InVec2,
8146                                    InVec3) &&
8147              ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
8148              ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
8149       X86Opcode = X86ISD::HSUB;
8150     else
8151       CanFold = false;
8152 
8153     if (CanFold) {
8154       // Do not try to expand this build_vector into a pair of horizontal
8155       // add/sub if we can emit a pair of scalar add/sub.
8156       if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
8157         return SDValue();
8158 
8159       // Convert this build_vector into a pair of horizontal binops followed by
8160       // a concat vector. We must adjust the outputs from the partial horizontal
8161       // matching calls above to account for undefined vector halves.
8162       SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
8163       SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
8164       assert((!V0.isUndef() || !V1.isUndef()) && "Horizontal-op of undefs?");
8165       bool isUndefLO = NumUndefsLO == Half;
8166       bool isUndefHI = NumUndefsHI == Half;
8167       return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO,
8168                                    isUndefHI);
8169     }
8170   }
8171 
8172   if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
8173       VT == MVT::v16i16) {
8174     unsigned X86Opcode;
8175     if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
8176       X86Opcode = X86ISD::HADD;
8177     else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, NumElts, InVec0,
8178                                    InVec1))
8179       X86Opcode = X86ISD::HSUB;
8180     else if (isHorizontalBinOpPart(BV, ISD::FADD, DAG, 0, NumElts, InVec0,
8181                                    InVec1))
8182       X86Opcode = X86ISD::FHADD;
8183     else if (isHorizontalBinOpPart(BV, ISD::FSUB, DAG, 0, NumElts, InVec0,
8184                                    InVec1))
8185       X86Opcode = X86ISD::FHSUB;
8186     else
8187       return SDValue();
8188 
8189     // Don't try to expand this build_vector into a pair of horizontal add/sub
8190     // if we can simply emit a pair of scalar add/sub.
8191     if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
8192       return SDValue();
8193 
8194     // Convert this build_vector into two horizontal add/sub followed by
8195     // a concat vector.
8196     bool isUndefLO = NumUndefsLO == Half;
8197     bool isUndefHI = NumUndefsHI == Half;
8198     return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
8199                                  isUndefLO, isUndefHI);
8200   }
8201 
8202   return SDValue();
8203 }
8204 
8205 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
8206                           SelectionDAG &DAG);
8207 
8208 /// If a BUILD_VECTOR's source elements all apply the same bit operation and
8209 /// one of their operands is constant, lower to a pair of BUILD_VECTOR and
8210 /// just apply the bit to the vectors.
8211 /// NOTE: Its not in our interest to start make a general purpose vectorizer
8212 /// from this, but enough scalar bit operations are created from the later
8213 /// legalization + scalarization stages to need basic support.
8214 static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op,
8215                                        const X86Subtarget &Subtarget,
8216                                        SelectionDAG &DAG) {
8217   SDLoc DL(Op);
8218   MVT VT = Op->getSimpleValueType(0);
8219   unsigned NumElems = VT.getVectorNumElements();
8220   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8221 
8222   // Check that all elements have the same opcode.
8223   // TODO: Should we allow UNDEFS and if so how many?
8224   unsigned Opcode = Op->getOperand(0).getOpcode();
8225   for (unsigned i = 1; i < NumElems; ++i)
8226     if (Opcode != Op->getOperand(i).getOpcode())
8227       return SDValue();
8228 
8229   // TODO: We may be able to add support for other Ops (ADD/SUB + shifts).
8230   bool IsShift = false;
8231   switch (Opcode) {
8232   default:
8233     return SDValue();
8234   case ISD::SHL:
8235   case ISD::SRL:
8236   case ISD::SRA:
8237     IsShift = true;
8238     break;
8239   case ISD::AND:
8240   case ISD::XOR:
8241   case ISD::OR:
8242     // Don't do this if the buildvector is a splat - we'd replace one
8243     // constant with an entire vector.
8244     if (Op->getSplatValue())
8245       return SDValue();
8246     if (!TLI.isOperationLegalOrPromote(Opcode, VT))
8247       return SDValue();
8248     break;
8249   }
8250 
8251   SmallVector<SDValue, 4> LHSElts, RHSElts;
8252   for (SDValue Elt : Op->ops()) {
8253     SDValue LHS = Elt.getOperand(0);
8254     SDValue RHS = Elt.getOperand(1);
8255 
8256     // We expect the canonicalized RHS operand to be the constant.
8257     if (!isa<ConstantSDNode>(RHS))
8258       return SDValue();
8259 
8260     // Extend shift amounts.
8261     if (RHS.getValueSizeInBits() != VT.getScalarSizeInBits()) {
8262       if (!IsShift)
8263         return SDValue();
8264       RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType());
8265     }
8266 
8267     LHSElts.push_back(LHS);
8268     RHSElts.push_back(RHS);
8269   }
8270 
8271   // Limit to shifts by uniform immediates.
8272   // TODO: Only accept vXi8/vXi64 special cases?
8273   // TODO: Permit non-uniform XOP/AVX2/MULLO cases?
8274   if (IsShift && any_of(RHSElts, [&](SDValue V) { return RHSElts[0] != V; }))
8275     return SDValue();
8276 
8277   SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
8278   SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
8279   SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
8280 
8281   if (!IsShift)
8282     return Res;
8283 
8284   // Immediately lower the shift to ensure the constant build vector doesn't
8285   // get converted to a constant pool before the shift is lowered.
8286   return LowerShift(Res, Subtarget, DAG);
8287 }
8288 
8289 /// Create a vector constant without a load. SSE/AVX provide the bare minimum
8290 /// functionality to do this, so it's all zeros, all ones, or some derivation
8291 /// that is cheap to calculate.
8292 static SDValue materializeVectorConstant(SDValue Op, SelectionDAG &DAG,
8293                                          const X86Subtarget &Subtarget) {
8294   SDLoc DL(Op);
8295   MVT VT = Op.getSimpleValueType();
8296 
8297   // Vectors containing all zeros can be matched by pxor and xorps.
8298   if (ISD::isBuildVectorAllZeros(Op.getNode()))
8299     return Op;
8300 
8301   // Vectors containing all ones can be matched by pcmpeqd on 128-bit width
8302   // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use
8303   // vpcmpeqd on 256-bit vectors.
8304   if (Subtarget.hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) {
8305     if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32)
8306       return Op;
8307 
8308     return getOnesVector(VT, DAG, DL);
8309   }
8310 
8311   return SDValue();
8312 }
8313 
8314 /// Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute
8315 /// from a vector of source values and a vector of extraction indices.
8316 /// The vectors might be manipulated to match the type of the permute op.
8317 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
8318                                      SDLoc &DL, SelectionDAG &DAG,
8319                                      const X86Subtarget &Subtarget) {
8320   MVT ShuffleVT = VT;
8321   EVT IndicesVT = EVT(VT).changeVectorElementTypeToInteger();
8322   unsigned NumElts = VT.getVectorNumElements();
8323   unsigned SizeInBits = VT.getSizeInBits();
8324 
8325   // Adjust IndicesVec to match VT size.
8326   assert(IndicesVec.getValueType().getVectorNumElements() >= NumElts &&
8327          "Illegal variable permute mask size");
8328   if (IndicesVec.getValueType().getVectorNumElements() > NumElts) {
8329     // Narrow/widen the indices vector to the correct size.
8330     if (IndicesVec.getValueSizeInBits() > SizeInBits)
8331       IndicesVec = extractSubVector(IndicesVec, 0, DAG, SDLoc(IndicesVec),
8332                                     NumElts * VT.getScalarSizeInBits());
8333     else if (IndicesVec.getValueSizeInBits() < SizeInBits)
8334       IndicesVec = widenSubVector(IndicesVec, false, Subtarget, DAG,
8335                                   SDLoc(IndicesVec), SizeInBits);
8336     // Zero-extend the index elements within the vector.
8337     if (IndicesVec.getValueType().getVectorNumElements() > NumElts)
8338       IndicesVec = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(IndicesVec),
8339                                IndicesVT, IndicesVec);
8340   }
8341   IndicesVec = DAG.getZExtOrTrunc(IndicesVec, SDLoc(IndicesVec), IndicesVT);
8342 
8343   // Handle SrcVec that don't match VT type.
8344   if (SrcVec.getValueSizeInBits() != SizeInBits) {
8345     if ((SrcVec.getValueSizeInBits() % SizeInBits) == 0) {
8346       // Handle larger SrcVec by treating it as a larger permute.
8347       unsigned Scale = SrcVec.getValueSizeInBits() / SizeInBits;
8348       VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts);
8349       IndicesVT = EVT(VT).changeVectorElementTypeToInteger();
8350       IndicesVec = widenSubVector(IndicesVT.getSimpleVT(), IndicesVec, false,
8351                                   Subtarget, DAG, SDLoc(IndicesVec));
8352       SDValue NewSrcVec =
8353           createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
8354       if (NewSrcVec)
8355         return extractSubVector(NewSrcVec, 0, DAG, DL, SizeInBits);
8356       return SDValue();
8357     } else if (SrcVec.getValueSizeInBits() < SizeInBits) {
8358       // Widen smaller SrcVec to match VT.
8359       SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec));
8360     } else
8361       return SDValue();
8362   }
8363 
8364   auto ScaleIndices = [&DAG](SDValue Idx, uint64_t Scale) {
8365     assert(isPowerOf2_64(Scale) && "Illegal variable permute shuffle scale");
8366     EVT SrcVT = Idx.getValueType();
8367     unsigned NumDstBits = SrcVT.getScalarSizeInBits() / Scale;
8368     uint64_t IndexScale = 0;
8369     uint64_t IndexOffset = 0;
8370 
8371     // If we're scaling a smaller permute op, then we need to repeat the
8372     // indices, scaling and offsetting them as well.
8373     // e.g. v4i32 -> v16i8 (Scale = 4)
8374     // IndexScale = v4i32 Splat(4 << 24 | 4 << 16 | 4 << 8 | 4)
8375     // IndexOffset = v4i32 Splat(3 << 24 | 2 << 16 | 1 << 8 | 0)
8376     for (uint64_t i = 0; i != Scale; ++i) {
8377       IndexScale |= Scale << (i * NumDstBits);
8378       IndexOffset |= i << (i * NumDstBits);
8379     }
8380 
8381     Idx = DAG.getNode(ISD::MUL, SDLoc(Idx), SrcVT, Idx,
8382                       DAG.getConstant(IndexScale, SDLoc(Idx), SrcVT));
8383     Idx = DAG.getNode(ISD::ADD, SDLoc(Idx), SrcVT, Idx,
8384                       DAG.getConstant(IndexOffset, SDLoc(Idx), SrcVT));
8385     return Idx;
8386   };
8387 
8388   unsigned Opcode = 0;
8389   switch (VT.SimpleTy) {
8390   default:
8391     break;
8392   case MVT::v16i8:
8393     if (Subtarget.hasSSSE3())
8394       Opcode = X86ISD::PSHUFB;
8395     break;
8396   case MVT::v8i16:
8397     if (Subtarget.hasVLX() && Subtarget.hasBWI())
8398       Opcode = X86ISD::VPERMV;
8399     else if (Subtarget.hasSSSE3()) {
8400       Opcode = X86ISD::PSHUFB;
8401       ShuffleVT = MVT::v16i8;
8402     }
8403     break;
8404   case MVT::v4f32:
8405   case MVT::v4i32:
8406     if (Subtarget.hasAVX()) {
8407       Opcode = X86ISD::VPERMILPV;
8408       ShuffleVT = MVT::v4f32;
8409     } else if (Subtarget.hasSSSE3()) {
8410       Opcode = X86ISD::PSHUFB;
8411       ShuffleVT = MVT::v16i8;
8412     }
8413     break;
8414   case MVT::v2f64:
8415   case MVT::v2i64:
8416     if (Subtarget.hasAVX()) {
8417       // VPERMILPD selects using bit#1 of the index vector, so scale IndicesVec.
8418       IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
8419       Opcode = X86ISD::VPERMILPV;
8420       ShuffleVT = MVT::v2f64;
8421     } else if (Subtarget.hasSSE41()) {
8422       // SSE41 can compare v2i64 - select between indices 0 and 1.
8423       return DAG.getSelectCC(
8424           DL, IndicesVec,
8425           getZeroVector(IndicesVT.getSimpleVT(), Subtarget, DAG, DL),
8426           DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}),
8427           DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}),
8428           ISD::CondCode::SETEQ);
8429     }
8430     break;
8431   case MVT::v32i8:
8432     if (Subtarget.hasVLX() && Subtarget.hasVBMI())
8433       Opcode = X86ISD::VPERMV;
8434     else if (Subtarget.hasXOP()) {
8435       SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
8436       SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
8437       SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
8438       SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
8439       return DAG.getNode(
8440           ISD::CONCAT_VECTORS, DL, VT,
8441           DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx),
8442           DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx));
8443     } else if (Subtarget.hasAVX()) {
8444       SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
8445       SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
8446       SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
8447       SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
8448       auto PSHUFBBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
8449                               ArrayRef<SDValue> Ops) {
8450         // Permute Lo and Hi and then select based on index range.
8451         // This works as SHUFB uses bits[3:0] to permute elements and we don't
8452         // care about the bit[7] as its just an index vector.
8453         SDValue Idx = Ops[2];
8454         EVT VT = Idx.getValueType();
8455         return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT),
8456                                DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx),
8457                                DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx),
8458                                ISD::CondCode::SETGT);
8459       };
8460       SDValue Ops[] = {LoLo, HiHi, IndicesVec};
8461       return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops,
8462                               PSHUFBBuilder);
8463     }
8464     break;
8465   case MVT::v16i16:
8466     if (Subtarget.hasVLX() && Subtarget.hasBWI())
8467       Opcode = X86ISD::VPERMV;
8468     else if (Subtarget.hasAVX()) {
8469       // Scale to v32i8 and perform as v32i8.
8470       IndicesVec = ScaleIndices(IndicesVec, 2);
8471       return DAG.getBitcast(
8472           VT, createVariablePermute(
8473                   MVT::v32i8, DAG.getBitcast(MVT::v32i8, SrcVec),
8474                   DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget));
8475     }
8476     break;
8477   case MVT::v8f32:
8478   case MVT::v8i32:
8479     if (Subtarget.hasAVX2())
8480       Opcode = X86ISD::VPERMV;
8481     else if (Subtarget.hasAVX()) {
8482       SrcVec = DAG.getBitcast(MVT::v8f32, SrcVec);
8483       SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8484                                           {0, 1, 2, 3, 0, 1, 2, 3});
8485       SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8486                                           {4, 5, 6, 7, 4, 5, 6, 7});
8487       if (Subtarget.hasXOP())
8488         return DAG.getBitcast(
8489             VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi,
8490                             IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
8491       // Permute Lo and Hi and then select based on index range.
8492       // This works as VPERMILPS only uses index bits[0:1] to permute elements.
8493       SDValue Res = DAG.getSelectCC(
8494           DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32),
8495           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec),
8496           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec),
8497           ISD::CondCode::SETGT);
8498       return DAG.getBitcast(VT, Res);
8499     }
8500     break;
8501   case MVT::v4i64:
8502   case MVT::v4f64:
8503     if (Subtarget.hasAVX512()) {
8504       if (!Subtarget.hasVLX()) {
8505         MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8);
8506         SrcVec = widenSubVector(WidenSrcVT, SrcVec, false, Subtarget, DAG,
8507                                 SDLoc(SrcVec));
8508         IndicesVec = widenSubVector(MVT::v8i64, IndicesVec, false, Subtarget,
8509                                     DAG, SDLoc(IndicesVec));
8510         SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL,
8511                                             DAG, Subtarget);
8512         return extract256BitVector(Res, 0, DAG, DL);
8513       }
8514       Opcode = X86ISD::VPERMV;
8515     } else if (Subtarget.hasAVX()) {
8516       SrcVec = DAG.getBitcast(MVT::v4f64, SrcVec);
8517       SDValue LoLo =
8518           DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1});
8519       SDValue HiHi =
8520           DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3});
8521       // VPERMIL2PD selects with bit#1 of the index vector, so scale IndicesVec.
8522       IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
8523       if (Subtarget.hasXOP())
8524         return DAG.getBitcast(
8525             VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi,
8526                             IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
8527       // Permute Lo and Hi and then select based on index range.
8528       // This works as VPERMILPD only uses index bit[1] to permute elements.
8529       SDValue Res = DAG.getSelectCC(
8530           DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64),
8531           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec),
8532           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec),
8533           ISD::CondCode::SETGT);
8534       return DAG.getBitcast(VT, Res);
8535     }
8536     break;
8537   case MVT::v64i8:
8538     if (Subtarget.hasVBMI())
8539       Opcode = X86ISD::VPERMV;
8540     break;
8541   case MVT::v32i16:
8542     if (Subtarget.hasBWI())
8543       Opcode = X86ISD::VPERMV;
8544     break;
8545   case MVT::v16f32:
8546   case MVT::v16i32:
8547   case MVT::v8f64:
8548   case MVT::v8i64:
8549     if (Subtarget.hasAVX512())
8550       Opcode = X86ISD::VPERMV;
8551     break;
8552   }
8553   if (!Opcode)
8554     return SDValue();
8555 
8556   assert((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) &&
8557          (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 &&
8558          "Illegal variable permute shuffle type");
8559 
8560   uint64_t Scale = VT.getScalarSizeInBits() / ShuffleVT.getScalarSizeInBits();
8561   if (Scale > 1)
8562     IndicesVec = ScaleIndices(IndicesVec, Scale);
8563 
8564   EVT ShuffleIdxVT = EVT(ShuffleVT).changeVectorElementTypeToInteger();
8565   IndicesVec = DAG.getBitcast(ShuffleIdxVT, IndicesVec);
8566 
8567   SrcVec = DAG.getBitcast(ShuffleVT, SrcVec);
8568   SDValue Res = Opcode == X86ISD::VPERMV
8569                     ? DAG.getNode(Opcode, DL, ShuffleVT, IndicesVec, SrcVec)
8570                     : DAG.getNode(Opcode, DL, ShuffleVT, SrcVec, IndicesVec);
8571   return DAG.getBitcast(VT, Res);
8572 }
8573 
8574 // Tries to lower a BUILD_VECTOR composed of extract-extract chains that can be
8575 // reasoned to be a permutation of a vector by indices in a non-constant vector.
8576 // (build_vector (extract_elt V, (extract_elt I, 0)),
8577 //               (extract_elt V, (extract_elt I, 1)),
8578 //                    ...
8579 // ->
8580 // (vpermv I, V)
8581 //
8582 // TODO: Handle undefs
8583 // TODO: Utilize pshufb and zero mask blending to support more efficient
8584 // construction of vectors with constant-0 elements.
8585 static SDValue
8586 LowerBUILD_VECTORAsVariablePermute(SDValue V, SelectionDAG &DAG,
8587                                    const X86Subtarget &Subtarget) {
8588   SDValue SrcVec, IndicesVec;
8589   // Check for a match of the permute source vector and permute index elements.
8590   // This is done by checking that the i-th build_vector operand is of the form:
8591   // (extract_elt SrcVec, (extract_elt IndicesVec, i)).
8592   for (unsigned Idx = 0, E = V.getNumOperands(); Idx != E; ++Idx) {
8593     SDValue Op = V.getOperand(Idx);
8594     if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
8595       return SDValue();
8596 
8597     // If this is the first extract encountered in V, set the source vector,
8598     // otherwise verify the extract is from the previously defined source
8599     // vector.
8600     if (!SrcVec)
8601       SrcVec = Op.getOperand(0);
8602     else if (SrcVec != Op.getOperand(0))
8603       return SDValue();
8604     SDValue ExtractedIndex = Op->getOperand(1);
8605     // Peek through extends.
8606     if (ExtractedIndex.getOpcode() == ISD::ZERO_EXTEND ||
8607         ExtractedIndex.getOpcode() == ISD::SIGN_EXTEND)
8608       ExtractedIndex = ExtractedIndex.getOperand(0);
8609     if (ExtractedIndex.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
8610       return SDValue();
8611 
8612     // If this is the first extract from the index vector candidate, set the
8613     // indices vector, otherwise verify the extract is from the previously
8614     // defined indices vector.
8615     if (!IndicesVec)
8616       IndicesVec = ExtractedIndex.getOperand(0);
8617     else if (IndicesVec != ExtractedIndex.getOperand(0))
8618       return SDValue();
8619 
8620     auto *PermIdx = dyn_cast<ConstantSDNode>(ExtractedIndex.getOperand(1));
8621     if (!PermIdx || PermIdx->getAPIntValue() != Idx)
8622       return SDValue();
8623   }
8624 
8625   SDLoc DL(V);
8626   MVT VT = V.getSimpleValueType();
8627   return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
8628 }
8629 
8630 SDValue
8631 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
8632   SDLoc dl(Op);
8633 
8634   MVT VT = Op.getSimpleValueType();
8635   MVT EltVT = VT.getVectorElementType();
8636   MVT OpEltVT = Op.getOperand(0).getSimpleValueType();
8637   unsigned NumElems = Op.getNumOperands();
8638 
8639   // Generate vectors for predicate vectors.
8640   if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512())
8641     return LowerBUILD_VECTORvXi1(Op, DAG, Subtarget);
8642 
8643   if (VT.getVectorElementType() == MVT::bf16 &&
8644       (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16()))
8645     return LowerBUILD_VECTORvXbf16(Op, DAG, Subtarget);
8646 
8647   if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget))
8648     return VectorConstant;
8649 
8650   unsigned EVTBits = EltVT.getSizeInBits();
8651   APInt UndefMask = APInt::getZero(NumElems);
8652   APInt FrozenUndefMask = APInt::getZero(NumElems);
8653   APInt ZeroMask = APInt::getZero(NumElems);
8654   APInt NonZeroMask = APInt::getZero(NumElems);
8655   bool IsAllConstants = true;
8656   bool OneUseFrozenUndefs = true;
8657   SmallSet<SDValue, 8> Values;
8658   unsigned NumConstants = NumElems;
8659   for (unsigned i = 0; i < NumElems; ++i) {
8660     SDValue Elt = Op.getOperand(i);
8661     if (Elt.isUndef()) {
8662       UndefMask.setBit(i);
8663       continue;
8664     }
8665     if (ISD::isFreezeUndef(Elt.getNode())) {
8666       OneUseFrozenUndefs = OneUseFrozenUndefs && Elt->hasOneUse();
8667       FrozenUndefMask.setBit(i);
8668       continue;
8669     }
8670     Values.insert(Elt);
8671     if (!isIntOrFPConstant(Elt)) {
8672       IsAllConstants = false;
8673       NumConstants--;
8674     }
8675     if (X86::isZeroNode(Elt)) {
8676       ZeroMask.setBit(i);
8677     } else {
8678       NonZeroMask.setBit(i);
8679     }
8680   }
8681 
8682   // All undef vector. Return an UNDEF.
8683   if (UndefMask.isAllOnes())
8684     return DAG.getUNDEF(VT);
8685 
8686   // All undef/freeze(undef) vector. Return a FREEZE UNDEF.
8687   if (OneUseFrozenUndefs && (UndefMask | FrozenUndefMask).isAllOnes())
8688     return DAG.getFreeze(DAG.getUNDEF(VT));
8689 
8690   // All undef/freeze(undef)/zero vector. Return a zero vector.
8691   if ((UndefMask | FrozenUndefMask | ZeroMask).isAllOnes())
8692     return getZeroVector(VT, Subtarget, DAG, dl);
8693 
8694   // If we have multiple FREEZE-UNDEF operands, we are likely going to end up
8695   // lowering into a suboptimal insertion sequence. Instead, thaw the UNDEF in
8696   // our source BUILD_VECTOR, create another FREEZE-UNDEF splat BUILD_VECTOR,
8697   // and blend the FREEZE-UNDEF operands back in.
8698   // FIXME: is this worthwhile even for a single FREEZE-UNDEF operand?
8699   if (unsigned NumFrozenUndefElts = FrozenUndefMask.popcount();
8700       NumFrozenUndefElts >= 2 && NumFrozenUndefElts < NumElems) {
8701     SmallVector<int, 16> BlendMask(NumElems, -1);
8702     SmallVector<SDValue, 16> Elts(NumElems, DAG.getUNDEF(OpEltVT));
8703     for (unsigned i = 0; i < NumElems; ++i) {
8704       if (UndefMask[i]) {
8705         BlendMask[i] = -1;
8706         continue;
8707       }
8708       BlendMask[i] = i;
8709       if (!FrozenUndefMask[i])
8710         Elts[i] = Op.getOperand(i);
8711       else
8712         BlendMask[i] += NumElems;
8713     }
8714     SDValue EltsBV = DAG.getBuildVector(VT, dl, Elts);
8715     SDValue FrozenUndefElt = DAG.getFreeze(DAG.getUNDEF(OpEltVT));
8716     SDValue FrozenUndefBV = DAG.getSplatBuildVector(VT, dl, FrozenUndefElt);
8717     return DAG.getVectorShuffle(VT, dl, EltsBV, FrozenUndefBV, BlendMask);
8718   }
8719 
8720   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
8721 
8722   // If the upper elts of a ymm/zmm are undef/freeze(undef)/zero then we might
8723   // be better off lowering to a smaller build vector and padding with
8724   // undef/zero.
8725   if ((VT.is256BitVector() || VT.is512BitVector()) &&
8726       !isFoldableUseOfShuffle(BV)) {
8727     unsigned UpperElems = NumElems / 2;
8728     APInt UndefOrZeroMask = FrozenUndefMask | UndefMask | ZeroMask;
8729     unsigned NumUpperUndefsOrZeros = UndefOrZeroMask.countl_one();
8730     if (NumUpperUndefsOrZeros >= UpperElems) {
8731       if (VT.is512BitVector() &&
8732           NumUpperUndefsOrZeros >= (NumElems - (NumElems / 4)))
8733         UpperElems = NumElems - (NumElems / 4);
8734       // If freeze(undef) is in any upper elements, force to zero.
8735       bool UndefUpper = UndefMask.countl_one() >= UpperElems;
8736       MVT LowerVT = MVT::getVectorVT(EltVT, NumElems - UpperElems);
8737       SDValue NewBV =
8738           DAG.getBuildVector(LowerVT, dl, Op->ops().drop_back(UpperElems));
8739       return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl);
8740     }
8741   }
8742 
8743   if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG))
8744     return AddSub;
8745   if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
8746     return HorizontalOp;
8747   if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG))
8748     return Broadcast;
8749   if (SDValue BitOp = lowerBuildVectorToBitOp(BV, Subtarget, DAG))
8750     return BitOp;
8751 
8752   unsigned NumZero = ZeroMask.popcount();
8753   unsigned NumNonZero = NonZeroMask.popcount();
8754 
8755   // If we are inserting one variable into a vector of non-zero constants, try
8756   // to avoid loading each constant element as a scalar. Load the constants as a
8757   // vector and then insert the variable scalar element. If insertion is not
8758   // supported, fall back to a shuffle to get the scalar blended with the
8759   // constants. Insertion into a zero vector is handled as a special-case
8760   // somewhere below here.
8761   if (NumConstants == NumElems - 1 && NumNonZero != 1 &&
8762       FrozenUndefMask.isZero() &&
8763       (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) ||
8764        isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) {
8765     // Create an all-constant vector. The variable element in the old
8766     // build vector is replaced by undef in the constant vector. Save the
8767     // variable scalar element and its index for use in the insertelement.
8768     LLVMContext &Context = *DAG.getContext();
8769     Type *EltType = Op.getValueType().getScalarType().getTypeForEVT(Context);
8770     SmallVector<Constant *, 16> ConstVecOps(NumElems, UndefValue::get(EltType));
8771     SDValue VarElt;
8772     SDValue InsIndex;
8773     for (unsigned i = 0; i != NumElems; ++i) {
8774       SDValue Elt = Op.getOperand(i);
8775       if (auto *C = dyn_cast<ConstantSDNode>(Elt))
8776         ConstVecOps[i] = ConstantInt::get(Context, C->getAPIntValue());
8777       else if (auto *C = dyn_cast<ConstantFPSDNode>(Elt))
8778         ConstVecOps[i] = ConstantFP::get(Context, C->getValueAPF());
8779       else if (!Elt.isUndef()) {
8780         assert(!VarElt.getNode() && !InsIndex.getNode() &&
8781                "Expected one variable element in this vector");
8782         VarElt = Elt;
8783         InsIndex = DAG.getVectorIdxConstant(i, dl);
8784       }
8785     }
8786     Constant *CV = ConstantVector::get(ConstVecOps);
8787     SDValue DAGConstVec = DAG.getConstantPool(CV, VT);
8788 
8789     // The constants we just created may not be legal (eg, floating point). We
8790     // must lower the vector right here because we can not guarantee that we'll
8791     // legalize it before loading it. This is also why we could not just create
8792     // a new build vector here. If the build vector contains illegal constants,
8793     // it could get split back up into a series of insert elements.
8794     // TODO: Improve this by using shorter loads with broadcast/VZEXT_LOAD.
8795     SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG);
8796     MachineFunction &MF = DAG.getMachineFunction();
8797     MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF);
8798     SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);
8799     unsigned InsertC = InsIndex->getAsZExtVal();
8800     unsigned NumEltsInLow128Bits = 128 / VT.getScalarSizeInBits();
8801     if (InsertC < NumEltsInLow128Bits)
8802       return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex);
8803 
8804     // There's no good way to insert into the high elements of a >128-bit
8805     // vector, so use shuffles to avoid an extract/insert sequence.
8806     assert(VT.getSizeInBits() > 128 && "Invalid insertion index?");
8807     assert(Subtarget.hasAVX() && "Must have AVX with >16-byte vector");
8808     SmallVector<int, 8> ShuffleMask;
8809     unsigned NumElts = VT.getVectorNumElements();
8810     for (unsigned i = 0; i != NumElts; ++i)
8811       ShuffleMask.push_back(i == InsertC ? NumElts : i);
8812     SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);
8813     return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask);
8814   }
8815 
8816   // Special case for single non-zero, non-undef, element.
8817   if (NumNonZero == 1) {
8818     unsigned Idx = NonZeroMask.countr_zero();
8819     SDValue Item = Op.getOperand(Idx);
8820 
8821     // If we have a constant or non-constant insertion into the low element of
8822     // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into
8823     // the rest of the elements.  This will be matched as movd/movq/movss/movsd
8824     // depending on what the source datatype is.
8825     if (Idx == 0) {
8826       if (NumZero == 0)
8827         return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
8828 
8829       if (EltVT == MVT::i32 || EltVT == MVT::f16 || EltVT == MVT::f32 ||
8830           EltVT == MVT::f64 || (EltVT == MVT::i64 && Subtarget.is64Bit()) ||
8831           (EltVT == MVT::i16 && Subtarget.hasFP16())) {
8832         assert((VT.is128BitVector() || VT.is256BitVector() ||
8833                 VT.is512BitVector()) &&
8834                "Expected an SSE value type!");
8835         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
8836         // Turn it into a MOVL (i.e. movsh, movss, movsd, movw or movd) to a
8837         // zero vector.
8838         return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
8839       }
8840 
8841       // We can't directly insert an i8 or i16 into a vector, so zero extend
8842       // it to i32 first.
8843       if (EltVT == MVT::i16 || EltVT == MVT::i8) {
8844         Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
8845         MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
8846         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, Item);
8847         Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
8848         return DAG.getBitcast(VT, Item);
8849       }
8850     }
8851 
8852     // Is it a vector logical left shift?
8853     if (NumElems == 2 && Idx == 1 &&
8854         X86::isZeroNode(Op.getOperand(0)) &&
8855         !X86::isZeroNode(Op.getOperand(1))) {
8856       unsigned NumBits = VT.getSizeInBits();
8857       return getVShift(true, VT,
8858                        DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
8859                                    VT, Op.getOperand(1)),
8860                        NumBits/2, DAG, *this, dl);
8861     }
8862 
8863     if (IsAllConstants) // Otherwise, it's better to do a constpool load.
8864       return SDValue();
8865 
8866     // Otherwise, if this is a vector with i32 or f32 elements, and the element
8867     // is a non-constant being inserted into an element other than the low one,
8868     // we can't use a constant pool load.  Instead, use SCALAR_TO_VECTOR (aka
8869     // movd/movss) to move this into the low element, then shuffle it into
8870     // place.
8871     if (EVTBits == 32) {
8872       Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
8873       return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
8874     }
8875   }
8876 
8877   // Splat is obviously ok. Let legalizer expand it to a shuffle.
8878   if (Values.size() == 1) {
8879     if (EVTBits == 32) {
8880       // Instead of a shuffle like this:
8881       // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0>
8882       // Check if it's possible to issue this instead.
8883       // shuffle (vload ptr)), undef, <1, 1, 1, 1>
8884       unsigned Idx = NonZeroMask.countr_zero();
8885       SDValue Item = Op.getOperand(Idx);
8886       if (Op.getNode()->isOnlyUserOf(Item.getNode()))
8887         return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
8888     }
8889     return SDValue();
8890   }
8891 
8892   // A vector full of immediates; various special cases are already
8893   // handled, so this is best done with a single constant-pool load.
8894   if (IsAllConstants)
8895     return SDValue();
8896 
8897   if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, DAG, Subtarget))
8898       return V;
8899 
8900   // See if we can use a vector load to get all of the elements.
8901   {
8902     SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
8903     if (SDValue LD =
8904             EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false))
8905       return LD;
8906   }
8907 
8908   // If this is a splat of pairs of 32-bit elements, we can use a narrower
8909   // build_vector and broadcast it.
8910   // TODO: We could probably generalize this more.
8911   if (Subtarget.hasAVX2() && EVTBits == 32 && Values.size() == 2) {
8912     SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
8913                        DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
8914     auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
8915       // Make sure all the even/odd operands match.
8916       for (unsigned i = 2; i != NumElems; ++i)
8917         if (Ops[i % 2] != Op.getOperand(i))
8918           return false;
8919       return true;
8920     };
8921     if (CanSplat(Op, NumElems, Ops)) {
8922       MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64;
8923       MVT NarrowVT = MVT::getVectorVT(EltVT, 4);
8924       // Create a new build vector and cast to v2i64/v2f64.
8925       SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
8926                                      DAG.getBuildVector(NarrowVT, dl, Ops));
8927       // Broadcast from v2i64/v2f64 and cast to final VT.
8928       MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems / 2);
8929       return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
8930                                             NewBV));
8931     }
8932   }
8933 
8934   // For AVX-length vectors, build the individual 128-bit pieces and use
8935   // shuffles to put them in place.
8936   if (VT.getSizeInBits() > 128) {
8937     MVT HVT = MVT::getVectorVT(EltVT, NumElems / 2);
8938 
8939     // Build both the lower and upper subvector.
8940     SDValue Lower =
8941         DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2));
8942     SDValue Upper = DAG.getBuildVector(
8943         HVT, dl, Op->ops().slice(NumElems / 2, NumElems /2));
8944 
8945     // Recreate the wider vector with the lower and upper part.
8946     return concatSubVectors(Lower, Upper, DAG, dl);
8947   }
8948 
8949   // Let legalizer expand 2-wide build_vectors.
8950   if (EVTBits == 64) {
8951     if (NumNonZero == 1) {
8952       // One half is zero or undef.
8953       unsigned Idx = NonZeroMask.countr_zero();
8954       SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
8955                                Op.getOperand(Idx));
8956       return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
8957     }
8958     return SDValue();
8959   }
8960 
8961   // If element VT is < 32 bits, convert it to inserts into a zero vector.
8962   if (EVTBits == 8 && NumElems == 16)
8963     if (SDValue V = LowerBuildVectorv16i8(Op, NonZeroMask, NumNonZero, NumZero,
8964                                           DAG, Subtarget))
8965       return V;
8966 
8967   if (EltVT == MVT::i16 && NumElems == 8)
8968     if (SDValue V = LowerBuildVectorv8i16(Op, NonZeroMask, NumNonZero, NumZero,
8969                                           DAG, Subtarget))
8970       return V;
8971 
8972   // If element VT is == 32 bits and has 4 elems, try to generate an INSERTPS
8973   if (EVTBits == 32 && NumElems == 4)
8974     if (SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget))
8975       return V;
8976 
8977   // If element VT is == 32 bits, turn it into a number of shuffles.
8978   if (NumElems == 4 && NumZero > 0) {
8979     SmallVector<SDValue, 8> Ops(NumElems);
8980     for (unsigned i = 0; i < 4; ++i) {
8981       bool isZero = !NonZeroMask[i];
8982       if (isZero)
8983         Ops[i] = getZeroVector(VT, Subtarget, DAG, dl);
8984       else
8985         Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
8986     }
8987 
8988     for (unsigned i = 0; i < 2; ++i) {
8989       switch (NonZeroMask.extractBitsAsZExtValue(2, i * 2)) {
8990         default: llvm_unreachable("Unexpected NonZero count");
8991         case 0:
8992           Ops[i] = Ops[i*2];  // Must be a zero vector.
8993           break;
8994         case 1:
8995           Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]);
8996           break;
8997         case 2:
8998           Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
8999           break;
9000         case 3:
9001           Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
9002           break;
9003       }
9004     }
9005 
9006     bool Reverse1 = NonZeroMask.extractBitsAsZExtValue(2, 0) == 2;
9007     bool Reverse2 = NonZeroMask.extractBitsAsZExtValue(2, 2) == 2;
9008     int MaskVec[] = {
9009       Reverse1 ? 1 : 0,
9010       Reverse1 ? 0 : 1,
9011       static_cast<int>(Reverse2 ? NumElems+1 : NumElems),
9012       static_cast<int>(Reverse2 ? NumElems   : NumElems+1)
9013     };
9014     return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec);
9015   }
9016 
9017   assert(Values.size() > 1 && "Expected non-undef and non-splat vector");
9018 
9019   // Check for a build vector from mostly shuffle plus few inserting.
9020   if (SDValue Sh = buildFromShuffleMostly(Op, DAG))
9021     return Sh;
9022 
9023   // For SSE 4.1, use insertps to put the high elements into the low element.
9024   if (Subtarget.hasSSE41() && EltVT != MVT::f16) {
9025     SDValue Result;
9026     if (!Op.getOperand(0).isUndef())
9027       Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
9028     else
9029       Result = DAG.getUNDEF(VT);
9030 
9031     for (unsigned i = 1; i < NumElems; ++i) {
9032       if (Op.getOperand(i).isUndef()) continue;
9033       Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
9034                            Op.getOperand(i), DAG.getIntPtrConstant(i, dl));
9035     }
9036     return Result;
9037   }
9038 
9039   // Otherwise, expand into a number of unpckl*, start by extending each of
9040   // our (non-undef) elements to the full vector width with the element in the
9041   // bottom slot of the vector (which generates no code for SSE).
9042   SmallVector<SDValue, 8> Ops(NumElems);
9043   for (unsigned i = 0; i < NumElems; ++i) {
9044     if (!Op.getOperand(i).isUndef())
9045       Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
9046     else
9047       Ops[i] = DAG.getUNDEF(VT);
9048   }
9049 
9050   // Next, we iteratively mix elements, e.g. for v4f32:
9051   //   Step 1: unpcklps 0, 1 ==> X: <?, ?, 1, 0>
9052   //         : unpcklps 2, 3 ==> Y: <?, ?, 3, 2>
9053   //   Step 2: unpcklpd X, Y ==>    <3, 2, 1, 0>
9054   for (unsigned Scale = 1; Scale < NumElems; Scale *= 2) {
9055     // Generate scaled UNPCKL shuffle mask.
9056     SmallVector<int, 16> Mask;
9057     for(unsigned i = 0; i != Scale; ++i)
9058       Mask.push_back(i);
9059     for (unsigned i = 0; i != Scale; ++i)
9060       Mask.push_back(NumElems+i);
9061     Mask.append(NumElems - Mask.size(), SM_SentinelUndef);
9062 
9063     for (unsigned i = 0, e = NumElems / (2 * Scale); i != e; ++i)
9064       Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask);
9065   }
9066   return Ops[0];
9067 }
9068 
9069 // 256-bit AVX can use the vinsertf128 instruction
9070 // to create 256-bit vectors from two other 128-bit ones.
9071 // TODO: Detect subvector broadcast here instead of DAG combine?
9072 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9073                                       const X86Subtarget &Subtarget) {
9074   SDLoc dl(Op);
9075   MVT ResVT = Op.getSimpleValueType();
9076 
9077   assert((ResVT.is256BitVector() ||
9078           ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide");
9079 
9080   unsigned NumOperands = Op.getNumOperands();
9081   unsigned NumFreezeUndef = 0;
9082   unsigned NumZero = 0;
9083   unsigned NumNonZero = 0;
9084   unsigned NonZeros = 0;
9085   for (unsigned i = 0; i != NumOperands; ++i) {
9086     SDValue SubVec = Op.getOperand(i);
9087     if (SubVec.isUndef())
9088       continue;
9089     if (ISD::isFreezeUndef(SubVec.getNode())) {
9090         // If the freeze(undef) has multiple uses then we must fold to zero.
9091         if (SubVec.hasOneUse())
9092           ++NumFreezeUndef;
9093         else
9094           ++NumZero;
9095     }
9096     else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
9097       ++NumZero;
9098     else {
9099       assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
9100       NonZeros |= 1 << i;
9101       ++NumNonZero;
9102     }
9103   }
9104 
9105   // If we have more than 2 non-zeros, build each half separately.
9106   if (NumNonZero > 2) {
9107     MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
9108     ArrayRef<SDUse> Ops = Op->ops();
9109     SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9110                              Ops.slice(0, NumOperands/2));
9111     SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9112                              Ops.slice(NumOperands/2));
9113     return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
9114   }
9115 
9116   // Otherwise, build it up through insert_subvectors.
9117   SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
9118                         : (NumFreezeUndef ? DAG.getFreeze(DAG.getUNDEF(ResVT))
9119                                           : DAG.getUNDEF(ResVT));
9120 
9121   MVT SubVT = Op.getOperand(0).getSimpleValueType();
9122   unsigned NumSubElems = SubVT.getVectorNumElements();
9123   for (unsigned i = 0; i != NumOperands; ++i) {
9124     if ((NonZeros & (1 << i)) == 0)
9125       continue;
9126 
9127     Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec,
9128                       Op.getOperand(i),
9129                       DAG.getIntPtrConstant(i * NumSubElems, dl));
9130   }
9131 
9132   return Vec;
9133 }
9134 
9135 // Returns true if the given node is a type promotion (by concatenating i1
9136 // zeros) of the result of a node that already zeros all upper bits of
9137 // k-register.
9138 // TODO: Merge this with LowerAVXCONCAT_VECTORS?
9139 static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
9140                                        const X86Subtarget &Subtarget,
9141                                        SelectionDAG & DAG) {
9142   SDLoc dl(Op);
9143   MVT ResVT = Op.getSimpleValueType();
9144   unsigned NumOperands = Op.getNumOperands();
9145 
9146   assert(NumOperands > 1 && isPowerOf2_32(NumOperands) &&
9147          "Unexpected number of operands in CONCAT_VECTORS");
9148 
9149   uint64_t Zeros = 0;
9150   uint64_t NonZeros = 0;
9151   for (unsigned i = 0; i != NumOperands; ++i) {
9152     SDValue SubVec = Op.getOperand(i);
9153     if (SubVec.isUndef())
9154       continue;
9155     assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
9156     if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
9157       Zeros |= (uint64_t)1 << i;
9158     else
9159       NonZeros |= (uint64_t)1 << i;
9160   }
9161 
9162   unsigned NumElems = ResVT.getVectorNumElements();
9163 
9164   // If we are inserting non-zero vector and there are zeros in LSBs and undef
9165   // in the MSBs we need to emit a KSHIFTL. The generic lowering to
9166   // insert_subvector will give us two kshifts.
9167   if (isPowerOf2_64(NonZeros) && Zeros != 0 && NonZeros > Zeros &&
9168       Log2_64(NonZeros) != NumOperands - 1) {
9169     unsigned Idx = Log2_64(NonZeros);
9170     SDValue SubVec = Op.getOperand(Idx);
9171     unsigned SubVecNumElts = SubVec.getSimpleValueType().getVectorNumElements();
9172     MVT ShiftVT = widenMaskVectorType(ResVT, Subtarget);
9173     Op = widenSubVector(ShiftVT, SubVec, false, Subtarget, DAG, dl);
9174     Op = DAG.getNode(X86ISD::KSHIFTL, dl, ShiftVT, Op,
9175                      DAG.getTargetConstant(Idx * SubVecNumElts, dl, MVT::i8));
9176     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, Op,
9177                        DAG.getIntPtrConstant(0, dl));
9178   }
9179 
9180   // If there are zero or one non-zeros we can handle this very simply.
9181   if (NonZeros == 0 || isPowerOf2_64(NonZeros)) {
9182     SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT);
9183     if (!NonZeros)
9184       return Vec;
9185     unsigned Idx = Log2_64(NonZeros);
9186     SDValue SubVec = Op.getOperand(Idx);
9187     unsigned SubVecNumElts = SubVec.getSimpleValueType().getVectorNumElements();
9188     return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, SubVec,
9189                        DAG.getIntPtrConstant(Idx * SubVecNumElts, dl));
9190   }
9191 
9192   if (NumOperands > 2) {
9193     MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
9194     ArrayRef<SDUse> Ops = Op->ops();
9195     SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9196                              Ops.slice(0, NumOperands/2));
9197     SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9198                              Ops.slice(NumOperands/2));
9199     return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
9200   }
9201 
9202   assert(llvm::popcount(NonZeros) == 2 && "Simple cases not handled?");
9203 
9204   if (ResVT.getVectorNumElements() >= 16)
9205     return Op; // The operation is legal with KUNPCK
9206 
9207   SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT,
9208                             DAG.getUNDEF(ResVT), Op.getOperand(0),
9209                             DAG.getIntPtrConstant(0, dl));
9210   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Op.getOperand(1),
9211                      DAG.getIntPtrConstant(NumElems/2, dl));
9212 }
9213 
9214 static SDValue LowerCONCAT_VECTORS(SDValue Op,
9215                                    const X86Subtarget &Subtarget,
9216                                    SelectionDAG &DAG) {
9217   MVT VT = Op.getSimpleValueType();
9218   if (VT.getVectorElementType() == MVT::i1)
9219     return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG);
9220 
9221   assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
9222          (VT.is512BitVector() && (Op.getNumOperands() == 2 ||
9223           Op.getNumOperands() == 4)));
9224 
9225   // AVX can use the vinsertf128 instruction to create 256-bit vectors
9226   // from two other 128-bit ones.
9227 
9228   // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors
9229   return LowerAVXCONCAT_VECTORS(Op, DAG, Subtarget);
9230 }
9231 
9232 //===----------------------------------------------------------------------===//
9233 // Vector shuffle lowering
9234 //
9235 // This is an experimental code path for lowering vector shuffles on x86. It is
9236 // designed to handle arbitrary vector shuffles and blends, gracefully
9237 // degrading performance as necessary. It works hard to recognize idiomatic
9238 // shuffles and lower them to optimal instruction patterns without leaving
9239 // a framework that allows reasonably efficient handling of all vector shuffle
9240 // patterns.
9241 //===----------------------------------------------------------------------===//
9242 
9243 /// Tiny helper function to identify a no-op mask.
9244 ///
9245 /// This is a somewhat boring predicate function. It checks whether the mask
9246 /// array input, which is assumed to be a single-input shuffle mask of the kind
9247 /// used by the X86 shuffle instructions (not a fully general
9248 /// ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an
9249 /// in-place shuffle are 'no-op's.
9250 static bool isNoopShuffleMask(ArrayRef<int> Mask) {
9251   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
9252     assert(Mask[i] >= -1 && "Out of bound mask element!");
9253     if (Mask[i] >= 0 && Mask[i] != i)
9254       return false;
9255   }
9256   return true;
9257 }
9258 
9259 /// Test whether there are elements crossing LaneSizeInBits lanes in this
9260 /// shuffle mask.
9261 ///
9262 /// X86 divides up its shuffles into in-lane and cross-lane shuffle operations
9263 /// and we routinely test for these.
9264 static bool isLaneCrossingShuffleMask(unsigned LaneSizeInBits,
9265                                       unsigned ScalarSizeInBits,
9266                                       ArrayRef<int> Mask) {
9267   assert(LaneSizeInBits && ScalarSizeInBits &&
9268          (LaneSizeInBits % ScalarSizeInBits) == 0 &&
9269          "Illegal shuffle lane size");
9270   int LaneSize = LaneSizeInBits / ScalarSizeInBits;
9271   int Size = Mask.size();
9272   for (int i = 0; i < Size; ++i)
9273     if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
9274       return true;
9275   return false;
9276 }
9277 
9278 /// Test whether there are elements crossing 128-bit lanes in this
9279 /// shuffle mask.
9280 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) {
9281   return isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), Mask);
9282 }
9283 
9284 /// Test whether elements in each LaneSizeInBits lane in this shuffle mask come
9285 /// from multiple lanes - this is different to isLaneCrossingShuffleMask to
9286 /// better support 'repeated mask + lane permute' style shuffles.
9287 static bool isMultiLaneShuffleMask(unsigned LaneSizeInBits,
9288                                    unsigned ScalarSizeInBits,
9289                                    ArrayRef<int> Mask) {
9290   assert(LaneSizeInBits && ScalarSizeInBits &&
9291          (LaneSizeInBits % ScalarSizeInBits) == 0 &&
9292          "Illegal shuffle lane size");
9293   int NumElts = Mask.size();
9294   int NumEltsPerLane = LaneSizeInBits / ScalarSizeInBits;
9295   int NumLanes = NumElts / NumEltsPerLane;
9296   if (NumLanes > 1) {
9297     for (int i = 0; i != NumLanes; ++i) {
9298       int SrcLane = -1;
9299       for (int j = 0; j != NumEltsPerLane; ++j) {
9300         int M = Mask[(i * NumEltsPerLane) + j];
9301         if (M < 0)
9302           continue;
9303         int Lane = (M % NumElts) / NumEltsPerLane;
9304         if (SrcLane >= 0 && SrcLane != Lane)
9305           return true;
9306         SrcLane = Lane;
9307       }
9308     }
9309   }
9310   return false;
9311 }
9312 
9313 /// Test whether a shuffle mask is equivalent within each sub-lane.
9314 ///
9315 /// This checks a shuffle mask to see if it is performing the same
9316 /// lane-relative shuffle in each sub-lane. This trivially implies
9317 /// that it is also not lane-crossing. It may however involve a blend from the
9318 /// same lane of a second vector.
9319 ///
9320 /// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
9321 /// non-trivial to compute in the face of undef lanes. The representation is
9322 /// suitable for use with existing 128-bit shuffles as entries from the second
9323 /// vector have been remapped to [LaneSize, 2*LaneSize).
9324 static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
9325                                   ArrayRef<int> Mask,
9326                                   SmallVectorImpl<int> &RepeatedMask) {
9327   auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
9328   RepeatedMask.assign(LaneSize, -1);
9329   int Size = Mask.size();
9330   for (int i = 0; i < Size; ++i) {
9331     assert(Mask[i] == SM_SentinelUndef || Mask[i] >= 0);
9332     if (Mask[i] < 0)
9333       continue;
9334     if ((Mask[i] % Size) / LaneSize != i / LaneSize)
9335       // This entry crosses lanes, so there is no way to model this shuffle.
9336       return false;
9337 
9338     // Ok, handle the in-lane shuffles by detecting if and when they repeat.
9339     // Adjust second vector indices to start at LaneSize instead of Size.
9340     int LocalM = Mask[i] < Size ? Mask[i] % LaneSize
9341                                 : Mask[i] % LaneSize + LaneSize;
9342     if (RepeatedMask[i % LaneSize] < 0)
9343       // This is the first non-undef entry in this slot of a 128-bit lane.
9344       RepeatedMask[i % LaneSize] = LocalM;
9345     else if (RepeatedMask[i % LaneSize] != LocalM)
9346       // Found a mismatch with the repeated mask.
9347       return false;
9348   }
9349   return true;
9350 }
9351 
9352 /// Test whether a shuffle mask is equivalent within each 128-bit lane.
9353 static bool
9354 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
9355                                 SmallVectorImpl<int> &RepeatedMask) {
9356   return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
9357 }
9358 
9359 static bool
9360 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {
9361   SmallVector<int, 32> RepeatedMask;
9362   return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
9363 }
9364 
9365 /// Test whether a shuffle mask is equivalent within each 256-bit lane.
9366 static bool
9367 is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
9368                                 SmallVectorImpl<int> &RepeatedMask) {
9369   return isRepeatedShuffleMask(256, VT, Mask, RepeatedMask);
9370 }
9371 
9372 /// Test whether a target shuffle mask is equivalent within each sub-lane.
9373 /// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
9374 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits,
9375                                         unsigned EltSizeInBits,
9376                                         ArrayRef<int> Mask,
9377                                         SmallVectorImpl<int> &RepeatedMask) {
9378   int LaneSize = LaneSizeInBits / EltSizeInBits;
9379   RepeatedMask.assign(LaneSize, SM_SentinelUndef);
9380   int Size = Mask.size();
9381   for (int i = 0; i < Size; ++i) {
9382     assert(isUndefOrZero(Mask[i]) || (Mask[i] >= 0));
9383     if (Mask[i] == SM_SentinelUndef)
9384       continue;
9385     if (Mask[i] == SM_SentinelZero) {
9386       if (!isUndefOrZero(RepeatedMask[i % LaneSize]))
9387         return false;
9388       RepeatedMask[i % LaneSize] = SM_SentinelZero;
9389       continue;
9390     }
9391     if ((Mask[i] % Size) / LaneSize != i / LaneSize)
9392       // This entry crosses lanes, so there is no way to model this shuffle.
9393       return false;
9394 
9395     // Handle the in-lane shuffles by detecting if and when they repeat. Adjust
9396     // later vector indices to start at multiples of LaneSize instead of Size.
9397     int LaneM = Mask[i] / Size;
9398     int LocalM = (Mask[i] % LaneSize) + (LaneM * LaneSize);
9399     if (RepeatedMask[i % LaneSize] == SM_SentinelUndef)
9400       // This is the first non-undef entry in this slot of a 128-bit lane.
9401       RepeatedMask[i % LaneSize] = LocalM;
9402     else if (RepeatedMask[i % LaneSize] != LocalM)
9403       // Found a mismatch with the repeated mask.
9404       return false;
9405   }
9406   return true;
9407 }
9408 
9409 /// Test whether a target shuffle mask is equivalent within each sub-lane.
9410 /// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
9411 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT,
9412                                         ArrayRef<int> Mask,
9413                                         SmallVectorImpl<int> &RepeatedMask) {
9414   return isRepeatedTargetShuffleMask(LaneSizeInBits, VT.getScalarSizeInBits(),
9415                                      Mask, RepeatedMask);
9416 }
9417 
9418 /// Checks whether the vector elements referenced by two shuffle masks are
9419 /// equivalent.
9420 static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
9421                                 int Idx, int ExpectedIdx) {
9422   assert(0 <= Idx && Idx < MaskSize && 0 <= ExpectedIdx &&
9423          ExpectedIdx < MaskSize && "Out of range element index");
9424   if (!Op || !ExpectedOp || Op.getOpcode() != ExpectedOp.getOpcode())
9425     return false;
9426 
9427   switch (Op.getOpcode()) {
9428   case ISD::BUILD_VECTOR:
9429     // If the values are build vectors, we can look through them to find
9430     // equivalent inputs that make the shuffles equivalent.
9431     // TODO: Handle MaskSize != Op.getNumOperands()?
9432     if (MaskSize == (int)Op.getNumOperands() &&
9433         MaskSize == (int)ExpectedOp.getNumOperands())
9434       return Op.getOperand(Idx) == ExpectedOp.getOperand(ExpectedIdx);
9435     break;
9436   case X86ISD::VBROADCAST:
9437   case X86ISD::VBROADCAST_LOAD:
9438     // TODO: Handle MaskSize != Op.getValueType().getVectorNumElements()?
9439     return (Op == ExpectedOp &&
9440             (int)Op.getValueType().getVectorNumElements() == MaskSize);
9441   case X86ISD::HADD:
9442   case X86ISD::HSUB:
9443   case X86ISD::FHADD:
9444   case X86ISD::FHSUB:
9445   case X86ISD::PACKSS:
9446   case X86ISD::PACKUS:
9447     // HOP(X,X) can refer to the elt from the lower/upper half of a lane.
9448     // TODO: Handle MaskSize != NumElts?
9449     // TODO: Handle HOP(X,Y) vs HOP(Y,X) equivalence cases.
9450     if (Op == ExpectedOp && Op.getOperand(0) == Op.getOperand(1)) {
9451       MVT VT = Op.getSimpleValueType();
9452       int NumElts = VT.getVectorNumElements();
9453       if (MaskSize == NumElts) {
9454         int NumLanes = VT.getSizeInBits() / 128;
9455         int NumEltsPerLane = NumElts / NumLanes;
9456         int NumHalfEltsPerLane = NumEltsPerLane / 2;
9457         bool SameLane =
9458             (Idx / NumEltsPerLane) == (ExpectedIdx / NumEltsPerLane);
9459         bool SameElt =
9460             (Idx % NumHalfEltsPerLane) == (ExpectedIdx % NumHalfEltsPerLane);
9461         return SameLane && SameElt;
9462       }
9463     }
9464     break;
9465   }
9466 
9467   return false;
9468 }
9469 
9470 /// Checks whether a shuffle mask is equivalent to an explicit list of
9471 /// arguments.
9472 ///
9473 /// This is a fast way to test a shuffle mask against a fixed pattern:
9474 ///
9475 ///   if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }
9476 ///
9477 /// It returns true if the mask is exactly as wide as the argument list, and
9478 /// each element of the mask is either -1 (signifying undef) or the value given
9479 /// in the argument.
9480 static bool isShuffleEquivalent(ArrayRef<int> Mask, ArrayRef<int> ExpectedMask,
9481                                 SDValue V1 = SDValue(),
9482                                 SDValue V2 = SDValue()) {
9483   int Size = Mask.size();
9484   if (Size != (int)ExpectedMask.size())
9485     return false;
9486 
9487   for (int i = 0; i < Size; ++i) {
9488     assert(Mask[i] >= -1 && "Out of bound mask element!");
9489     int MaskIdx = Mask[i];
9490     int ExpectedIdx = ExpectedMask[i];
9491     if (0 <= MaskIdx && MaskIdx != ExpectedIdx) {
9492       SDValue MaskV = MaskIdx < Size ? V1 : V2;
9493       SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9494       MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size);
9495       ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9496       if (!IsElementEquivalent(Size, MaskV, ExpectedV, MaskIdx, ExpectedIdx))
9497         return false;
9498     }
9499   }
9500   return true;
9501 }
9502 
9503 /// Checks whether a target shuffle mask is equivalent to an explicit pattern.
9504 ///
9505 /// The masks must be exactly the same width.
9506 ///
9507 /// If an element in Mask matches SM_SentinelUndef (-1) then the corresponding
9508 /// value in ExpectedMask is always accepted. Otherwise the indices must match.
9509 ///
9510 /// SM_SentinelZero is accepted as a valid negative index but must match in
9511 /// both, or via a known bits test.
9512 static bool isTargetShuffleEquivalent(MVT VT, ArrayRef<int> Mask,
9513                                       ArrayRef<int> ExpectedMask,
9514                                       const SelectionDAG &DAG,
9515                                       SDValue V1 = SDValue(),
9516                                       SDValue V2 = SDValue()) {
9517   int Size = Mask.size();
9518   if (Size != (int)ExpectedMask.size())
9519     return false;
9520   assert(llvm::all_of(ExpectedMask,
9521                       [Size](int M) { return isInRange(M, 0, 2 * Size); }) &&
9522          "Illegal target shuffle mask");
9523 
9524   // Check for out-of-range target shuffle mask indices.
9525   if (!isUndefOrZeroOrInRange(Mask, 0, 2 * Size))
9526     return false;
9527 
9528   // Don't use V1/V2 if they're not the same size as the shuffle mask type.
9529   if (V1 && (V1.getValueSizeInBits() != VT.getSizeInBits() ||
9530              !V1.getValueType().isVector()))
9531     V1 = SDValue();
9532   if (V2 && (V2.getValueSizeInBits() != VT.getSizeInBits() ||
9533              !V2.getValueType().isVector()))
9534     V2 = SDValue();
9535 
9536   APInt ZeroV1 = APInt::getZero(Size);
9537   APInt ZeroV2 = APInt::getZero(Size);
9538 
9539   for (int i = 0; i < Size; ++i) {
9540     int MaskIdx = Mask[i];
9541     int ExpectedIdx = ExpectedMask[i];
9542     if (MaskIdx == SM_SentinelUndef || MaskIdx == ExpectedIdx)
9543       continue;
9544     if (MaskIdx == SM_SentinelZero) {
9545       // If we need this expected index to be a zero element, then update the
9546       // relevant zero mask and perform the known bits at the end to minimize
9547       // repeated computes.
9548       SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9549       if (ExpectedV &&
9550           Size == (int)ExpectedV.getValueType().getVectorNumElements()) {
9551         int BitIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9552         APInt &ZeroMask = ExpectedIdx < Size ? ZeroV1 : ZeroV2;
9553         ZeroMask.setBit(BitIdx);
9554         continue;
9555       }
9556     }
9557     if (MaskIdx >= 0) {
9558       SDValue MaskV = MaskIdx < Size ? V1 : V2;
9559       SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9560       MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size);
9561       ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9562       if (IsElementEquivalent(Size, MaskV, ExpectedV, MaskIdx, ExpectedIdx))
9563         continue;
9564     }
9565     return false;
9566   }
9567   return (ZeroV1.isZero() || DAG.MaskedVectorIsZero(V1, ZeroV1)) &&
9568          (ZeroV2.isZero() || DAG.MaskedVectorIsZero(V2, ZeroV2));
9569 }
9570 
9571 // Check if the shuffle mask is suitable for the AVX vpunpcklwd or vpunpckhwd
9572 // instructions.
9573 static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT,
9574                                   const SelectionDAG &DAG) {
9575   if (VT != MVT::v8i32 && VT != MVT::v8f32)
9576     return false;
9577 
9578   SmallVector<int, 8> Unpcklwd;
9579   createUnpackShuffleMask(MVT::v8i16, Unpcklwd, /* Lo = */ true,
9580                           /* Unary = */ false);
9581   SmallVector<int, 8> Unpckhwd;
9582   createUnpackShuffleMask(MVT::v8i16, Unpckhwd, /* Lo = */ false,
9583                           /* Unary = */ false);
9584   bool IsUnpackwdMask = (isTargetShuffleEquivalent(VT, Mask, Unpcklwd, DAG) ||
9585                          isTargetShuffleEquivalent(VT, Mask, Unpckhwd, DAG));
9586   return IsUnpackwdMask;
9587 }
9588 
9589 static bool is128BitUnpackShuffleMask(ArrayRef<int> Mask,
9590                                       const SelectionDAG &DAG) {
9591   // Create 128-bit vector type based on mask size.
9592   MVT EltVT = MVT::getIntegerVT(128 / Mask.size());
9593   MVT VT = MVT::getVectorVT(EltVT, Mask.size());
9594 
9595   // We can't assume a canonical shuffle mask, so try the commuted version too.
9596   SmallVector<int, 4> CommutedMask(Mask);
9597   ShuffleVectorSDNode::commuteMask(CommutedMask);
9598 
9599   // Match any of unary/binary or low/high.
9600   for (unsigned i = 0; i != 4; ++i) {
9601     SmallVector<int, 16> UnpackMask;
9602     createUnpackShuffleMask(VT, UnpackMask, (i >> 1) % 2, i % 2);
9603     if (isTargetShuffleEquivalent(VT, Mask, UnpackMask, DAG) ||
9604         isTargetShuffleEquivalent(VT, CommutedMask, UnpackMask, DAG))
9605       return true;
9606   }
9607   return false;
9608 }
9609 
9610 /// Return true if a shuffle mask chooses elements identically in its top and
9611 /// bottom halves. For example, any splat mask has the same top and bottom
9612 /// halves. If an element is undefined in only one half of the mask, the halves
9613 /// are not considered identical.
9614 static bool hasIdenticalHalvesShuffleMask(ArrayRef<int> Mask) {
9615   assert(Mask.size() % 2 == 0 && "Expecting even number of elements in mask");
9616   unsigned HalfSize = Mask.size() / 2;
9617   for (unsigned i = 0; i != HalfSize; ++i) {
9618     if (Mask[i] != Mask[i + HalfSize])
9619       return false;
9620   }
9621   return true;
9622 }
9623 
9624 /// Get a 4-lane 8-bit shuffle immediate for a mask.
9625 ///
9626 /// This helper function produces an 8-bit shuffle immediate corresponding to
9627 /// the ubiquitous shuffle encoding scheme used in x86 instructions for
9628 /// shuffling 4 lanes. It can be used with most of the PSHUF instructions for
9629 /// example.
9630 ///
9631 /// NB: We rely heavily on "undef" masks preserving the input lane.
9632 static unsigned getV4X86ShuffleImm(ArrayRef<int> Mask) {
9633   assert(Mask.size() == 4 && "Only 4-lane shuffle masks");
9634   assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!");
9635   assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!");
9636   assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!");
9637   assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!");
9638 
9639   // If the mask only uses one non-undef element, then fully 'splat' it to
9640   // improve later broadcast matching.
9641   int FirstIndex = find_if(Mask, [](int M) { return M >= 0; }) - Mask.begin();
9642   assert(0 <= FirstIndex && FirstIndex < 4 && "All undef shuffle mask");
9643 
9644   int FirstElt = Mask[FirstIndex];
9645   if (all_of(Mask, [FirstElt](int M) { return M < 0 || M == FirstElt; }))
9646     return (FirstElt << 6) | (FirstElt << 4) | (FirstElt << 2) | FirstElt;
9647 
9648   unsigned Imm = 0;
9649   Imm |= (Mask[0] < 0 ? 0 : Mask[0]) << 0;
9650   Imm |= (Mask[1] < 0 ? 1 : Mask[1]) << 2;
9651   Imm |= (Mask[2] < 0 ? 2 : Mask[2]) << 4;
9652   Imm |= (Mask[3] < 0 ? 3 : Mask[3]) << 6;
9653   return Imm;
9654 }
9655 
9656 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
9657                                           SelectionDAG &DAG) {
9658   return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);
9659 }
9660 
9661 // The Shuffle result is as follow:
9662 // 0*a[0]0*a[1]...0*a[n] , n >=0 where a[] elements in a ascending order.
9663 // Each Zeroable's element correspond to a particular Mask's element.
9664 // As described in computeZeroableShuffleElements function.
9665 //
9666 // The function looks for a sub-mask that the nonzero elements are in
9667 // increasing order. If such sub-mask exist. The function returns true.
9668 static bool isNonZeroElementsInOrder(const APInt &Zeroable,
9669                                      ArrayRef<int> Mask, const EVT &VectorType,
9670                                      bool &IsZeroSideLeft) {
9671   int NextElement = -1;
9672   // Check if the Mask's nonzero elements are in increasing order.
9673   for (int i = 0, e = Mask.size(); i < e; i++) {
9674     // Checks if the mask's zeros elements are built from only zeros.
9675     assert(Mask[i] >= -1 && "Out of bound mask element!");
9676     if (Mask[i] < 0)
9677       return false;
9678     if (Zeroable[i])
9679       continue;
9680     // Find the lowest non zero element
9681     if (NextElement < 0) {
9682       NextElement = Mask[i] != 0 ? VectorType.getVectorNumElements() : 0;
9683       IsZeroSideLeft = NextElement != 0;
9684     }
9685     // Exit if the mask's non zero elements are not in increasing order.
9686     if (NextElement != Mask[i])
9687       return false;
9688     NextElement++;
9689   }
9690   return true;
9691 }
9692 
9693 /// Try to lower a shuffle with a single PSHUFB of V1 or V2.
9694 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
9695                                       ArrayRef<int> Mask, SDValue V1,
9696                                       SDValue V2, const APInt &Zeroable,
9697                                       const X86Subtarget &Subtarget,
9698                                       SelectionDAG &DAG) {
9699   int Size = Mask.size();
9700   int LaneSize = 128 / VT.getScalarSizeInBits();
9701   const int NumBytes = VT.getSizeInBits() / 8;
9702   const int NumEltBytes = VT.getScalarSizeInBits() / 8;
9703 
9704   assert((Subtarget.hasSSSE3() && VT.is128BitVector()) ||
9705          (Subtarget.hasAVX2() && VT.is256BitVector()) ||
9706          (Subtarget.hasBWI() && VT.is512BitVector()));
9707 
9708   SmallVector<SDValue, 64> PSHUFBMask(NumBytes);
9709   // Sign bit set in i8 mask means zero element.
9710   SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
9711 
9712   SDValue V;
9713   for (int i = 0; i < NumBytes; ++i) {
9714     int M = Mask[i / NumEltBytes];
9715     if (M < 0) {
9716       PSHUFBMask[i] = DAG.getUNDEF(MVT::i8);
9717       continue;
9718     }
9719     if (Zeroable[i / NumEltBytes]) {
9720       PSHUFBMask[i] = ZeroMask;
9721       continue;
9722     }
9723 
9724     // We can only use a single input of V1 or V2.
9725     SDValue SrcV = (M >= Size ? V2 : V1);
9726     if (V && V != SrcV)
9727       return SDValue();
9728     V = SrcV;
9729     M %= Size;
9730 
9731     // PSHUFB can't cross lanes, ensure this doesn't happen.
9732     if ((M / LaneSize) != ((i / NumEltBytes) / LaneSize))
9733       return SDValue();
9734 
9735     M = M % LaneSize;
9736     M = M * NumEltBytes + (i % NumEltBytes);
9737     PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8);
9738   }
9739   assert(V && "Failed to find a source input");
9740 
9741   MVT I8VT = MVT::getVectorVT(MVT::i8, NumBytes);
9742   return DAG.getBitcast(
9743       VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V),
9744                       DAG.getBuildVector(I8VT, DL, PSHUFBMask)));
9745 }
9746 
9747 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
9748                            const X86Subtarget &Subtarget, SelectionDAG &DAG,
9749                            const SDLoc &dl);
9750 
9751 // X86 has dedicated shuffle that can be lowered to VEXPAND
9752 static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,
9753                                     const APInt &Zeroable,
9754                                     ArrayRef<int> Mask, SDValue &V1,
9755                                     SDValue &V2, SelectionDAG &DAG,
9756                                     const X86Subtarget &Subtarget) {
9757   bool IsLeftZeroSide = true;
9758   if (!isNonZeroElementsInOrder(Zeroable, Mask, V1.getValueType(),
9759                                 IsLeftZeroSide))
9760     return SDValue();
9761   unsigned VEXPANDMask = (~Zeroable).getZExtValue();
9762   MVT IntegerType =
9763       MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));
9764   SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
9765   unsigned NumElts = VT.getVectorNumElements();
9766   assert((NumElts == 4 || NumElts == 8 || NumElts == 16) &&
9767          "Unexpected number of vector elements");
9768   SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts),
9769                               Subtarget, DAG, DL);
9770   SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
9771   SDValue ExpandedVector = IsLeftZeroSide ? V2 : V1;
9772   return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask);
9773 }
9774 
9775 static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
9776                                   unsigned &UnpackOpcode, bool IsUnary,
9777                                   ArrayRef<int> TargetMask, const SDLoc &DL,
9778                                   SelectionDAG &DAG,
9779                                   const X86Subtarget &Subtarget) {
9780   int NumElts = VT.getVectorNumElements();
9781 
9782   bool Undef1 = true, Undef2 = true, Zero1 = true, Zero2 = true;
9783   for (int i = 0; i != NumElts; i += 2) {
9784     int M1 = TargetMask[i + 0];
9785     int M2 = TargetMask[i + 1];
9786     Undef1 &= (SM_SentinelUndef == M1);
9787     Undef2 &= (SM_SentinelUndef == M2);
9788     Zero1 &= isUndefOrZero(M1);
9789     Zero2 &= isUndefOrZero(M2);
9790   }
9791   assert(!((Undef1 || Zero1) && (Undef2 || Zero2)) &&
9792          "Zeroable shuffle detected");
9793 
9794   // Attempt to match the target mask against the unpack lo/hi mask patterns.
9795   SmallVector<int, 64> Unpckl, Unpckh;
9796   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, IsUnary);
9797   if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG, V1,
9798                                 (IsUnary ? V1 : V2))) {
9799     UnpackOpcode = X86ISD::UNPCKL;
9800     V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
9801     V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
9802     return true;
9803   }
9804 
9805   createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, IsUnary);
9806   if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG, V1,
9807                                 (IsUnary ? V1 : V2))) {
9808     UnpackOpcode = X86ISD::UNPCKH;
9809     V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
9810     V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
9811     return true;
9812   }
9813 
9814   // If an unary shuffle, attempt to match as an unpack lo/hi with zero.
9815   if (IsUnary && (Zero1 || Zero2)) {
9816     // Don't bother if we can blend instead.
9817     if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) &&
9818         isSequentialOrUndefOrZeroInRange(TargetMask, 0, NumElts, 0))
9819       return false;
9820 
9821     bool MatchLo = true, MatchHi = true;
9822     for (int i = 0; (i != NumElts) && (MatchLo || MatchHi); ++i) {
9823       int M = TargetMask[i];
9824 
9825       // Ignore if the input is known to be zero or the index is undef.
9826       if ((((i & 1) == 0) && Zero1) || (((i & 1) == 1) && Zero2) ||
9827           (M == SM_SentinelUndef))
9828         continue;
9829 
9830       MatchLo &= (M == Unpckl[i]);
9831       MatchHi &= (M == Unpckh[i]);
9832     }
9833 
9834     if (MatchLo || MatchHi) {
9835       UnpackOpcode = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
9836       V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
9837       V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
9838       return true;
9839     }
9840   }
9841 
9842   // If a binary shuffle, commute and try again.
9843   if (!IsUnary) {
9844     ShuffleVectorSDNode::commuteMask(Unpckl);
9845     if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG)) {
9846       UnpackOpcode = X86ISD::UNPCKL;
9847       std::swap(V1, V2);
9848       return true;
9849     }
9850 
9851     ShuffleVectorSDNode::commuteMask(Unpckh);
9852     if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG)) {
9853       UnpackOpcode = X86ISD::UNPCKH;
9854       std::swap(V1, V2);
9855       return true;
9856     }
9857   }
9858 
9859   return false;
9860 }
9861 
9862 // X86 has dedicated unpack instructions that can handle specific blend
9863 // operations: UNPCKH and UNPCKL.
9864 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
9865                                      ArrayRef<int> Mask, SDValue V1, SDValue V2,
9866                                      SelectionDAG &DAG) {
9867   SmallVector<int, 8> Unpckl;
9868   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);
9869   if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
9870     return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
9871 
9872   SmallVector<int, 8> Unpckh;
9873   createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, /* Unary = */ false);
9874   if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
9875     return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
9876 
9877   // Commute and try again.
9878   ShuffleVectorSDNode::commuteMask(Unpckl);
9879   if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
9880     return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
9881 
9882   ShuffleVectorSDNode::commuteMask(Unpckh);
9883   if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
9884     return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
9885 
9886   return SDValue();
9887 }
9888 
9889 /// Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit)
9890 /// followed by unpack 256-bit.
9891 static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,
9892                                         ArrayRef<int> Mask, SDValue V1,
9893                                         SDValue V2, SelectionDAG &DAG) {
9894   SmallVector<int, 32> Unpckl, Unpckh;
9895   createSplat2ShuffleMask(VT, Unpckl, /* Lo */ true);
9896   createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false);
9897 
9898   unsigned UnpackOpcode;
9899   if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
9900     UnpackOpcode = X86ISD::UNPCKL;
9901   else if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
9902     UnpackOpcode = X86ISD::UNPCKH;
9903   else
9904     return SDValue();
9905 
9906   // This is a "natural" unpack operation (rather than the 128-bit sectored
9907   // operation implemented by AVX). We need to rearrange 64-bit chunks of the
9908   // input in order to use the x86 instruction.
9909   V1 = DAG.getVectorShuffle(MVT::v4f64, DL, DAG.getBitcast(MVT::v4f64, V1),
9910                             DAG.getUNDEF(MVT::v4f64), {0, 2, 1, 3});
9911   V1 = DAG.getBitcast(VT, V1);
9912   return DAG.getNode(UnpackOpcode, DL, VT, V1, V1);
9913 }
9914 
9915 // Check if the mask can be mapped to a TRUNCATE or VTRUNC, truncating the
9916 // source into the lower elements and zeroing the upper elements.
9917 static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT,
9918                                  ArrayRef<int> Mask, const APInt &Zeroable,
9919                                  const X86Subtarget &Subtarget) {
9920   if (!VT.is512BitVector() && !Subtarget.hasVLX())
9921     return false;
9922 
9923   unsigned NumElts = Mask.size();
9924   unsigned EltSizeInBits = VT.getScalarSizeInBits();
9925   unsigned MaxScale = 64 / EltSizeInBits;
9926 
9927   for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
9928     unsigned SrcEltBits = EltSizeInBits * Scale;
9929     if (SrcEltBits < 32 && !Subtarget.hasBWI())
9930       continue;
9931     unsigned NumSrcElts = NumElts / Scale;
9932     if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale))
9933       continue;
9934     unsigned UpperElts = NumElts - NumSrcElts;
9935     if (!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
9936       continue;
9937     SrcVT = MVT::getIntegerVT(EltSizeInBits * Scale);
9938     SrcVT = MVT::getVectorVT(SrcVT, NumSrcElts);
9939     DstVT = MVT::getIntegerVT(EltSizeInBits);
9940     if ((NumSrcElts * EltSizeInBits) >= 128) {
9941       // ISD::TRUNCATE
9942       DstVT = MVT::getVectorVT(DstVT, NumSrcElts);
9943     } else {
9944       // X86ISD::VTRUNC
9945       DstVT = MVT::getVectorVT(DstVT, 128 / EltSizeInBits);
9946     }
9947     return true;
9948   }
9949 
9950   return false;
9951 }
9952 
9953 // Helper to create TRUNCATE/VTRUNC nodes, optionally with zero/undef upper
9954 // element padding to the final DstVT.
9955 static SDValue getAVX512TruncNode(const SDLoc &DL, MVT DstVT, SDValue Src,
9956                                   const X86Subtarget &Subtarget,
9957                                   SelectionDAG &DAG, bool ZeroUppers) {
9958   MVT SrcVT = Src.getSimpleValueType();
9959   MVT DstSVT = DstVT.getScalarType();
9960   unsigned NumDstElts = DstVT.getVectorNumElements();
9961   unsigned NumSrcElts = SrcVT.getVectorNumElements();
9962   unsigned DstEltSizeInBits = DstVT.getScalarSizeInBits();
9963 
9964   if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT))
9965     return SDValue();
9966 
9967   // Perform a direct ISD::TRUNCATE if possible.
9968   if (NumSrcElts == NumDstElts)
9969     return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Src);
9970 
9971   if (NumSrcElts > NumDstElts) {
9972     MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts);
9973     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
9974     return extractSubVector(Trunc, 0, DAG, DL, DstVT.getSizeInBits());
9975   }
9976 
9977   if ((NumSrcElts * DstEltSizeInBits) >= 128) {
9978     MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts);
9979     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
9980     return widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL,
9981                           DstVT.getSizeInBits());
9982   }
9983 
9984   // Non-VLX targets must truncate from a 512-bit type, so we need to
9985   // widen, truncate and then possibly extract the original subvector.
9986   if (!Subtarget.hasVLX() && !SrcVT.is512BitVector()) {
9987     SDValue NewSrc = widenSubVector(Src, ZeroUppers, Subtarget, DAG, DL, 512);
9988     return getAVX512TruncNode(DL, DstVT, NewSrc, Subtarget, DAG, ZeroUppers);
9989   }
9990 
9991   // Fallback to a X86ISD::VTRUNC, padding if necessary.
9992   MVT TruncVT = MVT::getVectorVT(DstSVT, 128 / DstEltSizeInBits);
9993   SDValue Trunc = DAG.getNode(X86ISD::VTRUNC, DL, TruncVT, Src);
9994   if (DstVT != TruncVT)
9995     Trunc = widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL,
9996                            DstVT.getSizeInBits());
9997   return Trunc;
9998 }
9999 
10000 // Try to lower trunc+vector_shuffle to a vpmovdb or a vpmovdw instruction.
10001 //
10002 // An example is the following:
10003 //
10004 // t0: ch = EntryToken
10005 //           t2: v4i64,ch = CopyFromReg t0, Register:v4i64 %0
10006 //         t25: v4i32 = truncate t2
10007 //       t41: v8i16 = bitcast t25
10008 //       t21: v8i16 = BUILD_VECTOR undef:i16, undef:i16, undef:i16, undef:i16,
10009 //       Constant:i16<0>, Constant:i16<0>, Constant:i16<0>, Constant:i16<0>
10010 //     t51: v8i16 = vector_shuffle<0,2,4,6,12,13,14,15> t41, t21
10011 //   t18: v2i64 = bitcast t51
10012 //
10013 // One can just use a single vpmovdw instruction, without avx512vl we need to
10014 // use the zmm variant and extract the lower subvector, padding with zeroes.
10015 // TODO: Merge with lowerShuffleAsVTRUNC.
10016 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1,
10017                                      SDValue V2, ArrayRef<int> Mask,
10018                                      const APInt &Zeroable,
10019                                      const X86Subtarget &Subtarget,
10020                                      SelectionDAG &DAG) {
10021   assert((VT == MVT::v16i8 || VT == MVT::v8i16) && "Unexpected VTRUNC type");
10022   if (!Subtarget.hasAVX512())
10023     return SDValue();
10024 
10025   unsigned NumElts = VT.getVectorNumElements();
10026   unsigned EltSizeInBits = VT.getScalarSizeInBits();
10027   unsigned MaxScale = 64 / EltSizeInBits;
10028   for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
10029     unsigned SrcEltBits = EltSizeInBits * Scale;
10030     unsigned NumSrcElts = NumElts / Scale;
10031     unsigned UpperElts = NumElts - NumSrcElts;
10032     if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) ||
10033         !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10034       continue;
10035 
10036     // Attempt to find a matching source truncation, but as a fall back VLX
10037     // cases can use the VPMOV directly.
10038     SDValue Src = peekThroughBitcasts(V1);
10039     if (Src.getOpcode() == ISD::TRUNCATE &&
10040         Src.getScalarValueSizeInBits() == SrcEltBits) {
10041       Src = Src.getOperand(0);
10042     } else if (Subtarget.hasVLX()) {
10043       MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10044       MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10045       Src = DAG.getBitcast(SrcVT, Src);
10046       // Don't do this if PACKSS/PACKUS could perform it cheaper.
10047       if (Scale == 2 &&
10048           ((DAG.ComputeNumSignBits(Src) > EltSizeInBits) ||
10049            (DAG.computeKnownBits(Src).countMinLeadingZeros() >= EltSizeInBits)))
10050         return SDValue();
10051     } else
10052       return SDValue();
10053 
10054     // VPMOVWB is only available with avx512bw.
10055     if (!Subtarget.hasBWI() && Src.getScalarValueSizeInBits() < 32)
10056       return SDValue();
10057 
10058     bool UndefUppers = isUndefInRange(Mask, NumSrcElts, UpperElts);
10059     return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers);
10060   }
10061 
10062   return SDValue();
10063 }
10064 
10065 // Attempt to match binary shuffle patterns as a truncate.
10066 static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,
10067                                     SDValue V2, ArrayRef<int> Mask,
10068                                     const APInt &Zeroable,
10069                                     const X86Subtarget &Subtarget,
10070                                     SelectionDAG &DAG) {
10071   assert((VT.is128BitVector() || VT.is256BitVector()) &&
10072          "Unexpected VTRUNC type");
10073   if (!Subtarget.hasAVX512())
10074     return SDValue();
10075 
10076   unsigned NumElts = VT.getVectorNumElements();
10077   unsigned EltSizeInBits = VT.getScalarSizeInBits();
10078   unsigned MaxScale = 64 / EltSizeInBits;
10079   for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
10080     // TODO: Support non-BWI VPMOVWB truncations?
10081     unsigned SrcEltBits = EltSizeInBits * Scale;
10082     if (SrcEltBits < 32 && !Subtarget.hasBWI())
10083       continue;
10084 
10085     // Match shuffle <Ofs,Ofs+Scale,Ofs+2*Scale,..,undef_or_zero,undef_or_zero>
10086     // Bail if the V2 elements are undef.
10087     unsigned NumHalfSrcElts = NumElts / Scale;
10088     unsigned NumSrcElts = 2 * NumHalfSrcElts;
10089     for (unsigned Offset = 0; Offset != Scale; ++Offset) {
10090       if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, Offset, Scale) ||
10091           isUndefInRange(Mask, NumHalfSrcElts, NumHalfSrcElts))
10092         continue;
10093 
10094       // The elements beyond the truncation must be undef/zero.
10095       unsigned UpperElts = NumElts - NumSrcElts;
10096       if (UpperElts > 0 &&
10097           !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10098         continue;
10099       bool UndefUppers =
10100           UpperElts > 0 && isUndefInRange(Mask, NumSrcElts, UpperElts);
10101 
10102       // For offset truncations, ensure that the concat is cheap.
10103       if (Offset) {
10104         auto IsCheapConcat = [&](SDValue Lo, SDValue Hi) {
10105           if (Lo.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
10106               Hi.getOpcode() == ISD::EXTRACT_SUBVECTOR)
10107             return Lo.getOperand(0) == Hi.getOperand(0);
10108           if (ISD::isNormalLoad(Lo.getNode()) &&
10109               ISD::isNormalLoad(Hi.getNode())) {
10110             auto *LDLo = cast<LoadSDNode>(Lo);
10111             auto *LDHi = cast<LoadSDNode>(Hi);
10112             return DAG.areNonVolatileConsecutiveLoads(
10113                 LDHi, LDLo, Lo.getValueType().getStoreSize(), 1);
10114           }
10115           return false;
10116         };
10117         if (!IsCheapConcat(V1, V2))
10118           continue;
10119       }
10120 
10121       // As we're using both sources then we need to concat them together
10122       // and truncate from the double-sized src.
10123       MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2);
10124       SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
10125 
10126       MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10127       MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10128       Src = DAG.getBitcast(SrcVT, Src);
10129 
10130       // Shift the offset'd elements into place for the truncation.
10131       // TODO: Use getTargetVShiftByConstNode.
10132       if (Offset)
10133         Src = DAG.getNode(
10134             X86ISD::VSRLI, DL, SrcVT, Src,
10135             DAG.getTargetConstant(Offset * EltSizeInBits, DL, MVT::i8));
10136 
10137       return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers);
10138     }
10139   }
10140 
10141   return SDValue();
10142 }
10143 
10144 /// Check whether a compaction lowering can be done by dropping even/odd
10145 /// elements and compute how many times even/odd elements must be dropped.
10146 ///
10147 /// This handles shuffles which take every Nth element where N is a power of
10148 /// two. Example shuffle masks:
10149 ///
10150 /// (even)
10151 ///  N = 1:  0,  2,  4,  6,  8, 10, 12, 14,  0,  2,  4,  6,  8, 10, 12, 14
10152 ///  N = 1:  0,  2,  4,  6,  8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
10153 ///  N = 2:  0,  4,  8, 12,  0,  4,  8, 12,  0,  4,  8, 12,  0,  4,  8, 12
10154 ///  N = 2:  0,  4,  8, 12, 16, 20, 24, 28,  0,  4,  8, 12, 16, 20, 24, 28
10155 ///  N = 3:  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8,  0,  8
10156 ///  N = 3:  0,  8, 16, 24,  0,  8, 16, 24,  0,  8, 16, 24,  0,  8, 16, 24
10157 ///
10158 /// (odd)
10159 ///  N = 1:  1,  3,  5,  7,  9, 11, 13, 15,  0,  2,  4,  6,  8, 10, 12, 14
10160 ///  N = 1:  1,  3,  5,  7,  9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
10161 ///
10162 /// Any of these lanes can of course be undef.
10163 ///
10164 /// This routine only supports N <= 3.
10165 /// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
10166 /// for larger N.
10167 ///
10168 /// \returns N above, or the number of times even/odd elements must be dropped
10169 /// if there is such a number. Otherwise returns zero.
10170 static int canLowerByDroppingElements(ArrayRef<int> Mask, bool MatchEven,
10171                                       bool IsSingleInput) {
10172   // The modulus for the shuffle vector entries is based on whether this is
10173   // a single input or not.
10174   int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2);
10175   assert(isPowerOf2_32((uint32_t)ShuffleModulus) &&
10176          "We should only be called with masks with a power-of-2 size!");
10177 
10178   uint64_t ModMask = (uint64_t)ShuffleModulus - 1;
10179   int Offset = MatchEven ? 0 : 1;
10180 
10181   // We track whether the input is viable for all power-of-2 strides 2^1, 2^2,
10182   // and 2^3 simultaneously. This is because we may have ambiguity with
10183   // partially undef inputs.
10184   bool ViableForN[3] = {true, true, true};
10185 
10186   for (int i = 0, e = Mask.size(); i < e; ++i) {
10187     // Ignore undef lanes, we'll optimistically collapse them to the pattern we
10188     // want.
10189     if (Mask[i] < 0)
10190       continue;
10191 
10192     bool IsAnyViable = false;
10193     for (unsigned j = 0; j != std::size(ViableForN); ++j)
10194       if (ViableForN[j]) {
10195         uint64_t N = j + 1;
10196 
10197         // The shuffle mask must be equal to (i * 2^N) % M.
10198         if ((uint64_t)(Mask[i] - Offset) == (((uint64_t)i << N) & ModMask))
10199           IsAnyViable = true;
10200         else
10201           ViableForN[j] = false;
10202       }
10203     // Early exit if we exhaust the possible powers of two.
10204     if (!IsAnyViable)
10205       break;
10206   }
10207 
10208   for (unsigned j = 0; j != std::size(ViableForN); ++j)
10209     if (ViableForN[j])
10210       return j + 1;
10211 
10212   // Return 0 as there is no viable power of two.
10213   return 0;
10214 }
10215 
10216 // X86 has dedicated pack instructions that can handle specific truncation
10217 // operations: PACKSS and PACKUS.
10218 // Checks for compaction shuffle masks if MaxStages > 1.
10219 // TODO: Add support for matching multiple PACKSS/PACKUS stages.
10220 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
10221                                  unsigned &PackOpcode, ArrayRef<int> TargetMask,
10222                                  const SelectionDAG &DAG,
10223                                  const X86Subtarget &Subtarget,
10224                                  unsigned MaxStages = 1) {
10225   unsigned NumElts = VT.getVectorNumElements();
10226   unsigned BitSize = VT.getScalarSizeInBits();
10227   assert(0 < MaxStages && MaxStages <= 3 && (BitSize << MaxStages) <= 64 &&
10228          "Illegal maximum compaction");
10229 
10230   auto MatchPACK = [&](SDValue N1, SDValue N2, MVT PackVT) {
10231     unsigned NumSrcBits = PackVT.getScalarSizeInBits();
10232     unsigned NumPackedBits = NumSrcBits - BitSize;
10233     N1 = peekThroughBitcasts(N1);
10234     N2 = peekThroughBitcasts(N2);
10235     unsigned NumBits1 = N1.getScalarValueSizeInBits();
10236     unsigned NumBits2 = N2.getScalarValueSizeInBits();
10237     bool IsZero1 = llvm::isNullOrNullSplat(N1, /*AllowUndefs*/ false);
10238     bool IsZero2 = llvm::isNullOrNullSplat(N2, /*AllowUndefs*/ false);
10239     if ((!N1.isUndef() && !IsZero1 && NumBits1 != NumSrcBits) ||
10240         (!N2.isUndef() && !IsZero2 && NumBits2 != NumSrcBits))
10241       return false;
10242     if (Subtarget.hasSSE41() || BitSize == 8) {
10243       APInt ZeroMask = APInt::getHighBitsSet(NumSrcBits, NumPackedBits);
10244       if ((N1.isUndef() || IsZero1 || DAG.MaskedValueIsZero(N1, ZeroMask)) &&
10245           (N2.isUndef() || IsZero2 || DAG.MaskedValueIsZero(N2, ZeroMask))) {
10246         V1 = N1;
10247         V2 = N2;
10248         SrcVT = PackVT;
10249         PackOpcode = X86ISD::PACKUS;
10250         return true;
10251       }
10252     }
10253     bool IsAllOnes1 = llvm::isAllOnesOrAllOnesSplat(N1, /*AllowUndefs*/ false);
10254     bool IsAllOnes2 = llvm::isAllOnesOrAllOnesSplat(N2, /*AllowUndefs*/ false);
10255     if ((N1.isUndef() || IsZero1 || IsAllOnes1 ||
10256          DAG.ComputeNumSignBits(N1) > NumPackedBits) &&
10257         (N2.isUndef() || IsZero2 || IsAllOnes2 ||
10258          DAG.ComputeNumSignBits(N2) > NumPackedBits)) {
10259       V1 = N1;
10260       V2 = N2;
10261       SrcVT = PackVT;
10262       PackOpcode = X86ISD::PACKSS;
10263       return true;
10264     }
10265     return false;
10266   };
10267 
10268   // Attempt to match against wider and wider compaction patterns.
10269   for (unsigned NumStages = 1; NumStages <= MaxStages; ++NumStages) {
10270     MVT PackSVT = MVT::getIntegerVT(BitSize << NumStages);
10271     MVT PackVT = MVT::getVectorVT(PackSVT, NumElts >> NumStages);
10272 
10273     // Try binary shuffle.
10274     SmallVector<int, 32> BinaryMask;
10275     createPackShuffleMask(VT, BinaryMask, false, NumStages);
10276     if (isTargetShuffleEquivalent(VT, TargetMask, BinaryMask, DAG, V1, V2))
10277       if (MatchPACK(V1, V2, PackVT))
10278         return true;
10279 
10280     // Try unary shuffle.
10281     SmallVector<int, 32> UnaryMask;
10282     createPackShuffleMask(VT, UnaryMask, true, NumStages);
10283     if (isTargetShuffleEquivalent(VT, TargetMask, UnaryMask, DAG, V1))
10284       if (MatchPACK(V1, V1, PackVT))
10285         return true;
10286   }
10287 
10288   return false;
10289 }
10290 
10291 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
10292                                     SDValue V1, SDValue V2, SelectionDAG &DAG,
10293                                     const X86Subtarget &Subtarget) {
10294   MVT PackVT;
10295   unsigned PackOpcode;
10296   unsigned SizeBits = VT.getSizeInBits();
10297   unsigned EltBits = VT.getScalarSizeInBits();
10298   unsigned MaxStages = Log2_32(64 / EltBits);
10299   if (!matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,
10300                             Subtarget, MaxStages))
10301     return SDValue();
10302 
10303   unsigned CurrentEltBits = PackVT.getScalarSizeInBits();
10304   unsigned NumStages = Log2_32(CurrentEltBits / EltBits);
10305 
10306   // Don't lower multi-stage packs on AVX512, truncation is better.
10307   if (NumStages != 1 && SizeBits == 128 && Subtarget.hasVLX())
10308     return SDValue();
10309 
10310   // Pack to the largest type possible:
10311   // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB.
10312   unsigned MaxPackBits = 16;
10313   if (CurrentEltBits > 16 &&
10314       (PackOpcode == X86ISD::PACKSS || Subtarget.hasSSE41()))
10315     MaxPackBits = 32;
10316 
10317   // Repeatedly pack down to the target size.
10318   SDValue Res;
10319   for (unsigned i = 0; i != NumStages; ++i) {
10320     unsigned SrcEltBits = std::min(MaxPackBits, CurrentEltBits);
10321     unsigned NumSrcElts = SizeBits / SrcEltBits;
10322     MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10323     MVT DstSVT = MVT::getIntegerVT(SrcEltBits / 2);
10324     MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10325     MVT DstVT = MVT::getVectorVT(DstSVT, NumSrcElts * 2);
10326     Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1),
10327                       DAG.getBitcast(SrcVT, V2));
10328     V1 = V2 = Res;
10329     CurrentEltBits /= 2;
10330   }
10331   assert(Res && Res.getValueType() == VT &&
10332          "Failed to lower compaction shuffle");
10333   return Res;
10334 }
10335 
10336 /// Try to emit a bitmask instruction for a shuffle.
10337 ///
10338 /// This handles cases where we can model a blend exactly as a bitmask due to
10339 /// one of the inputs being zeroable.
10340 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
10341                                      SDValue V2, ArrayRef<int> Mask,
10342                                      const APInt &Zeroable,
10343                                      const X86Subtarget &Subtarget,
10344                                      SelectionDAG &DAG) {
10345   MVT MaskVT = VT;
10346   MVT EltVT = VT.getVectorElementType();
10347   SDValue Zero, AllOnes;
10348   // Use f64 if i64 isn't legal.
10349   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
10350     EltVT = MVT::f64;
10351     MaskVT = MVT::getVectorVT(EltVT, Mask.size());
10352   }
10353 
10354   MVT LogicVT = VT;
10355   if (EltVT == MVT::f32 || EltVT == MVT::f64) {
10356     Zero = DAG.getConstantFP(0.0, DL, EltVT);
10357     APFloat AllOnesValue =
10358         APFloat::getAllOnesValue(SelectionDAG::EVTToAPFloatSemantics(EltVT));
10359     AllOnes = DAG.getConstantFP(AllOnesValue, DL, EltVT);
10360     LogicVT =
10361         MVT::getVectorVT(EltVT == MVT::f64 ? MVT::i64 : MVT::i32, Mask.size());
10362   } else {
10363     Zero = DAG.getConstant(0, DL, EltVT);
10364     AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10365   }
10366 
10367   SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
10368   SDValue V;
10369   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10370     if (Zeroable[i])
10371       continue;
10372     if (Mask[i] % Size != i)
10373       return SDValue(); // Not a blend.
10374     if (!V)
10375       V = Mask[i] < Size ? V1 : V2;
10376     else if (V != (Mask[i] < Size ? V1 : V2))
10377       return SDValue(); // Can only let one input through the mask.
10378 
10379     VMaskOps[i] = AllOnes;
10380   }
10381   if (!V)
10382     return SDValue(); // No non-zeroable elements!
10383 
10384   SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
10385   VMask = DAG.getBitcast(LogicVT, VMask);
10386   V = DAG.getBitcast(LogicVT, V);
10387   SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
10388   return DAG.getBitcast(VT, And);
10389 }
10390 
10391 /// Try to emit a blend instruction for a shuffle using bit math.
10392 ///
10393 /// This is used as a fallback approach when first class blend instructions are
10394 /// unavailable. Currently it is only suitable for integer vectors, but could
10395 /// be generalized for floating point vectors if desirable.
10396 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
10397                                       SDValue V2, ArrayRef<int> Mask,
10398                                       SelectionDAG &DAG) {
10399   assert(VT.isInteger() && "Only supports integer vector types!");
10400   MVT EltVT = VT.getVectorElementType();
10401   SDValue Zero = DAG.getConstant(0, DL, EltVT);
10402   SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10403   SmallVector<SDValue, 16> MaskOps;
10404   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10405     if (Mask[i] >= 0 && Mask[i] != i && Mask[i] != i + Size)
10406       return SDValue(); // Shuffled input!
10407     MaskOps.push_back(Mask[i] < Size ? AllOnes : Zero);
10408   }
10409 
10410   SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
10411   return getBitSelect(DL, VT, V1, V2, V1Mask, DAG);
10412 }
10413 
10414 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
10415                                     SDValue PreservedSrc,
10416                                     const X86Subtarget &Subtarget,
10417                                     SelectionDAG &DAG);
10418 
10419 static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2,
10420                                 MutableArrayRef<int> Mask,
10421                                 const APInt &Zeroable, bool &ForceV1Zero,
10422                                 bool &ForceV2Zero, uint64_t &BlendMask) {
10423   bool V1IsZeroOrUndef =
10424       V1.isUndef() || ISD::isBuildVectorAllZeros(V1.getNode());
10425   bool V2IsZeroOrUndef =
10426       V2.isUndef() || ISD::isBuildVectorAllZeros(V2.getNode());
10427 
10428   BlendMask = 0;
10429   ForceV1Zero = false, ForceV2Zero = false;
10430   assert(Mask.size() <= 64 && "Shuffle mask too big for blend mask");
10431 
10432   int NumElts = Mask.size();
10433   int NumLanes = VT.getSizeInBits() / 128;
10434   int NumEltsPerLane = NumElts / NumLanes;
10435   assert((NumLanes * NumEltsPerLane) == NumElts && "Value type mismatch");
10436 
10437   // For 32/64-bit elements, if we only reference one input (plus any undefs),
10438   // then ensure the blend mask part for that lane just references that input.
10439   bool ForceWholeLaneMasks =
10440       VT.is256BitVector() && VT.getScalarSizeInBits() >= 32;
10441 
10442   // Attempt to generate the binary blend mask. If an input is zero then
10443   // we can use any lane.
10444   for (int Lane = 0; Lane != NumLanes; ++Lane) {
10445     // Keep track of the inputs used per lane.
10446     bool LaneV1InUse = false;
10447     bool LaneV2InUse = false;
10448     uint64_t LaneBlendMask = 0;
10449     for (int LaneElt = 0; LaneElt != NumEltsPerLane; ++LaneElt) {
10450       int Elt = (Lane * NumEltsPerLane) + LaneElt;
10451       int M = Mask[Elt];
10452       if (M == SM_SentinelUndef)
10453         continue;
10454       if (M == Elt || (0 <= M && M < NumElts &&
10455                      IsElementEquivalent(NumElts, V1, V1, M, Elt))) {
10456         Mask[Elt] = Elt;
10457         LaneV1InUse = true;
10458         continue;
10459       }
10460       if (M == (Elt + NumElts) ||
10461           (NumElts <= M &&
10462            IsElementEquivalent(NumElts, V2, V2, M - NumElts, Elt))) {
10463         LaneBlendMask |= 1ull << LaneElt;
10464         Mask[Elt] = Elt + NumElts;
10465         LaneV2InUse = true;
10466         continue;
10467       }
10468       if (Zeroable[Elt]) {
10469         if (V1IsZeroOrUndef) {
10470           ForceV1Zero = true;
10471           Mask[Elt] = Elt;
10472           LaneV1InUse = true;
10473           continue;
10474         }
10475         if (V2IsZeroOrUndef) {
10476           ForceV2Zero = true;
10477           LaneBlendMask |= 1ull << LaneElt;
10478           Mask[Elt] = Elt + NumElts;
10479           LaneV2InUse = true;
10480           continue;
10481         }
10482       }
10483       return false;
10484     }
10485 
10486     // If we only used V2 then splat the lane blend mask to avoid any demanded
10487     // elts from V1 in this lane (the V1 equivalent is implicit with a zero
10488     // blend mask bit).
10489     if (ForceWholeLaneMasks && LaneV2InUse && !LaneV1InUse)
10490       LaneBlendMask = (1ull << NumEltsPerLane) - 1;
10491 
10492     BlendMask |= LaneBlendMask << (Lane * NumEltsPerLane);
10493   }
10494   return true;
10495 }
10496 
10497 static uint64_t scaleVectorShuffleBlendMask(uint64_t BlendMask, int Size,
10498                                             int Scale) {
10499   uint64_t ScaledMask = 0;
10500   for (int i = 0; i != Size; ++i)
10501     if (BlendMask & (1ull << i))
10502       ScaledMask |= ((1ull << Scale) - 1) << (i * Scale);
10503   return ScaledMask;
10504 }
10505 
10506 /// Try to emit a blend instruction for a shuffle.
10507 ///
10508 /// This doesn't do any checks for the availability of instructions for blending
10509 /// these values. It relies on the availability of the X86ISD::BLENDI pattern to
10510 /// be matched in the backend with the type given. What it does check for is
10511 /// that the shuffle mask is a blend, or convertible into a blend with zero.
10512 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
10513                                    SDValue V2, ArrayRef<int> Original,
10514                                    const APInt &Zeroable,
10515                                    const X86Subtarget &Subtarget,
10516                                    SelectionDAG &DAG) {
10517   uint64_t BlendMask = 0;
10518   bool ForceV1Zero = false, ForceV2Zero = false;
10519   SmallVector<int, 64> Mask(Original);
10520   if (!matchShuffleAsBlend(VT, V1, V2, Mask, Zeroable, ForceV1Zero, ForceV2Zero,
10521                            BlendMask))
10522     return SDValue();
10523 
10524   // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
10525   if (ForceV1Zero)
10526     V1 = getZeroVector(VT, Subtarget, DAG, DL);
10527   if (ForceV2Zero)
10528     V2 = getZeroVector(VT, Subtarget, DAG, DL);
10529 
10530   unsigned NumElts = VT.getVectorNumElements();
10531 
10532   switch (VT.SimpleTy) {
10533   case MVT::v4i64:
10534   case MVT::v8i32:
10535     assert(Subtarget.hasAVX2() && "256-bit integer blends require AVX2!");
10536     [[fallthrough]];
10537   case MVT::v4f64:
10538   case MVT::v8f32:
10539     assert(Subtarget.hasAVX() && "256-bit float blends require AVX!");
10540     [[fallthrough]];
10541   case MVT::v2f64:
10542   case MVT::v2i64:
10543   case MVT::v4f32:
10544   case MVT::v4i32:
10545   case MVT::v8i16:
10546     assert(Subtarget.hasSSE41() && "128-bit blends require SSE41!");
10547     return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
10548                        DAG.getTargetConstant(BlendMask, DL, MVT::i8));
10549   case MVT::v16i16: {
10550     assert(Subtarget.hasAVX2() && "v16i16 blends require AVX2!");
10551     SmallVector<int, 8> RepeatedMask;
10552     if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
10553       // We can lower these with PBLENDW which is mirrored across 128-bit lanes.
10554       assert(RepeatedMask.size() == 8 && "Repeated mask size doesn't match!");
10555       BlendMask = 0;
10556       for (int i = 0; i < 8; ++i)
10557         if (RepeatedMask[i] >= 8)
10558           BlendMask |= 1ull << i;
10559       return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10560                          DAG.getTargetConstant(BlendMask, DL, MVT::i8));
10561     }
10562     // Use PBLENDW for lower/upper lanes and then blend lanes.
10563     // TODO - we should allow 2 PBLENDW here and leave shuffle combine to
10564     // merge to VSELECT where useful.
10565     uint64_t LoMask = BlendMask & 0xFF;
10566     uint64_t HiMask = (BlendMask >> 8) & 0xFF;
10567     if (LoMask == 0 || LoMask == 255 || HiMask == 0 || HiMask == 255) {
10568       SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10569                                DAG.getTargetConstant(LoMask, DL, MVT::i8));
10570       SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10571                                DAG.getTargetConstant(HiMask, DL, MVT::i8));
10572       return DAG.getVectorShuffle(
10573           MVT::v16i16, DL, Lo, Hi,
10574           {0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31});
10575     }
10576     [[fallthrough]];
10577   }
10578   case MVT::v32i8:
10579     assert(Subtarget.hasAVX2() && "256-bit byte-blends require AVX2!");
10580     [[fallthrough]];
10581   case MVT::v16i8: {
10582     assert(Subtarget.hasSSE41() && "128-bit byte-blends require SSE41!");
10583 
10584     // Attempt to lower to a bitmask if we can. VPAND is faster than VPBLENDVB.
10585     if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
10586                                                Subtarget, DAG))
10587       return Masked;
10588 
10589     if (Subtarget.hasBWI() && Subtarget.hasVLX()) {
10590       MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8));
10591       SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
10592       return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
10593     }
10594 
10595     // If we have VPTERNLOG, we can use that as a bit blend.
10596     if (Subtarget.hasVLX())
10597       if (SDValue BitBlend =
10598               lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
10599         return BitBlend;
10600 
10601     // Scale the blend by the number of bytes per element.
10602     int Scale = VT.getScalarSizeInBits() / 8;
10603 
10604     // This form of blend is always done on bytes. Compute the byte vector
10605     // type.
10606     MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
10607 
10608     // x86 allows load folding with blendvb from the 2nd source operand. But
10609     // we are still using LLVM select here (see comment below), so that's V1.
10610     // If V2 can be load-folded and V1 cannot be load-folded, then commute to
10611     // allow that load-folding possibility.
10612     if (!ISD::isNormalLoad(V1.getNode()) && ISD::isNormalLoad(V2.getNode())) {
10613       ShuffleVectorSDNode::commuteMask(Mask);
10614       std::swap(V1, V2);
10615     }
10616 
10617     // Compute the VSELECT mask. Note that VSELECT is really confusing in the
10618     // mix of LLVM's code generator and the x86 backend. We tell the code
10619     // generator that boolean values in the elements of an x86 vector register
10620     // are -1 for true and 0 for false. We then use the LLVM semantics of 'true'
10621     // mapping a select to operand #1, and 'false' mapping to operand #2. The
10622     // reality in x86 is that vector masks (pre-AVX-512) use only the high bit
10623     // of the element (the remaining are ignored) and 0 in that high bit would
10624     // mean operand #1 while 1 in the high bit would mean operand #2. So while
10625     // the LLVM model for boolean values in vector elements gets the relevant
10626     // bit set, it is set backwards and over constrained relative to x86's
10627     // actual model.
10628     SmallVector<SDValue, 32> VSELECTMask;
10629     for (int i = 0, Size = Mask.size(); i < Size; ++i)
10630       for (int j = 0; j < Scale; ++j)
10631         VSELECTMask.push_back(
10632             Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)
10633                         : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL,
10634                                           MVT::i8));
10635 
10636     V1 = DAG.getBitcast(BlendVT, V1);
10637     V2 = DAG.getBitcast(BlendVT, V2);
10638     return DAG.getBitcast(
10639         VT,
10640         DAG.getSelect(DL, BlendVT, DAG.getBuildVector(BlendVT, DL, VSELECTMask),
10641                       V1, V2));
10642   }
10643   case MVT::v16f32:
10644   case MVT::v8f64:
10645   case MVT::v8i64:
10646   case MVT::v16i32:
10647   case MVT::v32i16:
10648   case MVT::v64i8: {
10649     // Attempt to lower to a bitmask if we can. Only if not optimizing for size.
10650     bool OptForSize = DAG.shouldOptForSize();
10651     if (!OptForSize) {
10652       if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
10653                                                  Subtarget, DAG))
10654         return Masked;
10655     }
10656 
10657     // Otherwise load an immediate into a GPR, cast to k-register, and use a
10658     // masked move.
10659     MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8));
10660     SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
10661     return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
10662   }
10663   default:
10664     llvm_unreachable("Not a supported integer vector type!");
10665   }
10666 }
10667 
10668 /// Try to lower as a blend of elements from two inputs followed by
10669 /// a single-input permutation.
10670 ///
10671 /// This matches the pattern where we can blend elements from two inputs and
10672 /// then reduce the shuffle to a single-input permutation.
10673 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
10674                                              SDValue V1, SDValue V2,
10675                                              ArrayRef<int> Mask,
10676                                              SelectionDAG &DAG,
10677                                              bool ImmBlends = false) {
10678   // We build up the blend mask while checking whether a blend is a viable way
10679   // to reduce the shuffle.
10680   SmallVector<int, 32> BlendMask(Mask.size(), -1);
10681   SmallVector<int, 32> PermuteMask(Mask.size(), -1);
10682 
10683   for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10684     if (Mask[i] < 0)
10685       continue;
10686 
10687     assert(Mask[i] < Size * 2 && "Shuffle input is out of bounds.");
10688 
10689     if (BlendMask[Mask[i] % Size] < 0)
10690       BlendMask[Mask[i] % Size] = Mask[i];
10691     else if (BlendMask[Mask[i] % Size] != Mask[i])
10692       return SDValue(); // Can't blend in the needed input!
10693 
10694     PermuteMask[i] = Mask[i] % Size;
10695   }
10696 
10697   // If only immediate blends, then bail if the blend mask can't be widened to
10698   // i16.
10699   unsigned EltSize = VT.getScalarSizeInBits();
10700   if (ImmBlends && EltSize == 8 && !canWidenShuffleElements(BlendMask))
10701     return SDValue();
10702 
10703   SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
10704   return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);
10705 }
10706 
10707 /// Try to lower as an unpack of elements from two inputs followed by
10708 /// a single-input permutation.
10709 ///
10710 /// This matches the pattern where we can unpack elements from two inputs and
10711 /// then reduce the shuffle to a single-input (wider) permutation.
10712 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
10713                                              SDValue V1, SDValue V2,
10714                                              ArrayRef<int> Mask,
10715                                              SelectionDAG &DAG) {
10716   int NumElts = Mask.size();
10717   int NumLanes = VT.getSizeInBits() / 128;
10718   int NumLaneElts = NumElts / NumLanes;
10719   int NumHalfLaneElts = NumLaneElts / 2;
10720 
10721   bool MatchLo = true, MatchHi = true;
10722   SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
10723 
10724   // Determine UNPCKL/UNPCKH type and operand order.
10725   for (int Elt = 0; Elt != NumElts; ++Elt) {
10726     int M = Mask[Elt];
10727     if (M < 0)
10728       continue;
10729 
10730     // Normalize the mask value depending on whether it's V1 or V2.
10731     int NormM = M;
10732     SDValue &Op = Ops[Elt & 1];
10733     if (M < NumElts && (Op.isUndef() || Op == V1))
10734       Op = V1;
10735     else if (NumElts <= M && (Op.isUndef() || Op == V2)) {
10736       Op = V2;
10737       NormM -= NumElts;
10738     } else
10739       return SDValue();
10740 
10741     bool MatchLoAnyLane = false, MatchHiAnyLane = false;
10742     for (int Lane = 0; Lane != NumElts; Lane += NumLaneElts) {
10743       int Lo = Lane, Mid = Lane + NumHalfLaneElts, Hi = Lane + NumLaneElts;
10744       MatchLoAnyLane |= isUndefOrInRange(NormM, Lo, Mid);
10745       MatchHiAnyLane |= isUndefOrInRange(NormM, Mid, Hi);
10746       if (MatchLoAnyLane || MatchHiAnyLane) {
10747         assert((MatchLoAnyLane ^ MatchHiAnyLane) &&
10748                "Failed to match UNPCKLO/UNPCKHI");
10749         break;
10750       }
10751     }
10752     MatchLo &= MatchLoAnyLane;
10753     MatchHi &= MatchHiAnyLane;
10754     if (!MatchLo && !MatchHi)
10755       return SDValue();
10756   }
10757   assert((MatchLo ^ MatchHi) && "Failed to match UNPCKLO/UNPCKHI");
10758 
10759   // Element indices have changed after unpacking. Calculate permute mask
10760   // so that they will be put back to the position as dictated by the
10761   // original shuffle mask indices.
10762   SmallVector<int, 32> PermuteMask(NumElts, -1);
10763   for (int Elt = 0; Elt != NumElts; ++Elt) {
10764     int M = Mask[Elt];
10765     if (M < 0)
10766       continue;
10767     int NormM = M;
10768     if (NumElts <= M)
10769       NormM -= NumElts;
10770     bool IsFirstOp = M < NumElts;
10771     int BaseMaskElt =
10772         NumLaneElts * (NormM / NumLaneElts) + (2 * (NormM % NumHalfLaneElts));
10773     if ((IsFirstOp && V1 == Ops[0]) || (!IsFirstOp && V2 == Ops[0]))
10774       PermuteMask[Elt] = BaseMaskElt;
10775     else if ((IsFirstOp && V1 == Ops[1]) || (!IsFirstOp && V2 == Ops[1]))
10776       PermuteMask[Elt] = BaseMaskElt + 1;
10777     assert(PermuteMask[Elt] != -1 &&
10778            "Input mask element is defined but failed to assign permute mask");
10779   }
10780 
10781   unsigned UnpckOp = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
10782   SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
10783   return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask);
10784 }
10785 
10786 /// Try to lower a shuffle as a permute of the inputs followed by an
10787 /// UNPCK instruction.
10788 ///
10789 /// This specifically targets cases where we end up with alternating between
10790 /// the two inputs, and so can permute them into something that feeds a single
10791 /// UNPCK instruction. Note that this routine only targets integer vectors
10792 /// because for floating point vectors we have a generalized SHUFPS lowering
10793 /// strategy that handles everything that doesn't *exactly* match an unpack,
10794 /// making this clever lowering unnecessary.
10795 static SDValue lowerShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
10796                                               SDValue V1, SDValue V2,
10797                                               ArrayRef<int> Mask,
10798                                               const X86Subtarget &Subtarget,
10799                                               SelectionDAG &DAG) {
10800   int Size = Mask.size();
10801   assert(Mask.size() >= 2 && "Single element masks are invalid.");
10802 
10803   // This routine only supports 128-bit integer dual input vectors.
10804   if (VT.isFloatingPoint() || !VT.is128BitVector() || V2.isUndef())
10805     return SDValue();
10806 
10807   int NumLoInputs =
10808       count_if(Mask, [Size](int M) { return M >= 0 && M % Size < Size / 2; });
10809   int NumHiInputs =
10810       count_if(Mask, [Size](int M) { return M % Size >= Size / 2; });
10811 
10812   bool UnpackLo = NumLoInputs >= NumHiInputs;
10813 
10814   auto TryUnpack = [&](int ScalarSize, int Scale) {
10815     SmallVector<int, 16> V1Mask((unsigned)Size, -1);
10816     SmallVector<int, 16> V2Mask((unsigned)Size, -1);
10817 
10818     for (int i = 0; i < Size; ++i) {
10819       if (Mask[i] < 0)
10820         continue;
10821 
10822       // Each element of the unpack contains Scale elements from this mask.
10823       int UnpackIdx = i / Scale;
10824 
10825       // We only handle the case where V1 feeds the first slots of the unpack.
10826       // We rely on canonicalization to ensure this is the case.
10827       if ((UnpackIdx % 2 == 0) != (Mask[i] < Size))
10828         return SDValue();
10829 
10830       // Setup the mask for this input. The indexing is tricky as we have to
10831       // handle the unpack stride.
10832       SmallVectorImpl<int> &VMask = (UnpackIdx % 2 == 0) ? V1Mask : V2Mask;
10833       VMask[(UnpackIdx / 2) * Scale + i % Scale + (UnpackLo ? 0 : Size / 2)] =
10834           Mask[i] % Size;
10835     }
10836 
10837     // If we will have to shuffle both inputs to use the unpack, check whether
10838     // we can just unpack first and shuffle the result. If so, skip this unpack.
10839     if ((NumLoInputs == 0 || NumHiInputs == 0) && !isNoopShuffleMask(V1Mask) &&
10840         !isNoopShuffleMask(V2Mask))
10841       return SDValue();
10842 
10843     // Shuffle the inputs into place.
10844     V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
10845     V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
10846 
10847     // Cast the inputs to the type we will use to unpack them.
10848     MVT UnpackVT =
10849         MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), Size / Scale);
10850     V1 = DAG.getBitcast(UnpackVT, V1);
10851     V2 = DAG.getBitcast(UnpackVT, V2);
10852 
10853     // Unpack the inputs and cast the result back to the desired type.
10854     return DAG.getBitcast(
10855         VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
10856                         UnpackVT, V1, V2));
10857   };
10858 
10859   // We try each unpack from the largest to the smallest to try and find one
10860   // that fits this mask.
10861   int OrigScalarSize = VT.getScalarSizeInBits();
10862   for (int ScalarSize = 64; ScalarSize >= OrigScalarSize; ScalarSize /= 2)
10863     if (SDValue Unpack = TryUnpack(ScalarSize, ScalarSize / OrigScalarSize))
10864       return Unpack;
10865 
10866   // If we're shuffling with a zero vector then we're better off not doing
10867   // VECTOR_SHUFFLE(UNPCK()) as we lose track of those zero elements.
10868   if (ISD::isBuildVectorAllZeros(V1.getNode()) ||
10869       ISD::isBuildVectorAllZeros(V2.getNode()))
10870     return SDValue();
10871 
10872   // If none of the unpack-rooted lowerings worked (or were profitable) try an
10873   // initial unpack.
10874   if (NumLoInputs == 0 || NumHiInputs == 0) {
10875     assert((NumLoInputs > 0 || NumHiInputs > 0) &&
10876            "We have to have *some* inputs!");
10877     int HalfOffset = NumLoInputs == 0 ? Size / 2 : 0;
10878 
10879     // FIXME: We could consider the total complexity of the permute of each
10880     // possible unpacking. Or at the least we should consider how many
10881     // half-crossings are created.
10882     // FIXME: We could consider commuting the unpacks.
10883 
10884     SmallVector<int, 32> PermMask((unsigned)Size, -1);
10885     for (int i = 0; i < Size; ++i) {
10886       if (Mask[i] < 0)
10887         continue;
10888 
10889       assert(Mask[i] % Size >= HalfOffset && "Found input from wrong half!");
10890 
10891       PermMask[i] =
10892           2 * ((Mask[i] % Size) - HalfOffset) + (Mask[i] < Size ? 0 : 1);
10893     }
10894     return DAG.getVectorShuffle(
10895         VT, DL,
10896         DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, DL, VT,
10897                     V1, V2),
10898         DAG.getUNDEF(VT), PermMask);
10899   }
10900 
10901   return SDValue();
10902 }
10903 
10904 /// Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then
10905 /// permuting the elements of the result in place.
10906 static SDValue lowerShuffleAsByteRotateAndPermute(
10907     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
10908     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
10909   if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) ||
10910       (VT.is256BitVector() && !Subtarget.hasAVX2()) ||
10911       (VT.is512BitVector() && !Subtarget.hasBWI()))
10912     return SDValue();
10913 
10914   // We don't currently support lane crossing permutes.
10915   if (is128BitLaneCrossingShuffleMask(VT, Mask))
10916     return SDValue();
10917 
10918   int Scale = VT.getScalarSizeInBits() / 8;
10919   int NumLanes = VT.getSizeInBits() / 128;
10920   int NumElts = VT.getVectorNumElements();
10921   int NumEltsPerLane = NumElts / NumLanes;
10922 
10923   // Determine range of mask elts.
10924   bool Blend1 = true;
10925   bool Blend2 = true;
10926   std::pair<int, int> Range1 = std::make_pair(INT_MAX, INT_MIN);
10927   std::pair<int, int> Range2 = std::make_pair(INT_MAX, INT_MIN);
10928   for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
10929     for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
10930       int M = Mask[Lane + Elt];
10931       if (M < 0)
10932         continue;
10933       if (M < NumElts) {
10934         Blend1 &= (M == (Lane + Elt));
10935         assert(Lane <= M && M < (Lane + NumEltsPerLane) && "Out of range mask");
10936         M = M % NumEltsPerLane;
10937         Range1.first = std::min(Range1.first, M);
10938         Range1.second = std::max(Range1.second, M);
10939       } else {
10940         M -= NumElts;
10941         Blend2 &= (M == (Lane + Elt));
10942         assert(Lane <= M && M < (Lane + NumEltsPerLane) && "Out of range mask");
10943         M = M % NumEltsPerLane;
10944         Range2.first = std::min(Range2.first, M);
10945         Range2.second = std::max(Range2.second, M);
10946       }
10947     }
10948   }
10949 
10950   // Bail if we don't need both elements.
10951   // TODO - it might be worth doing this for unary shuffles if the permute
10952   // can be widened.
10953   if (!(0 <= Range1.first && Range1.second < NumEltsPerLane) ||
10954       !(0 <= Range2.first && Range2.second < NumEltsPerLane))
10955     return SDValue();
10956 
10957   if (VT.getSizeInBits() > 128 && (Blend1 || Blend2))
10958     return SDValue();
10959 
10960   // Rotate the 2 ops so we can access both ranges, then permute the result.
10961   auto RotateAndPermute = [&](SDValue Lo, SDValue Hi, int RotAmt, int Ofs) {
10962     MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
10963     SDValue Rotate = DAG.getBitcast(
10964         VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi),
10965                         DAG.getBitcast(ByteVT, Lo),
10966                         DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8)));
10967     SmallVector<int, 64> PermMask(NumElts, SM_SentinelUndef);
10968     for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
10969       for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
10970         int M = Mask[Lane + Elt];
10971         if (M < 0)
10972           continue;
10973         if (M < NumElts)
10974           PermMask[Lane + Elt] = Lane + ((M + Ofs - RotAmt) % NumEltsPerLane);
10975         else
10976           PermMask[Lane + Elt] = Lane + ((M - Ofs - RotAmt) % NumEltsPerLane);
10977       }
10978     }
10979     return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask);
10980   };
10981 
10982   // Check if the ranges are small enough to rotate from either direction.
10983   if (Range2.second < Range1.first)
10984     return RotateAndPermute(V1, V2, Range1.first, 0);
10985   if (Range1.second < Range2.first)
10986     return RotateAndPermute(V2, V1, Range2.first, NumElts);
10987   return SDValue();
10988 }
10989 
10990 static bool isBroadcastShuffleMask(ArrayRef<int> Mask) {
10991   return isUndefOrEqual(Mask, 0);
10992 }
10993 
10994 static bool isNoopOrBroadcastShuffleMask(ArrayRef<int> Mask) {
10995   return isNoopShuffleMask(Mask) || isBroadcastShuffleMask(Mask);
10996 }
10997 
10998 /// Check if the Mask consists of the same element repeated multiple times.
10999 static bool isSingleElementRepeatedMask(ArrayRef<int> Mask) {
11000   size_t NumUndefs = 0;
11001   std::optional<int> UniqueElt;
11002   for (int Elt : Mask) {
11003     if (Elt == SM_SentinelUndef) {
11004       NumUndefs++;
11005       continue;
11006     }
11007     if (UniqueElt.has_value() && UniqueElt.value() != Elt)
11008       return false;
11009     UniqueElt = Elt;
11010   }
11011   // Make sure the element is repeated enough times by checking the number of
11012   // undefs is small.
11013   return NumUndefs <= Mask.size() / 2 && UniqueElt.has_value();
11014 }
11015 
11016 /// Generic routine to decompose a shuffle and blend into independent
11017 /// blends and permutes.
11018 ///
11019 /// This matches the extremely common pattern for handling combined
11020 /// shuffle+blend operations on newer X86 ISAs where we have very fast blend
11021 /// operations. It will try to pick the best arrangement of shuffles and
11022 /// blends. For vXi8/vXi16 shuffles we may use unpack instead of blend.
11023 static SDValue lowerShuffleAsDecomposedShuffleMerge(
11024     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11025     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11026   int NumElts = Mask.size();
11027   int NumLanes = VT.getSizeInBits() / 128;
11028   int NumEltsPerLane = NumElts / NumLanes;
11029 
11030   // Shuffle the input elements into the desired positions in V1 and V2 and
11031   // unpack/blend them together.
11032   bool IsAlternating = true;
11033   SmallVector<int, 32> V1Mask(NumElts, -1);
11034   SmallVector<int, 32> V2Mask(NumElts, -1);
11035   SmallVector<int, 32> FinalMask(NumElts, -1);
11036   for (int i = 0; i < NumElts; ++i) {
11037     int M = Mask[i];
11038     if (M >= 0 && M < NumElts) {
11039       V1Mask[i] = M;
11040       FinalMask[i] = i;
11041       IsAlternating &= (i & 1) == 0;
11042     } else if (M >= NumElts) {
11043       V2Mask[i] = M - NumElts;
11044       FinalMask[i] = i + NumElts;
11045       IsAlternating &= (i & 1) == 1;
11046     }
11047   }
11048 
11049   // If we effectively only demand the 0'th element of \p Input, and not only
11050   // as 0'th element, then broadcast said input,
11051   // and change \p InputMask to be a no-op (identity) mask.
11052   auto canonicalizeBroadcastableInput = [DL, VT, &Subtarget,
11053                                          &DAG](SDValue &Input,
11054                                                MutableArrayRef<int> InputMask) {
11055     unsigned EltSizeInBits = Input.getScalarValueSizeInBits();
11056     if (!Subtarget.hasAVX2() && (!Subtarget.hasAVX() || EltSizeInBits < 32 ||
11057                                  !X86::mayFoldLoad(Input, Subtarget)))
11058       return;
11059     if (isNoopShuffleMask(InputMask))
11060       return;
11061     assert(isBroadcastShuffleMask(InputMask) &&
11062            "Expected to demand only the 0'th element.");
11063     Input = DAG.getNode(X86ISD::VBROADCAST, DL, VT, Input);
11064     for (auto I : enumerate(InputMask)) {
11065       int &InputMaskElt = I.value();
11066       if (InputMaskElt >= 0)
11067         InputMaskElt = I.index();
11068     }
11069   };
11070 
11071   // Currently, we may need to produce one shuffle per input, and blend results.
11072   // It is possible that the shuffle for one of the inputs is already a no-op.
11073   // See if we can simplify non-no-op shuffles into broadcasts,
11074   // which we consider to be strictly better than an arbitrary shuffle.
11075   if (isNoopOrBroadcastShuffleMask(V1Mask) &&
11076       isNoopOrBroadcastShuffleMask(V2Mask)) {
11077     canonicalizeBroadcastableInput(V1, V1Mask);
11078     canonicalizeBroadcastableInput(V2, V2Mask);
11079   }
11080 
11081   // Try to lower with the simpler initial blend/unpack/rotate strategies unless
11082   // one of the input shuffles would be a no-op. We prefer to shuffle inputs as
11083   // the shuffle may be able to fold with a load or other benefit. However, when
11084   // we'll have to do 2x as many shuffles in order to achieve this, a 2-input
11085   // pre-shuffle first is a better strategy.
11086   if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask)) {
11087     // Only prefer immediate blends to unpack/rotate.
11088     if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11089                                                           DAG, true))
11090       return BlendPerm;
11091     // If either input vector provides only a single element which is repeated
11092     // multiple times, unpacking from both input vectors would generate worse
11093     // code. e.g. for
11094     // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2, t4
11095     // it is better to process t4 first to create a vector of t4[0], then unpack
11096     // that vector with t2.
11097     if (!isSingleElementRepeatedMask(V1Mask) &&
11098         !isSingleElementRepeatedMask(V2Mask))
11099       if (SDValue UnpackPerm =
11100               lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG))
11101         return UnpackPerm;
11102     if (SDValue RotatePerm = lowerShuffleAsByteRotateAndPermute(
11103             DL, VT, V1, V2, Mask, Subtarget, DAG))
11104       return RotatePerm;
11105     // Unpack/rotate failed - try again with variable blends.
11106     if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11107                                                           DAG))
11108       return BlendPerm;
11109     if (VT.getScalarSizeInBits() >= 32)
11110       if (SDValue PermUnpack = lowerShuffleAsPermuteAndUnpack(
11111               DL, VT, V1, V2, Mask, Subtarget, DAG))
11112         return PermUnpack;
11113   }
11114 
11115   // If the final mask is an alternating blend of vXi8/vXi16, convert to an
11116   // UNPCKL(SHUFFLE, SHUFFLE) pattern.
11117   // TODO: It doesn't have to be alternating - but each lane mustn't have more
11118   // than half the elements coming from each source.
11119   if (IsAlternating && VT.getScalarSizeInBits() < 32) {
11120     V1Mask.assign(NumElts, -1);
11121     V2Mask.assign(NumElts, -1);
11122     FinalMask.assign(NumElts, -1);
11123     for (int i = 0; i != NumElts; i += NumEltsPerLane)
11124       for (int j = 0; j != NumEltsPerLane; ++j) {
11125         int M = Mask[i + j];
11126         if (M >= 0 && M < NumElts) {
11127           V1Mask[i + (j / 2)] = M;
11128           FinalMask[i + j] = i + (j / 2);
11129         } else if (M >= NumElts) {
11130           V2Mask[i + (j / 2)] = M - NumElts;
11131           FinalMask[i + j] = i + (j / 2) + NumElts;
11132         }
11133       }
11134   }
11135 
11136   V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
11137   V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
11138   return DAG.getVectorShuffle(VT, DL, V1, V2, FinalMask);
11139 }
11140 
11141 static int matchShuffleAsBitRotate(MVT &RotateVT, int EltSizeInBits,
11142                                    const X86Subtarget &Subtarget,
11143                                    ArrayRef<int> Mask) {
11144   assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11145   assert(EltSizeInBits < 64 && "Can't rotate 64-bit integers");
11146 
11147   // AVX512 only has vXi32/vXi64 rotates, so limit the rotation sub group size.
11148   int MinSubElts = Subtarget.hasAVX512() ? std::max(32 / EltSizeInBits, 2) : 2;
11149   int MaxSubElts = 64 / EltSizeInBits;
11150   unsigned RotateAmt, NumSubElts;
11151   if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, MinSubElts,
11152                                           MaxSubElts, NumSubElts, RotateAmt))
11153     return -1;
11154   unsigned NumElts = Mask.size();
11155   MVT RotateSVT = MVT::getIntegerVT(EltSizeInBits * NumSubElts);
11156   RotateVT = MVT::getVectorVT(RotateSVT, NumElts / NumSubElts);
11157   return RotateAmt;
11158 }
11159 
11160 /// Lower shuffle using X86ISD::VROTLI rotations.
11161 static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1,
11162                                        ArrayRef<int> Mask,
11163                                        const X86Subtarget &Subtarget,
11164                                        SelectionDAG &DAG) {
11165   // Only XOP + AVX512 targets have bit rotation instructions.
11166   // If we at least have SSSE3 (PSHUFB) then we shouldn't attempt to use this.
11167   bool IsLegal =
11168       (VT.is128BitVector() && Subtarget.hasXOP()) || Subtarget.hasAVX512();
11169   if (!IsLegal && Subtarget.hasSSE3())
11170     return SDValue();
11171 
11172   MVT RotateVT;
11173   int RotateAmt = matchShuffleAsBitRotate(RotateVT, VT.getScalarSizeInBits(),
11174                                           Subtarget, Mask);
11175   if (RotateAmt < 0)
11176     return SDValue();
11177 
11178   // For pre-SSSE3 targets, if we are shuffling vXi8 elts then ISD::ROTL,
11179   // expanded to OR(SRL,SHL), will be more efficient, but if they can
11180   // widen to vXi16 or more then existing lowering should will be better.
11181   if (!IsLegal) {
11182     if ((RotateAmt % 16) == 0)
11183       return SDValue();
11184     // TODO: Use getTargetVShiftByConstNode.
11185     unsigned ShlAmt = RotateAmt;
11186     unsigned SrlAmt = RotateVT.getScalarSizeInBits() - RotateAmt;
11187     V1 = DAG.getBitcast(RotateVT, V1);
11188     SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1,
11189                               DAG.getTargetConstant(ShlAmt, DL, MVT::i8));
11190     SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1,
11191                               DAG.getTargetConstant(SrlAmt, DL, MVT::i8));
11192     SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL);
11193     return DAG.getBitcast(VT, Rot);
11194   }
11195 
11196   SDValue Rot =
11197       DAG.getNode(X86ISD::VROTLI, DL, RotateVT, DAG.getBitcast(RotateVT, V1),
11198                   DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
11199   return DAG.getBitcast(VT, Rot);
11200 }
11201 
11202 /// Try to match a vector shuffle as an element rotation.
11203 ///
11204 /// This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
11205 static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2,
11206                                        ArrayRef<int> Mask) {
11207   int NumElts = Mask.size();
11208 
11209   // We need to detect various ways of spelling a rotation:
11210   //   [11, 12, 13, 14, 15,  0,  1,  2]
11211   //   [-1, 12, 13, 14, -1, -1,  1, -1]
11212   //   [-1, -1, -1, -1, -1, -1,  1,  2]
11213   //   [ 3,  4,  5,  6,  7,  8,  9, 10]
11214   //   [-1,  4,  5,  6, -1, -1,  9, -1]
11215   //   [-1,  4,  5,  6, -1, -1, -1, -1]
11216   int Rotation = 0;
11217   SDValue Lo, Hi;
11218   for (int i = 0; i < NumElts; ++i) {
11219     int M = Mask[i];
11220     assert((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) &&
11221            "Unexpected mask index.");
11222     if (M < 0)
11223       continue;
11224 
11225     // Determine where a rotated vector would have started.
11226     int StartIdx = i - (M % NumElts);
11227     if (StartIdx == 0)
11228       // The identity rotation isn't interesting, stop.
11229       return -1;
11230 
11231     // If we found the tail of a vector the rotation must be the missing
11232     // front. If we found the head of a vector, it must be how much of the
11233     // head.
11234     int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
11235 
11236     if (Rotation == 0)
11237       Rotation = CandidateRotation;
11238     else if (Rotation != CandidateRotation)
11239       // The rotations don't match, so we can't match this mask.
11240       return -1;
11241 
11242     // Compute which value this mask is pointing at.
11243     SDValue MaskV = M < NumElts ? V1 : V2;
11244 
11245     // Compute which of the two target values this index should be assigned
11246     // to. This reflects whether the high elements are remaining or the low
11247     // elements are remaining.
11248     SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
11249 
11250     // Either set up this value if we've not encountered it before, or check
11251     // that it remains consistent.
11252     if (!TargetV)
11253       TargetV = MaskV;
11254     else if (TargetV != MaskV)
11255       // This may be a rotation, but it pulls from the inputs in some
11256       // unsupported interleaving.
11257       return -1;
11258   }
11259 
11260   // Check that we successfully analyzed the mask, and normalize the results.
11261   assert(Rotation != 0 && "Failed to locate a viable rotation!");
11262   assert((Lo || Hi) && "Failed to find a rotated input vector!");
11263   if (!Lo)
11264     Lo = Hi;
11265   else if (!Hi)
11266     Hi = Lo;
11267 
11268   V1 = Lo;
11269   V2 = Hi;
11270 
11271   return Rotation;
11272 }
11273 
11274 /// Try to lower a vector shuffle as a byte rotation.
11275 ///
11276 /// SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary
11277 /// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use
11278 /// a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will
11279 /// try to generically lower a vector shuffle through such an pattern. It
11280 /// does not check for the profitability of lowering either as PALIGNR or
11281 /// PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form.
11282 /// This matches shuffle vectors that look like:
11283 ///
11284 ///   v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
11285 ///
11286 /// Essentially it concatenates V1 and V2, shifts right by some number of
11287 /// elements, and takes the low elements as the result. Note that while this is
11288 /// specified as a *right shift* because x86 is little-endian, it is a *left
11289 /// rotate* of the vector lanes.
11290 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
11291                                     ArrayRef<int> Mask) {
11292   // Don't accept any shuffles with zero elements.
11293   if (isAnyZero(Mask))
11294     return -1;
11295 
11296   // PALIGNR works on 128-bit lanes.
11297   SmallVector<int, 16> RepeatedMask;
11298   if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask))
11299     return -1;
11300 
11301   int Rotation = matchShuffleAsElementRotate(V1, V2, RepeatedMask);
11302   if (Rotation <= 0)
11303     return -1;
11304 
11305   // PALIGNR rotates bytes, so we need to scale the
11306   // rotation based on how many bytes are in the vector lane.
11307   int NumElts = RepeatedMask.size();
11308   int Scale = 16 / NumElts;
11309   return Rotation * Scale;
11310 }
11311 
11312 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
11313                                         SDValue V2, ArrayRef<int> Mask,
11314                                         const X86Subtarget &Subtarget,
11315                                         SelectionDAG &DAG) {
11316   assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11317 
11318   SDValue Lo = V1, Hi = V2;
11319   int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask);
11320   if (ByteRotation <= 0)
11321     return SDValue();
11322 
11323   // Cast the inputs to i8 vector of correct length to match PALIGNR or
11324   // PSLLDQ/PSRLDQ.
11325   MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
11326   Lo = DAG.getBitcast(ByteVT, Lo);
11327   Hi = DAG.getBitcast(ByteVT, Hi);
11328 
11329   // SSSE3 targets can use the palignr instruction.
11330   if (Subtarget.hasSSSE3()) {
11331     assert((!VT.is512BitVector() || Subtarget.hasBWI()) &&
11332            "512-bit PALIGNR requires BWI instructions");
11333     return DAG.getBitcast(
11334         VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
11335                         DAG.getTargetConstant(ByteRotation, DL, MVT::i8)));
11336   }
11337 
11338   assert(VT.is128BitVector() &&
11339          "Rotate-based lowering only supports 128-bit lowering!");
11340   assert(Mask.size() <= 16 &&
11341          "Can shuffle at most 16 bytes in a 128-bit vector!");
11342   assert(ByteVT == MVT::v16i8 &&
11343          "SSE2 rotate lowering only needed for v16i8!");
11344 
11345   // Default SSE2 implementation
11346   int LoByteShift = 16 - ByteRotation;
11347   int HiByteShift = ByteRotation;
11348 
11349   SDValue LoShift =
11350       DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
11351                   DAG.getTargetConstant(LoByteShift, DL, MVT::i8));
11352   SDValue HiShift =
11353       DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
11354                   DAG.getTargetConstant(HiByteShift, DL, MVT::i8));
11355   return DAG.getBitcast(VT,
11356                         DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift));
11357 }
11358 
11359 /// Try to lower a vector shuffle as a dword/qword rotation.
11360 ///
11361 /// AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary
11362 /// rotation of the concatenation of two vectors; This routine will
11363 /// try to generically lower a vector shuffle through such an pattern.
11364 ///
11365 /// Essentially it concatenates V1 and V2, shifts right by some number of
11366 /// elements, and takes the low elements as the result. Note that while this is
11367 /// specified as a *right shift* because x86 is little-endian, it is a *left
11368 /// rotate* of the vector lanes.
11369 static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
11370                                     SDValue V2, ArrayRef<int> Mask,
11371                                     const APInt &Zeroable,
11372                                     const X86Subtarget &Subtarget,
11373                                     SelectionDAG &DAG) {
11374   assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
11375          "Only 32-bit and 64-bit elements are supported!");
11376 
11377   // 128/256-bit vectors are only supported with VLX.
11378   assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector()))
11379          && "VLX required for 128/256-bit vectors");
11380 
11381   SDValue Lo = V1, Hi = V2;
11382   int Rotation = matchShuffleAsElementRotate(Lo, Hi, Mask);
11383   if (0 < Rotation)
11384     return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
11385                        DAG.getTargetConstant(Rotation, DL, MVT::i8));
11386 
11387   // See if we can use VALIGN as a cross-lane version of VSHLDQ/VSRLDQ.
11388   // TODO: Pull this out as a matchShuffleAsElementShift helper?
11389   // TODO: We can probably make this more aggressive and use shift-pairs like
11390   // lowerShuffleAsByteShiftMask.
11391   unsigned NumElts = Mask.size();
11392   unsigned ZeroLo = Zeroable.countr_one();
11393   unsigned ZeroHi = Zeroable.countl_one();
11394   assert((ZeroLo + ZeroHi) < NumElts && "Zeroable shuffle detected");
11395   if (!ZeroLo && !ZeroHi)
11396     return SDValue();
11397 
11398   if (ZeroLo) {
11399     SDValue Src = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11400     int Low = Mask[ZeroLo] < (int)NumElts ? 0 : NumElts;
11401     if (isSequentialOrUndefInRange(Mask, ZeroLo, NumElts - ZeroLo, Low))
11402       return DAG.getNode(X86ISD::VALIGN, DL, VT, Src,
11403                          getZeroVector(VT, Subtarget, DAG, DL),
11404                          DAG.getTargetConstant(NumElts - ZeroLo, DL, MVT::i8));
11405   }
11406 
11407   if (ZeroHi) {
11408     SDValue Src = Mask[0] < (int)NumElts ? V1 : V2;
11409     int Low = Mask[0] < (int)NumElts ? 0 : NumElts;
11410     if (isSequentialOrUndefInRange(Mask, 0, NumElts - ZeroHi, Low + ZeroHi))
11411       return DAG.getNode(X86ISD::VALIGN, DL, VT,
11412                          getZeroVector(VT, Subtarget, DAG, DL), Src,
11413                          DAG.getTargetConstant(ZeroHi, DL, MVT::i8));
11414   }
11415 
11416   return SDValue();
11417 }
11418 
11419 /// Try to lower a vector shuffle as a byte shift sequence.
11420 static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1,
11421                                            SDValue V2, ArrayRef<int> Mask,
11422                                            const APInt &Zeroable,
11423                                            const X86Subtarget &Subtarget,
11424                                            SelectionDAG &DAG) {
11425   assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11426   assert(VT.is128BitVector() && "Only 128-bit vectors supported");
11427 
11428   // We need a shuffle that has zeros at one/both ends and a sequential
11429   // shuffle from one source within.
11430   unsigned ZeroLo = Zeroable.countr_one();
11431   unsigned ZeroHi = Zeroable.countl_one();
11432   if (!ZeroLo && !ZeroHi)
11433     return SDValue();
11434 
11435   unsigned NumElts = Mask.size();
11436   unsigned Len = NumElts - (ZeroLo + ZeroHi);
11437   if (!isSequentialOrUndefInRange(Mask, ZeroLo, Len, Mask[ZeroLo]))
11438     return SDValue();
11439 
11440   unsigned Scale = VT.getScalarSizeInBits() / 8;
11441   ArrayRef<int> StubMask = Mask.slice(ZeroLo, Len);
11442   if (!isUndefOrInRange(StubMask, 0, NumElts) &&
11443       !isUndefOrInRange(StubMask, NumElts, 2 * NumElts))
11444     return SDValue();
11445 
11446   SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11447   Res = DAG.getBitcast(MVT::v16i8, Res);
11448 
11449   // Use VSHLDQ/VSRLDQ ops to zero the ends of a vector and leave an
11450   // inner sequential set of elements, possibly offset:
11451   // 01234567 --> zzzzzz01 --> 1zzzzzzz
11452   // 01234567 --> 4567zzzz --> zzzzz456
11453   // 01234567 --> z0123456 --> 3456zzzz --> zz3456zz
11454   if (ZeroLo == 0) {
11455     unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
11456     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11457                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11458     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11459                       DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8));
11460   } else if (ZeroHi == 0) {
11461     unsigned Shift = Mask[ZeroLo] % NumElts;
11462     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11463                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11464     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11465                       DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11466   } else if (!Subtarget.hasSSSE3()) {
11467     // If we don't have PSHUFB then its worth avoiding an AND constant mask
11468     // by performing 3 byte shifts. Shuffle combining can kick in above that.
11469     // TODO: There may be some cases where VSH{LR}DQ+PAND is still better.
11470     unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
11471     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11472                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11473     Shift += Mask[ZeroLo] % NumElts;
11474     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11475                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11476     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11477                       DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11478   } else
11479     return SDValue();
11480 
11481   return DAG.getBitcast(VT, Res);
11482 }
11483 
11484 /// Try to lower a vector shuffle as a bit shift (shifts in zeros).
11485 ///
11486 /// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and
11487 /// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function
11488 /// matches elements from one of the input vectors shuffled to the left or
11489 /// right with zeroable elements 'shifted in'. It handles both the strictly
11490 /// bit-wise element shifts and the byte shift across an entire 128-bit double
11491 /// quad word lane.
11492 ///
11493 /// PSHL : (little-endian) left bit shift.
11494 /// [ zz, 0, zz,  2 ]
11495 /// [ -1, 4, zz, -1 ]
11496 /// PSRL : (little-endian) right bit shift.
11497 /// [  1, zz,  3, zz]
11498 /// [ -1, -1,  7, zz]
11499 /// PSLLDQ : (little-endian) left byte shift
11500 /// [ zz,  0,  1,  2,  3,  4,  5,  6]
11501 /// [ zz, zz, -1, -1,  2,  3,  4, -1]
11502 /// [ zz, zz, zz, zz, zz, zz, -1,  1]
11503 /// PSRLDQ : (little-endian) right byte shift
11504 /// [  5, 6,  7, zz, zz, zz, zz, zz]
11505 /// [ -1, 5,  6,  7, zz, zz, zz, zz]
11506 /// [  1, 2, -1, -1, -1, -1, zz, zz]
11507 static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
11508                                unsigned ScalarSizeInBits, ArrayRef<int> Mask,
11509                                int MaskOffset, const APInt &Zeroable,
11510                                const X86Subtarget &Subtarget) {
11511   int Size = Mask.size();
11512   unsigned SizeInBits = Size * ScalarSizeInBits;
11513 
11514   auto CheckZeros = [&](int Shift, int Scale, bool Left) {
11515     for (int i = 0; i < Size; i += Scale)
11516       for (int j = 0; j < Shift; ++j)
11517         if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
11518           return false;
11519 
11520     return true;
11521   };
11522 
11523   auto MatchShift = [&](int Shift, int Scale, bool Left) {
11524     for (int i = 0; i != Size; i += Scale) {
11525       unsigned Pos = Left ? i + Shift : i;
11526       unsigned Low = Left ? i : i + Shift;
11527       unsigned Len = Scale - Shift;
11528       if (!isSequentialOrUndefInRange(Mask, Pos, Len, Low + MaskOffset))
11529         return -1;
11530     }
11531 
11532     int ShiftEltBits = ScalarSizeInBits * Scale;
11533     bool ByteShift = ShiftEltBits > 64;
11534     Opcode = Left ? (ByteShift ? X86ISD::VSHLDQ : X86ISD::VSHLI)
11535                   : (ByteShift ? X86ISD::VSRLDQ : X86ISD::VSRLI);
11536     int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
11537 
11538     // Normalize the scale for byte shifts to still produce an i64 element
11539     // type.
11540     Scale = ByteShift ? Scale / 2 : Scale;
11541 
11542     // We need to round trip through the appropriate type for the shift.
11543     MVT ShiftSVT = MVT::getIntegerVT(ScalarSizeInBits * Scale);
11544     ShiftVT = ByteShift ? MVT::getVectorVT(MVT::i8, SizeInBits / 8)
11545                         : MVT::getVectorVT(ShiftSVT, Size / Scale);
11546     return (int)ShiftAmt;
11547   };
11548 
11549   // SSE/AVX supports logical shifts up to 64-bit integers - so we can just
11550   // keep doubling the size of the integer elements up to that. We can
11551   // then shift the elements of the integer vector by whole multiples of
11552   // their width within the elements of the larger integer vector. Test each
11553   // multiple to see if we can find a match with the moved element indices
11554   // and that the shifted in elements are all zeroable.
11555   unsigned MaxWidth = ((SizeInBits == 512) && !Subtarget.hasBWI() ? 64 : 128);
11556   for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
11557     for (int Shift = 1; Shift != Scale; ++Shift)
11558       for (bool Left : {true, false})
11559         if (CheckZeros(Shift, Scale, Left)) {
11560           int ShiftAmt = MatchShift(Shift, Scale, Left);
11561           if (0 < ShiftAmt)
11562             return ShiftAmt;
11563         }
11564 
11565   // no match
11566   return -1;
11567 }
11568 
11569 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
11570                                    SDValue V2, ArrayRef<int> Mask,
11571                                    const APInt &Zeroable,
11572                                    const X86Subtarget &Subtarget,
11573                                    SelectionDAG &DAG, bool BitwiseOnly) {
11574   int Size = Mask.size();
11575   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11576 
11577   MVT ShiftVT;
11578   SDValue V = V1;
11579   unsigned Opcode;
11580 
11581   // Try to match shuffle against V1 shift.
11582   int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
11583                                      Mask, 0, Zeroable, Subtarget);
11584 
11585   // If V1 failed, try to match shuffle against V2 shift.
11586   if (ShiftAmt < 0) {
11587     ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
11588                                    Mask, Size, Zeroable, Subtarget);
11589     V = V2;
11590   }
11591 
11592   if (ShiftAmt < 0)
11593     return SDValue();
11594 
11595   if (BitwiseOnly && (Opcode == X86ISD::VSHLDQ || Opcode == X86ISD::VSRLDQ))
11596     return SDValue();
11597 
11598   assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
11599          "Illegal integer vector type");
11600   V = DAG.getBitcast(ShiftVT, V);
11601   V = DAG.getNode(Opcode, DL, ShiftVT, V,
11602                   DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
11603   return DAG.getBitcast(VT, V);
11604 }
11605 
11606 // EXTRQ: Extract Len elements from lower half of source, starting at Idx.
11607 // Remainder of lower half result is zero and upper half is all undef.
11608 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
11609                                 ArrayRef<int> Mask, uint64_t &BitLen,
11610                                 uint64_t &BitIdx, const APInt &Zeroable) {
11611   int Size = Mask.size();
11612   int HalfSize = Size / 2;
11613   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11614   assert(!Zeroable.isAllOnes() && "Fully zeroable shuffle mask");
11615 
11616   // Upper half must be undefined.
11617   if (!isUndefUpperHalf(Mask))
11618     return false;
11619 
11620   // Determine the extraction length from the part of the
11621   // lower half that isn't zeroable.
11622   int Len = HalfSize;
11623   for (; Len > 0; --Len)
11624     if (!Zeroable[Len - 1])
11625       break;
11626   assert(Len > 0 && "Zeroable shuffle mask");
11627 
11628   // Attempt to match first Len sequential elements from the lower half.
11629   SDValue Src;
11630   int Idx = -1;
11631   for (int i = 0; i != Len; ++i) {
11632     int M = Mask[i];
11633     if (M == SM_SentinelUndef)
11634       continue;
11635     SDValue &V = (M < Size ? V1 : V2);
11636     M = M % Size;
11637 
11638     // The extracted elements must start at a valid index and all mask
11639     // elements must be in the lower half.
11640     if (i > M || M >= HalfSize)
11641       return false;
11642 
11643     if (Idx < 0 || (Src == V && Idx == (M - i))) {
11644       Src = V;
11645       Idx = M - i;
11646       continue;
11647     }
11648     return false;
11649   }
11650 
11651   if (!Src || Idx < 0)
11652     return false;
11653 
11654   assert((Idx + Len) <= HalfSize && "Illegal extraction mask");
11655   BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
11656   BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
11657   V1 = Src;
11658   return true;
11659 }
11660 
11661 // INSERTQ: Extract lowest Len elements from lower half of second source and
11662 // insert over first source, starting at Idx.
11663 // { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... }
11664 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
11665                                   ArrayRef<int> Mask, uint64_t &BitLen,
11666                                   uint64_t &BitIdx) {
11667   int Size = Mask.size();
11668   int HalfSize = Size / 2;
11669   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11670 
11671   // Upper half must be undefined.
11672   if (!isUndefUpperHalf(Mask))
11673     return false;
11674 
11675   for (int Idx = 0; Idx != HalfSize; ++Idx) {
11676     SDValue Base;
11677 
11678     // Attempt to match first source from mask before insertion point.
11679     if (isUndefInRange(Mask, 0, Idx)) {
11680       /* EMPTY */
11681     } else if (isSequentialOrUndefInRange(Mask, 0, Idx, 0)) {
11682       Base = V1;
11683     } else if (isSequentialOrUndefInRange(Mask, 0, Idx, Size)) {
11684       Base = V2;
11685     } else {
11686       continue;
11687     }
11688 
11689     // Extend the extraction length looking to match both the insertion of
11690     // the second source and the remaining elements of the first.
11691     for (int Hi = Idx + 1; Hi <= HalfSize; ++Hi) {
11692       SDValue Insert;
11693       int Len = Hi - Idx;
11694 
11695       // Match insertion.
11696       if (isSequentialOrUndefInRange(Mask, Idx, Len, 0)) {
11697         Insert = V1;
11698       } else if (isSequentialOrUndefInRange(Mask, Idx, Len, Size)) {
11699         Insert = V2;
11700       } else {
11701         continue;
11702       }
11703 
11704       // Match the remaining elements of the lower half.
11705       if (isUndefInRange(Mask, Hi, HalfSize - Hi)) {
11706         /* EMPTY */
11707       } else if ((!Base || (Base == V1)) &&
11708                  isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, Hi)) {
11709         Base = V1;
11710       } else if ((!Base || (Base == V2)) &&
11711                  isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi,
11712                                             Size + Hi)) {
11713         Base = V2;
11714       } else {
11715         continue;
11716       }
11717 
11718       BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
11719       BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
11720       V1 = Base;
11721       V2 = Insert;
11722       return true;
11723     }
11724   }
11725 
11726   return false;
11727 }
11728 
11729 /// Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
11730 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
11731                                      SDValue V2, ArrayRef<int> Mask,
11732                                      const APInt &Zeroable, SelectionDAG &DAG) {
11733   uint64_t BitLen, BitIdx;
11734   if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
11735     return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
11736                        DAG.getTargetConstant(BitLen, DL, MVT::i8),
11737                        DAG.getTargetConstant(BitIdx, DL, MVT::i8));
11738 
11739   if (matchShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx))
11740     return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
11741                        V2 ? V2 : DAG.getUNDEF(VT),
11742                        DAG.getTargetConstant(BitLen, DL, MVT::i8),
11743                        DAG.getTargetConstant(BitIdx, DL, MVT::i8));
11744 
11745   return SDValue();
11746 }
11747 
11748 /// Lower a vector shuffle as a zero or any extension.
11749 ///
11750 /// Given a specific number of elements, element bit width, and extension
11751 /// stride, produce either a zero or any extension based on the available
11752 /// features of the subtarget. The extended elements are consecutive and
11753 /// begin and can start from an offsetted element index in the input; to
11754 /// avoid excess shuffling the offset must either being in the bottom lane
11755 /// or at the start of a higher lane. All extended elements must be from
11756 /// the same lane.
11757 static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
11758     const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
11759     ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11760   assert(Scale > 1 && "Need a scale to extend.");
11761   int EltBits = VT.getScalarSizeInBits();
11762   int NumElements = VT.getVectorNumElements();
11763   int NumEltsPerLane = 128 / EltBits;
11764   int OffsetLane = Offset / NumEltsPerLane;
11765   assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
11766          "Only 8, 16, and 32 bit elements can be extended.");
11767   assert(Scale * EltBits <= 64 && "Cannot zero extend past 64 bits.");
11768   assert(0 <= Offset && "Extension offset must be positive.");
11769   assert((Offset < NumEltsPerLane || Offset % NumEltsPerLane == 0) &&
11770          "Extension offset must be in the first lane or start an upper lane.");
11771 
11772   // Check that an index is in same lane as the base offset.
11773   auto SafeOffset = [&](int Idx) {
11774     return OffsetLane == (Idx / NumEltsPerLane);
11775   };
11776 
11777   // Shift along an input so that the offset base moves to the first element.
11778   auto ShuffleOffset = [&](SDValue V) {
11779     if (!Offset)
11780       return V;
11781 
11782     SmallVector<int, 8> ShMask((unsigned)NumElements, -1);
11783     for (int i = 0; i * Scale < NumElements; ++i) {
11784       int SrcIdx = i + Offset;
11785       ShMask[i] = SafeOffset(SrcIdx) ? SrcIdx : -1;
11786     }
11787     return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask);
11788   };
11789 
11790   // Found a valid a/zext mask! Try various lowering strategies based on the
11791   // input type and available ISA extensions.
11792   if (Subtarget.hasSSE41()) {
11793     // Not worth offsetting 128-bit vectors if scale == 2, a pattern using
11794     // PUNPCK will catch this in a later shuffle match.
11795     if (Offset && Scale == 2 && VT.is128BitVector())
11796       return SDValue();
11797     MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale),
11798                                  NumElements / Scale);
11799     InputV = DAG.getBitcast(VT, InputV);
11800     InputV = ShuffleOffset(InputV);
11801     InputV = getEXTEND_VECTOR_INREG(AnyExt ? ISD::ANY_EXTEND : ISD::ZERO_EXTEND,
11802                                     DL, ExtVT, InputV, DAG);
11803     return DAG.getBitcast(VT, InputV);
11804   }
11805 
11806   assert(VT.is128BitVector() && "Only 128-bit vectors can be extended.");
11807   InputV = DAG.getBitcast(VT, InputV);
11808 
11809   // For any extends we can cheat for larger element sizes and use shuffle
11810   // instructions that can fold with a load and/or copy.
11811   if (AnyExt && EltBits == 32) {
11812     int PSHUFDMask[4] = {Offset, -1, SafeOffset(Offset + 1) ? Offset + 1 : -1,
11813                          -1};
11814     return DAG.getBitcast(
11815         VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
11816                         DAG.getBitcast(MVT::v4i32, InputV),
11817                         getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
11818   }
11819   if (AnyExt && EltBits == 16 && Scale > 2) {
11820     int PSHUFDMask[4] = {Offset / 2, -1,
11821                          SafeOffset(Offset + 1) ? (Offset + 1) / 2 : -1, -1};
11822     InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
11823                          DAG.getBitcast(MVT::v4i32, InputV),
11824                          getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
11825     int PSHUFWMask[4] = {1, -1, -1, -1};
11826     unsigned OddEvenOp = (Offset & 1) ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
11827     return DAG.getBitcast(
11828         VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16,
11829                         DAG.getBitcast(MVT::v8i16, InputV),
11830                         getV4X86ShuffleImm8ForMask(PSHUFWMask, DL, DAG)));
11831   }
11832 
11833   // The SSE4A EXTRQ instruction can efficiently extend the first 2 lanes
11834   // to 64-bits.
11835   if ((Scale * EltBits) == 64 && EltBits < 32 && Subtarget.hasSSE4A()) {
11836     assert(NumElements == (int)Mask.size() && "Unexpected shuffle mask size!");
11837     assert(VT.is128BitVector() && "Unexpected vector width!");
11838 
11839     int LoIdx = Offset * EltBits;
11840     SDValue Lo = DAG.getBitcast(
11841         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
11842                                 DAG.getTargetConstant(EltBits, DL, MVT::i8),
11843                                 DAG.getTargetConstant(LoIdx, DL, MVT::i8)));
11844 
11845     if (isUndefUpperHalf(Mask) || !SafeOffset(Offset + 1))
11846       return DAG.getBitcast(VT, Lo);
11847 
11848     int HiIdx = (Offset + 1) * EltBits;
11849     SDValue Hi = DAG.getBitcast(
11850         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
11851                                 DAG.getTargetConstant(EltBits, DL, MVT::i8),
11852                                 DAG.getTargetConstant(HiIdx, DL, MVT::i8)));
11853     return DAG.getBitcast(VT,
11854                           DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
11855   }
11856 
11857   // If this would require more than 2 unpack instructions to expand, use
11858   // pshufb when available. We can only use more than 2 unpack instructions
11859   // when zero extending i8 elements which also makes it easier to use pshufb.
11860   if (Scale > 4 && EltBits == 8 && Subtarget.hasSSSE3()) {
11861     assert(NumElements == 16 && "Unexpected byte vector width!");
11862     SDValue PSHUFBMask[16];
11863     for (int i = 0; i < 16; ++i) {
11864       int Idx = Offset + (i / Scale);
11865       if ((i % Scale == 0 && SafeOffset(Idx))) {
11866         PSHUFBMask[i] = DAG.getConstant(Idx, DL, MVT::i8);
11867         continue;
11868       }
11869       PSHUFBMask[i] =
11870           AnyExt ? DAG.getUNDEF(MVT::i8) : DAG.getConstant(0x80, DL, MVT::i8);
11871     }
11872     InputV = DAG.getBitcast(MVT::v16i8, InputV);
11873     return DAG.getBitcast(
11874         VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
11875                         DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask)));
11876   }
11877 
11878   // If we are extending from an offset, ensure we start on a boundary that
11879   // we can unpack from.
11880   int AlignToUnpack = Offset % (NumElements / Scale);
11881   if (AlignToUnpack) {
11882     SmallVector<int, 8> ShMask((unsigned)NumElements, -1);
11883     for (int i = AlignToUnpack; i < NumElements; ++i)
11884       ShMask[i - AlignToUnpack] = i;
11885     InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask);
11886     Offset -= AlignToUnpack;
11887   }
11888 
11889   // Otherwise emit a sequence of unpacks.
11890   do {
11891     unsigned UnpackLoHi = X86ISD::UNPCKL;
11892     if (Offset >= (NumElements / 2)) {
11893       UnpackLoHi = X86ISD::UNPCKH;
11894       Offset -= (NumElements / 2);
11895     }
11896 
11897     MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);
11898     SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
11899                          : getZeroVector(InputVT, Subtarget, DAG, DL);
11900     InputV = DAG.getBitcast(InputVT, InputV);
11901     InputV = DAG.getNode(UnpackLoHi, DL, InputVT, InputV, Ext);
11902     Scale /= 2;
11903     EltBits *= 2;
11904     NumElements /= 2;
11905   } while (Scale > 1);
11906   return DAG.getBitcast(VT, InputV);
11907 }
11908 
11909 /// Try to lower a vector shuffle as a zero extension on any microarch.
11910 ///
11911 /// This routine will try to do everything in its power to cleverly lower
11912 /// a shuffle which happens to match the pattern of a zero extend. It doesn't
11913 /// check for the profitability of this lowering,  it tries to aggressively
11914 /// match this pattern. It will use all of the micro-architectural details it
11915 /// can to emit an efficient lowering. It handles both blends with all-zero
11916 /// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to
11917 /// masking out later).
11918 ///
11919 /// The reason we have dedicated lowering for zext-style shuffles is that they
11920 /// are both incredibly common and often quite performance sensitive.
11921 static SDValue lowerShuffleAsZeroOrAnyExtend(
11922     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11923     const APInt &Zeroable, const X86Subtarget &Subtarget,
11924     SelectionDAG &DAG) {
11925   int Bits = VT.getSizeInBits();
11926   int NumLanes = Bits / 128;
11927   int NumElements = VT.getVectorNumElements();
11928   int NumEltsPerLane = NumElements / NumLanes;
11929   assert(VT.getScalarSizeInBits() <= 32 &&
11930          "Exceeds 32-bit integer zero extension limit");
11931   assert((int)Mask.size() == NumElements && "Unexpected shuffle mask size");
11932 
11933   // Define a helper function to check a particular ext-scale and lower to it if
11934   // valid.
11935   auto Lower = [&](int Scale) -> SDValue {
11936     SDValue InputV;
11937     bool AnyExt = true;
11938     int Offset = 0;
11939     int Matches = 0;
11940     for (int i = 0; i < NumElements; ++i) {
11941       int M = Mask[i];
11942       if (M < 0)
11943         continue; // Valid anywhere but doesn't tell us anything.
11944       if (i % Scale != 0) {
11945         // Each of the extended elements need to be zeroable.
11946         if (!Zeroable[i])
11947           return SDValue();
11948 
11949         // We no longer are in the anyext case.
11950         AnyExt = false;
11951         continue;
11952       }
11953 
11954       // Each of the base elements needs to be consecutive indices into the
11955       // same input vector.
11956       SDValue V = M < NumElements ? V1 : V2;
11957       M = M % NumElements;
11958       if (!InputV) {
11959         InputV = V;
11960         Offset = M - (i / Scale);
11961       } else if (InputV != V)
11962         return SDValue(); // Flip-flopping inputs.
11963 
11964       // Offset must start in the lowest 128-bit lane or at the start of an
11965       // upper lane.
11966       // FIXME: Is it ever worth allowing a negative base offset?
11967       if (!((0 <= Offset && Offset < NumEltsPerLane) ||
11968             (Offset % NumEltsPerLane) == 0))
11969         return SDValue();
11970 
11971       // If we are offsetting, all referenced entries must come from the same
11972       // lane.
11973       if (Offset && (Offset / NumEltsPerLane) != (M / NumEltsPerLane))
11974         return SDValue();
11975 
11976       if ((M % NumElements) != (Offset + (i / Scale)))
11977         return SDValue(); // Non-consecutive strided elements.
11978       Matches++;
11979     }
11980 
11981     // If we fail to find an input, we have a zero-shuffle which should always
11982     // have already been handled.
11983     // FIXME: Maybe handle this here in case during blending we end up with one?
11984     if (!InputV)
11985       return SDValue();
11986 
11987     // If we are offsetting, don't extend if we only match a single input, we
11988     // can always do better by using a basic PSHUF or PUNPCK.
11989     if (Offset != 0 && Matches < 2)
11990       return SDValue();
11991 
11992     return lowerShuffleAsSpecificZeroOrAnyExtend(DL, VT, Scale, Offset, AnyExt,
11993                                                  InputV, Mask, Subtarget, DAG);
11994   };
11995 
11996   // The widest scale possible for extending is to a 64-bit integer.
11997   assert(Bits % 64 == 0 &&
11998          "The number of bits in a vector must be divisible by 64 on x86!");
11999   int NumExtElements = Bits / 64;
12000 
12001   // Each iteration, try extending the elements half as much, but into twice as
12002   // many elements.
12003   for (; NumExtElements < NumElements; NumExtElements *= 2) {
12004     assert(NumElements % NumExtElements == 0 &&
12005            "The input vector size must be divisible by the extended size.");
12006     if (SDValue V = Lower(NumElements / NumExtElements))
12007       return V;
12008   }
12009 
12010   // General extends failed, but 128-bit vectors may be able to use MOVQ.
12011   if (Bits != 128)
12012     return SDValue();
12013 
12014   // Returns one of the source operands if the shuffle can be reduced to a
12015   // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits.
12016   auto CanZExtLowHalf = [&]() {
12017     for (int i = NumElements / 2; i != NumElements; ++i)
12018       if (!Zeroable[i])
12019         return SDValue();
12020     if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, 0))
12021       return V1;
12022     if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, NumElements))
12023       return V2;
12024     return SDValue();
12025   };
12026 
12027   if (SDValue V = CanZExtLowHalf()) {
12028     V = DAG.getBitcast(MVT::v2i64, V);
12029     V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
12030     return DAG.getBitcast(VT, V);
12031   }
12032 
12033   // No viable ext lowering found.
12034   return SDValue();
12035 }
12036 
12037 /// Try to get a scalar value for a specific element of a vector.
12038 ///
12039 /// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
12040 static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
12041                                               SelectionDAG &DAG) {
12042   MVT VT = V.getSimpleValueType();
12043   MVT EltVT = VT.getVectorElementType();
12044   V = peekThroughBitcasts(V);
12045 
12046   // If the bitcasts shift the element size, we can't extract an equivalent
12047   // element from it.
12048   MVT NewVT = V.getSimpleValueType();
12049   if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
12050     return SDValue();
12051 
12052   if (V.getOpcode() == ISD::BUILD_VECTOR ||
12053       (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR)) {
12054     // Ensure the scalar operand is the same size as the destination.
12055     // FIXME: Add support for scalar truncation where possible.
12056     SDValue S = V.getOperand(Idx);
12057     if (EltVT.getSizeInBits() == S.getSimpleValueType().getSizeInBits())
12058       return DAG.getBitcast(EltVT, S);
12059   }
12060 
12061   return SDValue();
12062 }
12063 
12064 /// Helper to test for a load that can be folded with x86 shuffles.
12065 ///
12066 /// This is particularly important because the set of instructions varies
12067 /// significantly based on whether the operand is a load or not.
12068 static bool isShuffleFoldableLoad(SDValue V) {
12069   return V->hasOneUse() &&
12070          ISD::isNON_EXTLoad(peekThroughOneUseBitcasts(V).getNode());
12071 }
12072 
12073 template<typename T>
12074 static bool isSoftF16(T VT, const X86Subtarget &Subtarget) {
12075   T EltVT = VT.getScalarType();
12076   return EltVT == MVT::bf16 || (EltVT == MVT::f16 && !Subtarget.hasFP16());
12077 }
12078 
12079 /// Try to lower insertion of a single element into a zero vector.
12080 ///
12081 /// This is a common pattern that we have especially efficient patterns to lower
12082 /// across all subtarget feature sets.
12083 static SDValue lowerShuffleAsElementInsertion(
12084     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12085     const APInt &Zeroable, const X86Subtarget &Subtarget,
12086     SelectionDAG &DAG) {
12087   MVT ExtVT = VT;
12088   MVT EltVT = VT.getVectorElementType();
12089   unsigned NumElts = VT.getVectorNumElements();
12090   unsigned EltBits = VT.getScalarSizeInBits();
12091 
12092   if (isSoftF16(EltVT, Subtarget))
12093     return SDValue();
12094 
12095   int V2Index =
12096       find_if(Mask, [&Mask](int M) { return M >= (int)Mask.size(); }) -
12097       Mask.begin();
12098   bool IsV1Constant = getTargetConstantFromNode(V1) != nullptr;
12099   bool IsV1Zeroable = true;
12100   for (int i = 0, Size = Mask.size(); i < Size; ++i)
12101     if (i != V2Index && !Zeroable[i]) {
12102       IsV1Zeroable = false;
12103       break;
12104     }
12105 
12106   // Bail if a non-zero V1 isn't used in place.
12107   if (!IsV1Zeroable) {
12108     SmallVector<int, 8> V1Mask(Mask);
12109     V1Mask[V2Index] = -1;
12110     if (!isNoopShuffleMask(V1Mask))
12111       return SDValue();
12112   }
12113 
12114   // Check for a single input from a SCALAR_TO_VECTOR node.
12115   // FIXME: All of this should be canonicalized into INSERT_VECTOR_ELT and
12116   // all the smarts here sunk into that routine. However, the current
12117   // lowering of BUILD_VECTOR makes that nearly impossible until the old
12118   // vector shuffle lowering is dead.
12119   SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
12120                                                DAG);
12121   if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) {
12122     // We need to zext the scalar if it is smaller than an i32.
12123     V2S = DAG.getBitcast(EltVT, V2S);
12124     if (EltVT == MVT::i8 || (EltVT == MVT::i16 && !Subtarget.hasFP16())) {
12125       // Using zext to expand a narrow element won't work for non-zero
12126       // insertions. But we can use a masked constant vector if we're
12127       // inserting V2 into the bottom of V1.
12128       if (!IsV1Zeroable && !(IsV1Constant && V2Index == 0))
12129         return SDValue();
12130 
12131       // Zero-extend directly to i32.
12132       ExtVT = MVT::getVectorVT(MVT::i32, ExtVT.getSizeInBits() / 32);
12133       V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);
12134 
12135       // If we're inserting into a constant, mask off the inserted index
12136       // and OR with the zero-extended scalar.
12137       if (!IsV1Zeroable) {
12138         SmallVector<APInt> Bits(NumElts, APInt::getAllOnes(EltBits));
12139         Bits[V2Index] = APInt::getZero(EltBits);
12140         SDValue BitMask = getConstVector(Bits, VT, DAG, DL);
12141         V1 = DAG.getNode(ISD::AND, DL, VT, V1, BitMask);
12142         V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12143         V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2));
12144         return DAG.getNode(ISD::OR, DL, VT, V1, V2);
12145       }
12146     }
12147     V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12148   } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 ||
12149              EltVT == MVT::i16) {
12150     // Either not inserting from the low element of the input or the input
12151     // element size is too small to use VZEXT_MOVL to clear the high bits.
12152     return SDValue();
12153   }
12154 
12155   if (!IsV1Zeroable) {
12156     // If V1 can't be treated as a zero vector we have fewer options to lower
12157     // this. We can't support integer vectors or non-zero targets cheaply.
12158     assert(VT == ExtVT && "Cannot change extended type when non-zeroable!");
12159     if (!VT.isFloatingPoint() || V2Index != 0)
12160       return SDValue();
12161     if (!VT.is128BitVector())
12162       return SDValue();
12163 
12164     // Otherwise, use MOVSD, MOVSS or MOVSH.
12165     unsigned MovOpc = 0;
12166     if (EltVT == MVT::f16)
12167       MovOpc = X86ISD::MOVSH;
12168     else if (EltVT == MVT::f32)
12169       MovOpc = X86ISD::MOVSS;
12170     else if (EltVT == MVT::f64)
12171       MovOpc = X86ISD::MOVSD;
12172     else
12173       llvm_unreachable("Unsupported floating point element type to handle!");
12174     return DAG.getNode(MovOpc, DL, ExtVT, V1, V2);
12175   }
12176 
12177   // This lowering only works for the low element with floating point vectors.
12178   if (VT.isFloatingPoint() && V2Index != 0)
12179     return SDValue();
12180 
12181   V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
12182   if (ExtVT != VT)
12183     V2 = DAG.getBitcast(VT, V2);
12184 
12185   if (V2Index != 0) {
12186     // If we have 4 or fewer lanes we can cheaply shuffle the element into
12187     // the desired position. Otherwise it is more efficient to do a vector
12188     // shift left. We know that we can do a vector shift left because all
12189     // the inputs are zero.
12190     if (VT.isFloatingPoint() || NumElts <= 4) {
12191       SmallVector<int, 4> V2Shuffle(Mask.size(), 1);
12192       V2Shuffle[V2Index] = 0;
12193       V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
12194     } else {
12195       V2 = DAG.getBitcast(MVT::v16i8, V2);
12196       V2 = DAG.getNode(
12197           X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
12198           DAG.getTargetConstant(V2Index * EltBits / 8, DL, MVT::i8));
12199       V2 = DAG.getBitcast(VT, V2);
12200     }
12201   }
12202   return V2;
12203 }
12204 
12205 /// Try to lower broadcast of a single - truncated - integer element,
12206 /// coming from a scalar_to_vector/build_vector node \p V0 with larger elements.
12207 ///
12208 /// This assumes we have AVX2.
12209 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
12210                                             int BroadcastIdx,
12211                                             const X86Subtarget &Subtarget,
12212                                             SelectionDAG &DAG) {
12213   assert(Subtarget.hasAVX2() &&
12214          "We can only lower integer broadcasts with AVX2!");
12215 
12216   MVT EltVT = VT.getVectorElementType();
12217   MVT V0VT = V0.getSimpleValueType();
12218 
12219   assert(VT.isInteger() && "Unexpected non-integer trunc broadcast!");
12220   assert(V0VT.isVector() && "Unexpected non-vector vector-sized value!");
12221 
12222   MVT V0EltVT = V0VT.getVectorElementType();
12223   if (!V0EltVT.isInteger())
12224     return SDValue();
12225 
12226   const unsigned EltSize = EltVT.getSizeInBits();
12227   const unsigned V0EltSize = V0EltVT.getSizeInBits();
12228 
12229   // This is only a truncation if the original element type is larger.
12230   if (V0EltSize <= EltSize)
12231     return SDValue();
12232 
12233   assert(((V0EltSize % EltSize) == 0) &&
12234          "Scalar type sizes must all be powers of 2 on x86!");
12235 
12236   const unsigned V0Opc = V0.getOpcode();
12237   const unsigned Scale = V0EltSize / EltSize;
12238   const unsigned V0BroadcastIdx = BroadcastIdx / Scale;
12239 
12240   if ((V0Opc != ISD::SCALAR_TO_VECTOR || V0BroadcastIdx != 0) &&
12241       V0Opc != ISD::BUILD_VECTOR)
12242     return SDValue();
12243 
12244   SDValue Scalar = V0.getOperand(V0BroadcastIdx);
12245 
12246   // If we're extracting non-least-significant bits, shift so we can truncate.
12247   // Hopefully, we can fold away the trunc/srl/load into the broadcast.
12248   // Even if we can't (and !isShuffleFoldableLoad(Scalar)), prefer
12249   // vpbroadcast+vmovd+shr to vpshufb(m)+vmovd.
12250   if (const int OffsetIdx = BroadcastIdx % Scale)
12251     Scalar = DAG.getNode(ISD::SRL, DL, Scalar.getValueType(), Scalar,
12252                          DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8));
12253 
12254   return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
12255                      DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar));
12256 }
12257 
12258 /// Test whether this can be lowered with a single SHUFPS instruction.
12259 ///
12260 /// This is used to disable more specialized lowerings when the shufps lowering
12261 /// will happen to be efficient.
12262 static bool isSingleSHUFPSMask(ArrayRef<int> Mask) {
12263   // This routine only handles 128-bit shufps.
12264   assert(Mask.size() == 4 && "Unsupported mask size!");
12265   assert(Mask[0] >= -1 && Mask[0] < 8 && "Out of bound mask element!");
12266   assert(Mask[1] >= -1 && Mask[1] < 8 && "Out of bound mask element!");
12267   assert(Mask[2] >= -1 && Mask[2] < 8 && "Out of bound mask element!");
12268   assert(Mask[3] >= -1 && Mask[3] < 8 && "Out of bound mask element!");
12269 
12270   // To lower with a single SHUFPS we need to have the low half and high half
12271   // each requiring a single input.
12272   if (Mask[0] >= 0 && Mask[1] >= 0 && (Mask[0] < 4) != (Mask[1] < 4))
12273     return false;
12274   if (Mask[2] >= 0 && Mask[3] >= 0 && (Mask[2] < 4) != (Mask[3] < 4))
12275     return false;
12276 
12277   return true;
12278 }
12279 
12280 /// Test whether the specified input (0 or 1) is in-place blended by the
12281 /// given mask.
12282 ///
12283 /// This returns true if the elements from a particular input are already in the
12284 /// slot required by the given mask and require no permutation.
12285 static bool isShuffleMaskInputInPlace(int Input, ArrayRef<int> Mask) {
12286   assert((Input == 0 || Input == 1) && "Only two inputs to shuffles.");
12287   int Size = Mask.size();
12288   for (int i = 0; i < Size; ++i)
12289     if (Mask[i] >= 0 && Mask[i] / Size == Input && Mask[i] % Size != i)
12290       return false;
12291 
12292   return true;
12293 }
12294 
12295 /// If we are extracting two 128-bit halves of a vector and shuffling the
12296 /// result, match that to a 256-bit AVX2 vperm* instruction to avoid a
12297 /// multi-shuffle lowering.
12298 static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
12299                                              SDValue N1, ArrayRef<int> Mask,
12300                                              SelectionDAG &DAG) {
12301   MVT VT = N0.getSimpleValueType();
12302   assert((VT.is128BitVector() &&
12303           (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) &&
12304          "VPERM* family of shuffles requires 32-bit or 64-bit elements");
12305 
12306   // Check that both sources are extracts of the same source vector.
12307   if (N0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12308       N1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12309       N0.getOperand(0) != N1.getOperand(0) ||
12310       !N0.hasOneUse() || !N1.hasOneUse())
12311     return SDValue();
12312 
12313   SDValue WideVec = N0.getOperand(0);
12314   MVT WideVT = WideVec.getSimpleValueType();
12315   if (!WideVT.is256BitVector())
12316     return SDValue();
12317 
12318   // Match extracts of each half of the wide source vector. Commute the shuffle
12319   // if the extract of the low half is N1.
12320   unsigned NumElts = VT.getVectorNumElements();
12321   SmallVector<int, 4> NewMask(Mask);
12322   const APInt &ExtIndex0 = N0.getConstantOperandAPInt(1);
12323   const APInt &ExtIndex1 = N1.getConstantOperandAPInt(1);
12324   if (ExtIndex1 == 0 && ExtIndex0 == NumElts)
12325     ShuffleVectorSDNode::commuteMask(NewMask);
12326   else if (ExtIndex0 != 0 || ExtIndex1 != NumElts)
12327     return SDValue();
12328 
12329   // Final bailout: if the mask is simple, we are better off using an extract
12330   // and a simple narrow shuffle. Prefer extract+unpack(h/l)ps to vpermps
12331   // because that avoids a constant load from memory.
12332   if (NumElts == 4 &&
12333       (isSingleSHUFPSMask(NewMask) || is128BitUnpackShuffleMask(NewMask, DAG)))
12334     return SDValue();
12335 
12336   // Extend the shuffle mask with undef elements.
12337   NewMask.append(NumElts, -1);
12338 
12339   // shuf (extract X, 0), (extract X, 4), M --> extract (shuf X, undef, M'), 0
12340   SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
12341                                       NewMask);
12342   // This is free: ymm -> xmm.
12343   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
12344                      DAG.getIntPtrConstant(0, DL));
12345 }
12346 
12347 /// Try to lower broadcast of a single element.
12348 ///
12349 /// For convenience, this code also bundles all of the subtarget feature set
12350 /// filtering. While a little annoying to re-dispatch on type here, there isn't
12351 /// a convenient way to factor it out.
12352 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
12353                                        SDValue V2, ArrayRef<int> Mask,
12354                                        const X86Subtarget &Subtarget,
12355                                        SelectionDAG &DAG) {
12356   MVT EltVT = VT.getVectorElementType();
12357   if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) ||
12358         (Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||
12359         (Subtarget.hasAVX2() && (VT.isInteger() || EltVT == MVT::f16))))
12360     return SDValue();
12361 
12362   // With MOVDDUP (v2f64) we can broadcast from a register or a load, otherwise
12363   // we can only broadcast from a register with AVX2.
12364   unsigned NumEltBits = VT.getScalarSizeInBits();
12365   unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2())
12366                         ? X86ISD::MOVDDUP
12367                         : X86ISD::VBROADCAST;
12368   bool BroadcastFromReg = (Opcode == X86ISD::MOVDDUP) || Subtarget.hasAVX2();
12369 
12370   // Check that the mask is a broadcast.
12371   int BroadcastIdx = getSplatIndex(Mask);
12372   if (BroadcastIdx < 0)
12373     return SDValue();
12374   assert(BroadcastIdx < (int)Mask.size() && "We only expect to be called with "
12375                                             "a sorted mask where the broadcast "
12376                                             "comes from V1.");
12377 
12378   // Go up the chain of (vector) values to find a scalar load that we can
12379   // combine with the broadcast.
12380   // TODO: Combine this logic with findEltLoadSrc() used by
12381   //       EltsFromConsecutiveLoads().
12382   int BitOffset = BroadcastIdx * NumEltBits;
12383   SDValue V = V1;
12384   for (;;) {
12385     switch (V.getOpcode()) {
12386     case ISD::BITCAST: {
12387       V = V.getOperand(0);
12388       continue;
12389     }
12390     case ISD::CONCAT_VECTORS: {
12391       int OpBitWidth = V.getOperand(0).getValueSizeInBits();
12392       int OpIdx = BitOffset / OpBitWidth;
12393       V = V.getOperand(OpIdx);
12394       BitOffset %= OpBitWidth;
12395       continue;
12396     }
12397     case ISD::EXTRACT_SUBVECTOR: {
12398       // The extraction index adds to the existing offset.
12399       unsigned EltBitWidth = V.getScalarValueSizeInBits();
12400       unsigned Idx = V.getConstantOperandVal(1);
12401       unsigned BeginOffset = Idx * EltBitWidth;
12402       BitOffset += BeginOffset;
12403       V = V.getOperand(0);
12404       continue;
12405     }
12406     case ISD::INSERT_SUBVECTOR: {
12407       SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
12408       int EltBitWidth = VOuter.getScalarValueSizeInBits();
12409       int Idx = (int)V.getConstantOperandVal(2);
12410       int NumSubElts = (int)VInner.getSimpleValueType().getVectorNumElements();
12411       int BeginOffset = Idx * EltBitWidth;
12412       int EndOffset = BeginOffset + NumSubElts * EltBitWidth;
12413       if (BeginOffset <= BitOffset && BitOffset < EndOffset) {
12414         BitOffset -= BeginOffset;
12415         V = VInner;
12416       } else {
12417         V = VOuter;
12418       }
12419       continue;
12420     }
12421     }
12422     break;
12423   }
12424   assert((BitOffset % NumEltBits) == 0 && "Illegal bit-offset");
12425   BroadcastIdx = BitOffset / NumEltBits;
12426 
12427   // Do we need to bitcast the source to retrieve the original broadcast index?
12428   bool BitCastSrc = V.getScalarValueSizeInBits() != NumEltBits;
12429 
12430   // Check if this is a broadcast of a scalar. We special case lowering
12431   // for scalars so that we can more effectively fold with loads.
12432   // If the original value has a larger element type than the shuffle, the
12433   // broadcast element is in essence truncated. Make that explicit to ease
12434   // folding.
12435   if (BitCastSrc && VT.isInteger())
12436     if (SDValue TruncBroadcast = lowerShuffleAsTruncBroadcast(
12437             DL, VT, V, BroadcastIdx, Subtarget, DAG))
12438       return TruncBroadcast;
12439 
12440   // Also check the simpler case, where we can directly reuse the scalar.
12441   if (!BitCastSrc &&
12442       ((V.getOpcode() == ISD::BUILD_VECTOR && V.hasOneUse()) ||
12443        (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0))) {
12444     V = V.getOperand(BroadcastIdx);
12445 
12446     // If we can't broadcast from a register, check that the input is a load.
12447     if (!BroadcastFromReg && !isShuffleFoldableLoad(V))
12448       return SDValue();
12449   } else if (ISD::isNormalLoad(V.getNode()) &&
12450              cast<LoadSDNode>(V)->isSimple()) {
12451     // We do not check for one-use of the vector load because a broadcast load
12452     // is expected to be a win for code size, register pressure, and possibly
12453     // uops even if the original vector load is not eliminated.
12454 
12455     // Reduce the vector load and shuffle to a broadcasted scalar load.
12456     LoadSDNode *Ld = cast<LoadSDNode>(V);
12457     SDValue BaseAddr = Ld->getOperand(1);
12458     MVT SVT = VT.getScalarType();
12459     unsigned Offset = BroadcastIdx * SVT.getStoreSize();
12460     assert((int)(Offset * 8) == BitOffset && "Unexpected bit-offset");
12461     SDValue NewAddr =
12462         DAG.getMemBasePlusOffset(BaseAddr, TypeSize::getFixed(Offset), DL);
12463 
12464     // Directly form VBROADCAST_LOAD if we're using VBROADCAST opcode rather
12465     // than MOVDDUP.
12466     // FIXME: Should we add VBROADCAST_LOAD isel patterns for pre-AVX?
12467     if (Opcode == X86ISD::VBROADCAST) {
12468       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
12469       SDValue Ops[] = {Ld->getChain(), NewAddr};
12470       V = DAG.getMemIntrinsicNode(
12471           X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SVT,
12472           DAG.getMachineFunction().getMachineMemOperand(
12473               Ld->getMemOperand(), Offset, SVT.getStoreSize()));
12474       DAG.makeEquivalentMemoryOrdering(Ld, V);
12475       return DAG.getBitcast(VT, V);
12476     }
12477     assert(SVT == MVT::f64 && "Unexpected VT!");
12478     V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
12479                     DAG.getMachineFunction().getMachineMemOperand(
12480                         Ld->getMemOperand(), Offset, SVT.getStoreSize()));
12481     DAG.makeEquivalentMemoryOrdering(Ld, V);
12482   } else if (!BroadcastFromReg) {
12483     // We can't broadcast from a vector register.
12484     return SDValue();
12485   } else if (BitOffset != 0) {
12486     // We can only broadcast from the zero-element of a vector register,
12487     // but it can be advantageous to broadcast from the zero-element of a
12488     // subvector.
12489     if (!VT.is256BitVector() && !VT.is512BitVector())
12490       return SDValue();
12491 
12492     // VPERMQ/VPERMPD can perform the cross-lane shuffle directly.
12493     if (VT == MVT::v4f64 || VT == MVT::v4i64)
12494       return SDValue();
12495 
12496     // Only broadcast the zero-element of a 128-bit subvector.
12497     if ((BitOffset % 128) != 0)
12498       return SDValue();
12499 
12500     assert((BitOffset % V.getScalarValueSizeInBits()) == 0 &&
12501            "Unexpected bit-offset");
12502     assert((V.getValueSizeInBits() == 256 || V.getValueSizeInBits() == 512) &&
12503            "Unexpected vector size");
12504     unsigned ExtractIdx = BitOffset / V.getScalarValueSizeInBits();
12505     V = extract128BitVector(V, ExtractIdx, DAG, DL);
12506   }
12507 
12508   // On AVX we can use VBROADCAST directly for scalar sources.
12509   if (Opcode == X86ISD::MOVDDUP && !V.getValueType().isVector()) {
12510     V = DAG.getBitcast(MVT::f64, V);
12511     if (Subtarget.hasAVX()) {
12512       V = DAG.getNode(X86ISD::VBROADCAST, DL, MVT::v2f64, V);
12513       return DAG.getBitcast(VT, V);
12514     }
12515     V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V);
12516   }
12517 
12518   // If this is a scalar, do the broadcast on this type and bitcast.
12519   if (!V.getValueType().isVector()) {
12520     assert(V.getScalarValueSizeInBits() == NumEltBits &&
12521            "Unexpected scalar size");
12522     MVT BroadcastVT = MVT::getVectorVT(V.getSimpleValueType(),
12523                                        VT.getVectorNumElements());
12524     return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));
12525   }
12526 
12527   // We only support broadcasting from 128-bit vectors to minimize the
12528   // number of patterns we need to deal with in isel. So extract down to
12529   // 128-bits, removing as many bitcasts as possible.
12530   if (V.getValueSizeInBits() > 128)
12531     V = extract128BitVector(peekThroughBitcasts(V), 0, DAG, DL);
12532 
12533   // Otherwise cast V to a vector with the same element type as VT, but
12534   // possibly narrower than VT. Then perform the broadcast.
12535   unsigned NumSrcElts = V.getValueSizeInBits() / NumEltBits;
12536   MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(), NumSrcElts);
12537   return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V));
12538 }
12539 
12540 // Check for whether we can use INSERTPS to perform the shuffle. We only use
12541 // INSERTPS when the V1 elements are already in the correct locations
12542 // because otherwise we can just always use two SHUFPS instructions which
12543 // are much smaller to encode than a SHUFPS and an INSERTPS. We can also
12544 // perform INSERTPS if a single V1 element is out of place and all V2
12545 // elements are zeroable.
12546 static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
12547                                    unsigned &InsertPSMask,
12548                                    const APInt &Zeroable,
12549                                    ArrayRef<int> Mask, SelectionDAG &DAG) {
12550   assert(V1.getSimpleValueType().is128BitVector() && "Bad operand type!");
12551   assert(V2.getSimpleValueType().is128BitVector() && "Bad operand type!");
12552   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
12553 
12554   // Attempt to match INSERTPS with one element from VA or VB being
12555   // inserted into VA (or undef). If successful, V1, V2 and InsertPSMask
12556   // are updated.
12557   auto matchAsInsertPS = [&](SDValue VA, SDValue VB,
12558                              ArrayRef<int> CandidateMask) {
12559     unsigned ZMask = 0;
12560     int VADstIndex = -1;
12561     int VBDstIndex = -1;
12562     bool VAUsedInPlace = false;
12563 
12564     for (int i = 0; i < 4; ++i) {
12565       // Synthesize a zero mask from the zeroable elements (includes undefs).
12566       if (Zeroable[i]) {
12567         ZMask |= 1 << i;
12568         continue;
12569       }
12570 
12571       // Flag if we use any VA inputs in place.
12572       if (i == CandidateMask[i]) {
12573         VAUsedInPlace = true;
12574         continue;
12575       }
12576 
12577       // We can only insert a single non-zeroable element.
12578       if (VADstIndex >= 0 || VBDstIndex >= 0)
12579         return false;
12580 
12581       if (CandidateMask[i] < 4) {
12582         // VA input out of place for insertion.
12583         VADstIndex = i;
12584       } else {
12585         // VB input for insertion.
12586         VBDstIndex = i;
12587       }
12588     }
12589 
12590     // Don't bother if we have no (non-zeroable) element for insertion.
12591     if (VADstIndex < 0 && VBDstIndex < 0)
12592       return false;
12593 
12594     // Determine element insertion src/dst indices. The src index is from the
12595     // start of the inserted vector, not the start of the concatenated vector.
12596     unsigned VBSrcIndex = 0;
12597     if (VADstIndex >= 0) {
12598       // If we have a VA input out of place, we use VA as the V2 element
12599       // insertion and don't use the original V2 at all.
12600       VBSrcIndex = CandidateMask[VADstIndex];
12601       VBDstIndex = VADstIndex;
12602       VB = VA;
12603     } else {
12604       VBSrcIndex = CandidateMask[VBDstIndex] - 4;
12605     }
12606 
12607     // If no V1 inputs are used in place, then the result is created only from
12608     // the zero mask and the V2 insertion - so remove V1 dependency.
12609     if (!VAUsedInPlace)
12610       VA = DAG.getUNDEF(MVT::v4f32);
12611 
12612     // Update V1, V2 and InsertPSMask accordingly.
12613     V1 = VA;
12614     V2 = VB;
12615 
12616     // Insert the V2 element into the desired position.
12617     InsertPSMask = VBSrcIndex << 6 | VBDstIndex << 4 | ZMask;
12618     assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
12619     return true;
12620   };
12621 
12622   if (matchAsInsertPS(V1, V2, Mask))
12623     return true;
12624 
12625   // Commute and try again.
12626   SmallVector<int, 4> CommutedMask(Mask);
12627   ShuffleVectorSDNode::commuteMask(CommutedMask);
12628   if (matchAsInsertPS(V2, V1, CommutedMask))
12629     return true;
12630 
12631   return false;
12632 }
12633 
12634 static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
12635                                       ArrayRef<int> Mask, const APInt &Zeroable,
12636                                       SelectionDAG &DAG) {
12637   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12638   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12639 
12640   // Attempt to match the insertps pattern.
12641   unsigned InsertPSMask = 0;
12642   if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
12643     return SDValue();
12644 
12645   // Insert the V2 element into the desired position.
12646   return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
12647                      DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
12648 }
12649 
12650 /// Handle lowering of 2-lane 64-bit floating point shuffles.
12651 ///
12652 /// This is the basis function for the 2-lane 64-bit shuffles as we have full
12653 /// support for floating point shuffles but not integer shuffles. These
12654 /// instructions will incur a domain crossing penalty on some chips though so
12655 /// it is better to avoid lowering through this for integer vectors where
12656 /// possible.
12657 static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12658                                  const APInt &Zeroable, SDValue V1, SDValue V2,
12659                                  const X86Subtarget &Subtarget,
12660                                  SelectionDAG &DAG) {
12661   assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
12662   assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
12663   assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
12664 
12665   if (V2.isUndef()) {
12666     // Check for being able to broadcast a single element.
12667     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
12668                                                     Mask, Subtarget, DAG))
12669       return Broadcast;
12670 
12671     // Straight shuffle of a single input vector. Simulate this by using the
12672     // single input as both of the "inputs" to this instruction..
12673     unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1);
12674 
12675     if (Subtarget.hasAVX()) {
12676       // If we have AVX, we can use VPERMILPS which will allow folding a load
12677       // into the shuffle.
12678       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
12679                          DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12680     }
12681 
12682     return DAG.getNode(
12683         X86ISD::SHUFP, DL, MVT::v2f64,
12684         Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
12685         Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
12686         DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12687   }
12688   assert(Mask[0] >= 0 && "No undef lanes in multi-input v2 shuffles!");
12689   assert(Mask[1] >= 0 && "No undef lanes in multi-input v2 shuffles!");
12690   assert(Mask[0] < 2 && "We sort V1 to be the first input.");
12691   assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
12692 
12693   if (Subtarget.hasAVX2())
12694     if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12695       return Extract;
12696 
12697   // When loading a scalar and then shuffling it into a vector we can often do
12698   // the insertion cheaply.
12699   if (SDValue Insertion = lowerShuffleAsElementInsertion(
12700           DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
12701     return Insertion;
12702   // Try inverting the insertion since for v2 masks it is easy to do and we
12703   // can't reliably sort the mask one way or the other.
12704   int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
12705                         Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
12706   if (SDValue Insertion = lowerShuffleAsElementInsertion(
12707           DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
12708     return Insertion;
12709 
12710   // Try to use one of the special instruction patterns to handle two common
12711   // blend patterns if a zero-blend above didn't work.
12712   if (isShuffleEquivalent(Mask, {0, 3}, V1, V2) ||
12713       isShuffleEquivalent(Mask, {1, 3}, V1, V2))
12714     if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
12715       // We can either use a special instruction to load over the low double or
12716       // to move just the low double.
12717       return DAG.getNode(
12718           X86ISD::MOVSD, DL, MVT::v2f64, V2,
12719           DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
12720 
12721   if (Subtarget.hasSSE41())
12722     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
12723                                             Zeroable, Subtarget, DAG))
12724       return Blend;
12725 
12726   // Use dedicated unpack instructions for masks that match their pattern.
12727   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
12728     return V;
12729 
12730   unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
12731   return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
12732                      DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12733 }
12734 
12735 /// Handle lowering of 2-lane 64-bit integer shuffles.
12736 ///
12737 /// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by
12738 /// the integer unit to minimize domain crossing penalties. However, for blends
12739 /// it falls back to the floating point shuffle operation with appropriate bit
12740 /// casting.
12741 static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12742                                  const APInt &Zeroable, SDValue V1, SDValue V2,
12743                                  const X86Subtarget &Subtarget,
12744                                  SelectionDAG &DAG) {
12745   assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
12746   assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
12747   assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
12748 
12749   if (V2.isUndef()) {
12750     // Check for being able to broadcast a single element.
12751     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
12752                                                     Mask, Subtarget, DAG))
12753       return Broadcast;
12754 
12755     // Straight shuffle of a single input vector. For everything from SSE2
12756     // onward this has a single fast instruction with no scary immediates.
12757     // We have to map the mask as it is actually a v4i32 shuffle instruction.
12758     V1 = DAG.getBitcast(MVT::v4i32, V1);
12759     int WidenedMask[4] = {Mask[0] < 0 ? -1 : (Mask[0] * 2),
12760                           Mask[0] < 0 ? -1 : ((Mask[0] * 2) + 1),
12761                           Mask[1] < 0 ? -1 : (Mask[1] * 2),
12762                           Mask[1] < 0 ? -1 : ((Mask[1] * 2) + 1)};
12763     return DAG.getBitcast(
12764         MVT::v2i64,
12765         DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
12766                     getV4X86ShuffleImm8ForMask(WidenedMask, DL, DAG)));
12767   }
12768   assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!");
12769   assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!");
12770   assert(Mask[0] < 2 && "We sort V1 to be the first input.");
12771   assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
12772 
12773   if (Subtarget.hasAVX2())
12774     if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12775       return Extract;
12776 
12777   // Try to use shift instructions.
12778   if (SDValue Shift =
12779           lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget,
12780                               DAG, /*BitwiseOnly*/ false))
12781     return Shift;
12782 
12783   // When loading a scalar and then shuffling it into a vector we can often do
12784   // the insertion cheaply.
12785   if (SDValue Insertion = lowerShuffleAsElementInsertion(
12786           DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
12787     return Insertion;
12788   // Try inverting the insertion since for v2 masks it is easy to do and we
12789   // can't reliably sort the mask one way or the other.
12790   int InverseMask[2] = {Mask[0] ^ 2, Mask[1] ^ 2};
12791   if (SDValue Insertion = lowerShuffleAsElementInsertion(
12792           DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
12793     return Insertion;
12794 
12795   // We have different paths for blend lowering, but they all must use the
12796   // *exact* same predicate.
12797   bool IsBlendSupported = Subtarget.hasSSE41();
12798   if (IsBlendSupported)
12799     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
12800                                             Zeroable, Subtarget, DAG))
12801       return Blend;
12802 
12803   // Use dedicated unpack instructions for masks that match their pattern.
12804   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
12805     return V;
12806 
12807   // Try to use byte rotation instructions.
12808   // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
12809   if (Subtarget.hasSSSE3()) {
12810     if (Subtarget.hasVLX())
12811       if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v2i64, V1, V2, Mask,
12812                                                 Zeroable, Subtarget, DAG))
12813         return Rotate;
12814 
12815     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
12816                                                   Subtarget, DAG))
12817       return Rotate;
12818   }
12819 
12820   // If we have direct support for blends, we should lower by decomposing into
12821   // a permute. That will be faster than the domain cross.
12822   if (IsBlendSupported)
12823     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v2i64, V1, V2, Mask,
12824                                                 Subtarget, DAG);
12825 
12826   // We implement this with SHUFPD which is pretty lame because it will likely
12827   // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
12828   // However, all the alternatives are still more cycles and newer chips don't
12829   // have this problem. It would be really nice if x86 had better shuffles here.
12830   V1 = DAG.getBitcast(MVT::v2f64, V1);
12831   V2 = DAG.getBitcast(MVT::v2f64, V2);
12832   return DAG.getBitcast(MVT::v2i64,
12833                         DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
12834 }
12835 
12836 /// Lower a vector shuffle using the SHUFPS instruction.
12837 ///
12838 /// This is a helper routine dedicated to lowering vector shuffles using SHUFPS.
12839 /// It makes no assumptions about whether this is the *best* lowering, it simply
12840 /// uses it.
12841 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
12842                                       ArrayRef<int> Mask, SDValue V1,
12843                                       SDValue V2, SelectionDAG &DAG) {
12844   SDValue LowV = V1, HighV = V2;
12845   SmallVector<int, 4> NewMask(Mask);
12846   int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
12847 
12848   if (NumV2Elements == 1) {
12849     int V2Index = find_if(Mask, [](int M) { return M >= 4; }) - Mask.begin();
12850 
12851     // Compute the index adjacent to V2Index and in the same half by toggling
12852     // the low bit.
12853     int V2AdjIndex = V2Index ^ 1;
12854 
12855     if (Mask[V2AdjIndex] < 0) {
12856       // Handles all the cases where we have a single V2 element and an undef.
12857       // This will only ever happen in the high lanes because we commute the
12858       // vector otherwise.
12859       if (V2Index < 2)
12860         std::swap(LowV, HighV);
12861       NewMask[V2Index] -= 4;
12862     } else {
12863       // Handle the case where the V2 element ends up adjacent to a V1 element.
12864       // To make this work, blend them together as the first step.
12865       int V1Index = V2AdjIndex;
12866       int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0};
12867       V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
12868                        getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
12869 
12870       // Now proceed to reconstruct the final blend as we have the necessary
12871       // high or low half formed.
12872       if (V2Index < 2) {
12873         LowV = V2;
12874         HighV = V1;
12875       } else {
12876         HighV = V2;
12877       }
12878       NewMask[V1Index] = 2; // We put the V1 element in V2[2].
12879       NewMask[V2Index] = 0; // We shifted the V2 element into V2[0].
12880     }
12881   } else if (NumV2Elements == 2) {
12882     if (Mask[0] < 4 && Mask[1] < 4) {
12883       // Handle the easy case where we have V1 in the low lanes and V2 in the
12884       // high lanes.
12885       NewMask[2] -= 4;
12886       NewMask[3] -= 4;
12887     } else if (Mask[2] < 4 && Mask[3] < 4) {
12888       // We also handle the reversed case because this utility may get called
12889       // when we detect a SHUFPS pattern but can't easily commute the shuffle to
12890       // arrange things in the right direction.
12891       NewMask[0] -= 4;
12892       NewMask[1] -= 4;
12893       HighV = V1;
12894       LowV = V2;
12895     } else {
12896       // We have a mixture of V1 and V2 in both low and high lanes. Rather than
12897       // trying to place elements directly, just blend them and set up the final
12898       // shuffle to place them.
12899 
12900       // The first two blend mask elements are for V1, the second two are for
12901       // V2.
12902       int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1],
12903                           Mask[2] < 4 ? Mask[2] : Mask[3],
12904                           (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4,
12905                           (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4};
12906       V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
12907                        getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
12908 
12909       // Now we do a normal shuffle of V1 by giving V1 as both operands to
12910       // a blend.
12911       LowV = HighV = V1;
12912       NewMask[0] = Mask[0] < 4 ? 0 : 2;
12913       NewMask[1] = Mask[0] < 4 ? 2 : 0;
12914       NewMask[2] = Mask[2] < 4 ? 1 : 3;
12915       NewMask[3] = Mask[2] < 4 ? 3 : 1;
12916     }
12917   } else if (NumV2Elements == 3) {
12918     // Ideally canonicalizeShuffleMaskWithCommute should have caught this, but
12919     // we can get here due to other paths (e.g repeated mask matching) that we
12920     // don't want to do another round of lowerVECTOR_SHUFFLE.
12921     ShuffleVectorSDNode::commuteMask(NewMask);
12922     return lowerShuffleWithSHUFPS(DL, VT, NewMask, V2, V1, DAG);
12923   }
12924   return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
12925                      getV4X86ShuffleImm8ForMask(NewMask, DL, DAG));
12926 }
12927 
12928 /// Lower 4-lane 32-bit floating point shuffles.
12929 ///
12930 /// Uses instructions exclusively from the floating point unit to minimize
12931 /// domain crossing penalties, as these are sufficient to implement all v4f32
12932 /// shuffles.
12933 static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12934                                  const APInt &Zeroable, SDValue V1, SDValue V2,
12935                                  const X86Subtarget &Subtarget,
12936                                  SelectionDAG &DAG) {
12937   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12938   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12939   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
12940 
12941   if (Subtarget.hasSSE41())
12942     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
12943                                             Zeroable, Subtarget, DAG))
12944       return Blend;
12945 
12946   int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
12947 
12948   if (NumV2Elements == 0) {
12949     // Check for being able to broadcast a single element.
12950     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
12951                                                     Mask, Subtarget, DAG))
12952       return Broadcast;
12953 
12954     // Use even/odd duplicate instructions for masks that match their pattern.
12955     if (Subtarget.hasSSE3()) {
12956       if (isShuffleEquivalent(Mask, {0, 0, 2, 2}, V1, V2))
12957         return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
12958       if (isShuffleEquivalent(Mask, {1, 1, 3, 3}, V1, V2))
12959         return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
12960     }
12961 
12962     if (Subtarget.hasAVX()) {
12963       // If we have AVX, we can use VPERMILPS which will allow folding a load
12964       // into the shuffle.
12965       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
12966                          getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
12967     }
12968 
12969     // Use MOVLHPS/MOVHLPS to simulate unary shuffles. These are only valid
12970     // in SSE1 because otherwise they are widened to v2f64 and never get here.
12971     if (!Subtarget.hasSSE2()) {
12972       if (isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1, V2))
12973         return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1);
12974       if (isShuffleEquivalent(Mask, {2, 3, 2, 3}, V1, V2))
12975         return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1);
12976     }
12977 
12978     // Otherwise, use a straight shuffle of a single input vector. We pass the
12979     // input vector to both operands to simulate this with a SHUFPS.
12980     return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
12981                        getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
12982   }
12983 
12984   if (Subtarget.hasSSE2())
12985     if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
12986             DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) {
12987       ZExt = DAG.getBitcast(MVT::v4f32, ZExt);
12988       return ZExt;
12989     }
12990 
12991   if (Subtarget.hasAVX2())
12992     if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12993       return Extract;
12994 
12995   // There are special ways we can lower some single-element blends. However, we
12996   // have custom ways we can lower more complex single-element blends below that
12997   // we defer to if both this and BLENDPS fail to match, so restrict this to
12998   // when the V2 input is targeting element 0 of the mask -- that is the fast
12999   // case here.
13000   if (NumV2Elements == 1 && Mask[0] >= 4)
13001     if (SDValue V = lowerShuffleAsElementInsertion(
13002             DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13003       return V;
13004 
13005   if (Subtarget.hasSSE41()) {
13006     // Use INSERTPS if we can complete the shuffle efficiently.
13007     if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13008       return V;
13009 
13010     if (!isSingleSHUFPSMask(Mask))
13011       if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
13012                                                             V2, Mask, DAG))
13013         return BlendPerm;
13014   }
13015 
13016   // Use low/high mov instructions. These are only valid in SSE1 because
13017   // otherwise they are widened to v2f64 and never get here.
13018   if (!Subtarget.hasSSE2()) {
13019     if (isShuffleEquivalent(Mask, {0, 1, 4, 5}, V1, V2))
13020       return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
13021     if (isShuffleEquivalent(Mask, {2, 3, 6, 7}, V1, V2))
13022       return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
13023   }
13024 
13025   // Use dedicated unpack instructions for masks that match their pattern.
13026   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
13027     return V;
13028 
13029   // Otherwise fall back to a SHUFPS lowering strategy.
13030   return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
13031 }
13032 
13033 /// Lower 4-lane i32 vector shuffles.
13034 ///
13035 /// We try to handle these with integer-domain shuffles where we can, but for
13036 /// blends we use the floating point domain blend instructions.
13037 static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13038                                  const APInt &Zeroable, SDValue V1, SDValue V2,
13039                                  const X86Subtarget &Subtarget,
13040                                  SelectionDAG &DAG) {
13041   assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
13042   assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
13043   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
13044 
13045   // Whenever we can lower this as a zext, that instruction is strictly faster
13046   // than any alternative. It also allows us to fold memory operands into the
13047   // shuffle in many cases.
13048   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
13049                                                    Zeroable, Subtarget, DAG))
13050     return ZExt;
13051 
13052   int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
13053 
13054   // Try to use shift instructions if fast.
13055   if (Subtarget.preferLowerShuffleAsShift()) {
13056     if (SDValue Shift =
13057             lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, Zeroable,
13058                                 Subtarget, DAG, /*BitwiseOnly*/ true))
13059       return Shift;
13060     if (NumV2Elements == 0)
13061       if (SDValue Rotate =
13062               lowerShuffleAsBitRotate(DL, MVT::v4i32, V1, Mask, Subtarget, DAG))
13063         return Rotate;
13064   }
13065 
13066   if (NumV2Elements == 0) {
13067     // Try to use broadcast unless the mask only has one non-undef element.
13068     if (count_if(Mask, [](int M) { return M >= 0 && M < 4; }) > 1) {
13069       if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
13070                                                       Mask, Subtarget, DAG))
13071         return Broadcast;
13072     }
13073 
13074     // Straight shuffle of a single input vector. For everything from SSE2
13075     // onward this has a single fast instruction with no scary immediates.
13076     // We coerce the shuffle pattern to be compatible with UNPCK instructions
13077     // but we aren't actually going to use the UNPCK instruction because doing
13078     // so prevents folding a load into this instruction or making a copy.
13079     const int UnpackLoMask[] = {0, 0, 1, 1};
13080     const int UnpackHiMask[] = {2, 2, 3, 3};
13081     if (isShuffleEquivalent(Mask, {0, 0, 1, 1}, V1, V2))
13082       Mask = UnpackLoMask;
13083     else if (isShuffleEquivalent(Mask, {2, 2, 3, 3}, V1, V2))
13084       Mask = UnpackHiMask;
13085 
13086     return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
13087                        getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13088   }
13089 
13090   if (Subtarget.hasAVX2())
13091     if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13092       return Extract;
13093 
13094   // Try to use shift instructions.
13095   if (SDValue Shift =
13096           lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget,
13097                               DAG, /*BitwiseOnly*/ false))
13098     return Shift;
13099 
13100   // There are special ways we can lower some single-element blends.
13101   if (NumV2Elements == 1)
13102     if (SDValue V = lowerShuffleAsElementInsertion(
13103             DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13104       return V;
13105 
13106   // We have different paths for blend lowering, but they all must use the
13107   // *exact* same predicate.
13108   bool IsBlendSupported = Subtarget.hasSSE41();
13109   if (IsBlendSupported)
13110     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
13111                                             Zeroable, Subtarget, DAG))
13112       return Blend;
13113 
13114   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
13115                                              Zeroable, Subtarget, DAG))
13116     return Masked;
13117 
13118   // Use dedicated unpack instructions for masks that match their pattern.
13119   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
13120     return V;
13121 
13122   // Try to use byte rotation instructions.
13123   // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
13124   if (Subtarget.hasSSSE3()) {
13125     if (Subtarget.hasVLX())
13126       if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i32, V1, V2, Mask,
13127                                                 Zeroable, Subtarget, DAG))
13128         return Rotate;
13129 
13130     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
13131                                                   Subtarget, DAG))
13132       return Rotate;
13133   }
13134 
13135   // Assume that a single SHUFPS is faster than an alternative sequence of
13136   // multiple instructions (even if the CPU has a domain penalty).
13137   // If some CPU is harmed by the domain switch, we can fix it in a later pass.
13138   if (!isSingleSHUFPSMask(Mask)) {
13139     // If we have direct support for blends, we should lower by decomposing into
13140     // a permute. That will be faster than the domain cross.
13141     if (IsBlendSupported)
13142       return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i32, V1, V2, Mask,
13143                                                   Subtarget, DAG);
13144 
13145     // Try to lower by permuting the inputs into an unpack instruction.
13146     if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
13147                                                         Mask, Subtarget, DAG))
13148       return Unpack;
13149   }
13150 
13151   // We implement this with SHUFPS because it can blend from two vectors.
13152   // Because we're going to eventually use SHUFPS, we use SHUFPS even to build
13153   // up the inputs, bypassing domain shift penalties that we would incur if we
13154   // directly used PSHUFD on Nehalem and older. For newer chips, this isn't
13155   // relevant.
13156   SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);
13157   SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);
13158   SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
13159   return DAG.getBitcast(MVT::v4i32, ShufPS);
13160 }
13161 
13162 /// Lowering of single-input v8i16 shuffles is the cornerstone of SSE2
13163 /// shuffle lowering, and the most complex part.
13164 ///
13165 /// The lowering strategy is to try to form pairs of input lanes which are
13166 /// targeted at the same half of the final vector, and then use a dword shuffle
13167 /// to place them onto the right half, and finally unpack the paired lanes into
13168 /// their final position.
13169 ///
13170 /// The exact breakdown of how to form these dword pairs and align them on the
13171 /// correct sides is really tricky. See the comments within the function for
13172 /// more of the details.
13173 ///
13174 /// This code also handles repeated 128-bit lanes of v8i16 shuffles, but each
13175 /// lane must shuffle the *exact* same way. In fact, you must pass a v8 Mask to
13176 /// this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16
13177 /// vector, form the analogous 128-bit 8-element Mask.
13178 static SDValue lowerV8I16GeneralSingleInputShuffle(
13179     const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
13180     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
13181   assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!");
13182   MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
13183 
13184   assert(Mask.size() == 8 && "Shuffle mask length doesn't match!");
13185   MutableArrayRef<int> LoMask = Mask.slice(0, 4);
13186   MutableArrayRef<int> HiMask = Mask.slice(4, 4);
13187 
13188   // Attempt to directly match PSHUFLW or PSHUFHW.
13189   if (isUndefOrInRange(LoMask, 0, 4) &&
13190       isSequentialOrUndefInRange(HiMask, 0, 4, 4)) {
13191     return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13192                        getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
13193   }
13194   if (isUndefOrInRange(HiMask, 4, 8) &&
13195       isSequentialOrUndefInRange(LoMask, 0, 4, 0)) {
13196     for (int i = 0; i != 4; ++i)
13197       HiMask[i] = (HiMask[i] < 0 ? HiMask[i] : (HiMask[i] - 4));
13198     return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13199                        getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
13200   }
13201 
13202   SmallVector<int, 4> LoInputs;
13203   copy_if(LoMask, std::back_inserter(LoInputs), [](int M) { return M >= 0; });
13204   array_pod_sort(LoInputs.begin(), LoInputs.end());
13205   LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), LoInputs.end());
13206   SmallVector<int, 4> HiInputs;
13207   copy_if(HiMask, std::back_inserter(HiInputs), [](int M) { return M >= 0; });
13208   array_pod_sort(HiInputs.begin(), HiInputs.end());
13209   HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), HiInputs.end());
13210   int NumLToL = llvm::lower_bound(LoInputs, 4) - LoInputs.begin();
13211   int NumHToL = LoInputs.size() - NumLToL;
13212   int NumLToH = llvm::lower_bound(HiInputs, 4) - HiInputs.begin();
13213   int NumHToH = HiInputs.size() - NumLToH;
13214   MutableArrayRef<int> LToLInputs(LoInputs.data(), NumLToL);
13215   MutableArrayRef<int> LToHInputs(HiInputs.data(), NumLToH);
13216   MutableArrayRef<int> HToLInputs(LoInputs.data() + NumLToL, NumHToL);
13217   MutableArrayRef<int> HToHInputs(HiInputs.data() + NumLToH, NumHToH);
13218 
13219   // If we are shuffling values from one half - check how many different DWORD
13220   // pairs we need to create. If only 1 or 2 then we can perform this as a
13221   // PSHUFLW/PSHUFHW + PSHUFD instead of the PSHUFD+PSHUFLW+PSHUFHW chain below.
13222   auto ShuffleDWordPairs = [&](ArrayRef<int> PSHUFHalfMask,
13223                                ArrayRef<int> PSHUFDMask, unsigned ShufWOp) {
13224     V = DAG.getNode(ShufWOp, DL, VT, V,
13225                     getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13226     V = DAG.getBitcast(PSHUFDVT, V);
13227     V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V,
13228                     getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
13229     return DAG.getBitcast(VT, V);
13230   };
13231 
13232   if ((NumHToL + NumHToH) == 0 || (NumLToL + NumLToH) == 0) {
13233     int PSHUFDMask[4] = { -1, -1, -1, -1 };
13234     SmallVector<std::pair<int, int>, 4> DWordPairs;
13235     int DOffset = ((NumHToL + NumHToH) == 0 ? 0 : 2);
13236 
13237     // Collect the different DWORD pairs.
13238     for (int DWord = 0; DWord != 4; ++DWord) {
13239       int M0 = Mask[2 * DWord + 0];
13240       int M1 = Mask[2 * DWord + 1];
13241       M0 = (M0 >= 0 ? M0 % 4 : M0);
13242       M1 = (M1 >= 0 ? M1 % 4 : M1);
13243       if (M0 < 0 && M1 < 0)
13244         continue;
13245 
13246       bool Match = false;
13247       for (int j = 0, e = DWordPairs.size(); j < e; ++j) {
13248         auto &DWordPair = DWordPairs[j];
13249         if ((M0 < 0 || isUndefOrEqual(DWordPair.first, M0)) &&
13250             (M1 < 0 || isUndefOrEqual(DWordPair.second, M1))) {
13251           DWordPair.first = (M0 >= 0 ? M0 : DWordPair.first);
13252           DWordPair.second = (M1 >= 0 ? M1 : DWordPair.second);
13253           PSHUFDMask[DWord] = DOffset + j;
13254           Match = true;
13255           break;
13256         }
13257       }
13258       if (!Match) {
13259         PSHUFDMask[DWord] = DOffset + DWordPairs.size();
13260         DWordPairs.push_back(std::make_pair(M0, M1));
13261       }
13262     }
13263 
13264     if (DWordPairs.size() <= 2) {
13265       DWordPairs.resize(2, std::make_pair(-1, -1));
13266       int PSHUFHalfMask[4] = {DWordPairs[0].first, DWordPairs[0].second,
13267                               DWordPairs[1].first, DWordPairs[1].second};
13268       if ((NumHToL + NumHToH) == 0)
13269         return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFLW);
13270       if ((NumLToL + NumLToH) == 0)
13271         return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFHW);
13272     }
13273   }
13274 
13275   // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all
13276   // such inputs we can swap two of the dwords across the half mark and end up
13277   // with <=2 inputs to each half in each half. Once there, we can fall through
13278   // to the generic code below. For example:
13279   //
13280   // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
13281   // Mask:  [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5]
13282   //
13283   // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half
13284   // and an existing 2-into-2 on the other half. In this case we may have to
13285   // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or
13286   // 1-into-3 which could cause us to cycle endlessly fixing each side in turn.
13287   // Fortunately, we don't have to handle anything but a 2-into-2 pattern
13288   // because any other situation (including a 3-into-1 or 1-into-3 in the other
13289   // half than the one we target for fixing) will be fixed when we re-enter this
13290   // path. We will also combine away any sequence of PSHUFD instructions that
13291   // result into a single instruction. Here is an example of the tricky case:
13292   //
13293   // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
13294   // Mask:  [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3]
13295   //
13296   // This now has a 1-into-3 in the high half! Instead, we do two shuffles:
13297   //
13298   // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h]
13299   // Mask:  [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6]
13300   //
13301   // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h]
13302   // Mask:  [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6]
13303   //
13304   // The result is fine to be handled by the generic logic.
13305   auto balanceSides = [&](ArrayRef<int> AToAInputs, ArrayRef<int> BToAInputs,
13306                           ArrayRef<int> BToBInputs, ArrayRef<int> AToBInputs,
13307                           int AOffset, int BOffset) {
13308     assert((AToAInputs.size() == 3 || AToAInputs.size() == 1) &&
13309            "Must call this with A having 3 or 1 inputs from the A half.");
13310     assert((BToAInputs.size() == 1 || BToAInputs.size() == 3) &&
13311            "Must call this with B having 1 or 3 inputs from the B half.");
13312     assert(AToAInputs.size() + BToAInputs.size() == 4 &&
13313            "Must call this with either 3:1 or 1:3 inputs (summing to 4).");
13314 
13315     bool ThreeAInputs = AToAInputs.size() == 3;
13316 
13317     // Compute the index of dword with only one word among the three inputs in
13318     // a half by taking the sum of the half with three inputs and subtracting
13319     // the sum of the actual three inputs. The difference is the remaining
13320     // slot.
13321     int ADWord = 0, BDWord = 0;
13322     int &TripleDWord = ThreeAInputs ? ADWord : BDWord;
13323     int &OneInputDWord = ThreeAInputs ? BDWord : ADWord;
13324     int TripleInputOffset = ThreeAInputs ? AOffset : BOffset;
13325     ArrayRef<int> TripleInputs = ThreeAInputs ? AToAInputs : BToAInputs;
13326     int OneInput = ThreeAInputs ? BToAInputs[0] : AToAInputs[0];
13327     int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset);
13328     int TripleNonInputIdx =
13329         TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
13330     TripleDWord = TripleNonInputIdx / 2;
13331 
13332     // We use xor with one to compute the adjacent DWord to whichever one the
13333     // OneInput is in.
13334     OneInputDWord = (OneInput / 2) ^ 1;
13335 
13336     // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA
13337     // and BToA inputs. If there is also such a problem with the BToB and AToB
13338     // inputs, we don't try to fix it necessarily -- we'll recurse and see it in
13339     // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it
13340     // is essential that we don't *create* a 3<-1 as then we might oscillate.
13341     if (BToBInputs.size() == 2 && AToBInputs.size() == 2) {
13342       // Compute how many inputs will be flipped by swapping these DWords. We
13343       // need
13344       // to balance this to ensure we don't form a 3-1 shuffle in the other
13345       // half.
13346       int NumFlippedAToBInputs = llvm::count(AToBInputs, 2 * ADWord) +
13347                                  llvm::count(AToBInputs, 2 * ADWord + 1);
13348       int NumFlippedBToBInputs = llvm::count(BToBInputs, 2 * BDWord) +
13349                                  llvm::count(BToBInputs, 2 * BDWord + 1);
13350       if ((NumFlippedAToBInputs == 1 &&
13351            (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) ||
13352           (NumFlippedBToBInputs == 1 &&
13353            (NumFlippedAToBInputs == 0 || NumFlippedAToBInputs == 2))) {
13354         // We choose whether to fix the A half or B half based on whether that
13355         // half has zero flipped inputs. At zero, we may not be able to fix it
13356         // with that half. We also bias towards fixing the B half because that
13357         // will more commonly be the high half, and we have to bias one way.
13358         auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord,
13359                                                        ArrayRef<int> Inputs) {
13360           int FixIdx = PinnedIdx ^ 1; // The adjacent slot to the pinned slot.
13361           bool IsFixIdxInput = is_contained(Inputs, PinnedIdx ^ 1);
13362           // Determine whether the free index is in the flipped dword or the
13363           // unflipped dword based on where the pinned index is. We use this bit
13364           // in an xor to conditionally select the adjacent dword.
13365           int FixFreeIdx = 2 * (DWord ^ (PinnedIdx / 2 == DWord));
13366           bool IsFixFreeIdxInput = is_contained(Inputs, FixFreeIdx);
13367           if (IsFixIdxInput == IsFixFreeIdxInput)
13368             FixFreeIdx += 1;
13369           IsFixFreeIdxInput = is_contained(Inputs, FixFreeIdx);
13370           assert(IsFixIdxInput != IsFixFreeIdxInput &&
13371                  "We need to be changing the number of flipped inputs!");
13372           int PSHUFHalfMask[] = {0, 1, 2, 3};
13373           std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]);
13374           V = DAG.getNode(
13375               FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
13376               MVT::getVectorVT(MVT::i16, V.getValueSizeInBits() / 16), V,
13377               getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13378 
13379           for (int &M : Mask)
13380             if (M >= 0 && M == FixIdx)
13381               M = FixFreeIdx;
13382             else if (M >= 0 && M == FixFreeIdx)
13383               M = FixIdx;
13384         };
13385         if (NumFlippedBToBInputs != 0) {
13386           int BPinnedIdx =
13387               BToAInputs.size() == 3 ? TripleNonInputIdx : OneInput;
13388           FixFlippedInputs(BPinnedIdx, BDWord, BToBInputs);
13389         } else {
13390           assert(NumFlippedAToBInputs != 0 && "Impossible given predicates!");
13391           int APinnedIdx = ThreeAInputs ? TripleNonInputIdx : OneInput;
13392           FixFlippedInputs(APinnedIdx, ADWord, AToBInputs);
13393         }
13394       }
13395     }
13396 
13397     int PSHUFDMask[] = {0, 1, 2, 3};
13398     PSHUFDMask[ADWord] = BDWord;
13399     PSHUFDMask[BDWord] = ADWord;
13400     V = DAG.getBitcast(
13401         VT,
13402         DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
13403                     getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
13404 
13405     // Adjust the mask to match the new locations of A and B.
13406     for (int &M : Mask)
13407       if (M >= 0 && M/2 == ADWord)
13408         M = 2 * BDWord + M % 2;
13409       else if (M >= 0 && M/2 == BDWord)
13410         M = 2 * ADWord + M % 2;
13411 
13412     // Recurse back into this routine to re-compute state now that this isn't
13413     // a 3 and 1 problem.
13414     return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG);
13415   };
13416   if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3))
13417     return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4);
13418   if ((NumHToH == 3 && NumLToH == 1) || (NumHToH == 1 && NumLToH == 3))
13419     return balanceSides(HToHInputs, LToHInputs, LToLInputs, HToLInputs, 4, 0);
13420 
13421   // At this point there are at most two inputs to the low and high halves from
13422   // each half. That means the inputs can always be grouped into dwords and
13423   // those dwords can then be moved to the correct half with a dword shuffle.
13424   // We use at most one low and one high word shuffle to collect these paired
13425   // inputs into dwords, and finally a dword shuffle to place them.
13426   int PSHUFLMask[4] = {-1, -1, -1, -1};
13427   int PSHUFHMask[4] = {-1, -1, -1, -1};
13428   int PSHUFDMask[4] = {-1, -1, -1, -1};
13429 
13430   // First fix the masks for all the inputs that are staying in their
13431   // original halves. This will then dictate the targets of the cross-half
13432   // shuffles.
13433   auto fixInPlaceInputs =
13434       [&PSHUFDMask](ArrayRef<int> InPlaceInputs, ArrayRef<int> IncomingInputs,
13435                     MutableArrayRef<int> SourceHalfMask,
13436                     MutableArrayRef<int> HalfMask, int HalfOffset) {
13437     if (InPlaceInputs.empty())
13438       return;
13439     if (InPlaceInputs.size() == 1) {
13440       SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
13441           InPlaceInputs[0] - HalfOffset;
13442       PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
13443       return;
13444     }
13445     if (IncomingInputs.empty()) {
13446       // Just fix all of the in place inputs.
13447       for (int Input : InPlaceInputs) {
13448         SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
13449         PSHUFDMask[Input / 2] = Input / 2;
13450       }
13451       return;
13452     }
13453 
13454     assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
13455     SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
13456         InPlaceInputs[0] - HalfOffset;
13457     // Put the second input next to the first so that they are packed into
13458     // a dword. We find the adjacent index by toggling the low bit.
13459     int AdjIndex = InPlaceInputs[0] ^ 1;
13460     SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
13461     std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex);
13462     PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
13463   };
13464   fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0);
13465   fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4);
13466 
13467   // Now gather the cross-half inputs and place them into a free dword of
13468   // their target half.
13469   // FIXME: This operation could almost certainly be simplified dramatically to
13470   // look more like the 3-1 fixing operation.
13471   auto moveInputsToRightHalf = [&PSHUFDMask](
13472       MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
13473       MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
13474       MutableArrayRef<int> FinalSourceHalfMask, int SourceOffset,
13475       int DestOffset) {
13476     auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
13477       return SourceHalfMask[Word] >= 0 && SourceHalfMask[Word] != Word;
13478     };
13479     auto isDWordClobbered = [&isWordClobbered](ArrayRef<int> SourceHalfMask,
13480                                                int Word) {
13481       int LowWord = Word & ~1;
13482       int HighWord = Word | 1;
13483       return isWordClobbered(SourceHalfMask, LowWord) ||
13484              isWordClobbered(SourceHalfMask, HighWord);
13485     };
13486 
13487     if (IncomingInputs.empty())
13488       return;
13489 
13490     if (ExistingInputs.empty()) {
13491       // Map any dwords with inputs from them into the right half.
13492       for (int Input : IncomingInputs) {
13493         // If the source half mask maps over the inputs, turn those into
13494         // swaps and use the swapped lane.
13495         if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) {
13496           if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] < 0) {
13497             SourceHalfMask[SourceHalfMask[Input - SourceOffset]] =
13498                 Input - SourceOffset;
13499             // We have to swap the uses in our half mask in one sweep.
13500             for (int &M : HalfMask)
13501               if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset)
13502                 M = Input;
13503               else if (M == Input)
13504                 M = SourceHalfMask[Input - SourceOffset] + SourceOffset;
13505           } else {
13506             assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] ==
13507                        Input - SourceOffset &&
13508                    "Previous placement doesn't match!");
13509           }
13510           // Note that this correctly re-maps both when we do a swap and when
13511           // we observe the other side of the swap above. We rely on that to
13512           // avoid swapping the members of the input list directly.
13513           Input = SourceHalfMask[Input - SourceOffset] + SourceOffset;
13514         }
13515 
13516         // Map the input's dword into the correct half.
13517         if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] < 0)
13518           PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2;
13519         else
13520           assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] ==
13521                      Input / 2 &&
13522                  "Previous placement doesn't match!");
13523       }
13524 
13525       // And just directly shift any other-half mask elements to be same-half
13526       // as we will have mirrored the dword containing the element into the
13527       // same position within that half.
13528       for (int &M : HalfMask)
13529         if (M >= SourceOffset && M < SourceOffset + 4) {
13530           M = M - SourceOffset + DestOffset;
13531           assert(M >= 0 && "This should never wrap below zero!");
13532         }
13533       return;
13534     }
13535 
13536     // Ensure we have the input in a viable dword of its current half. This
13537     // is particularly tricky because the original position may be clobbered
13538     // by inputs being moved and *staying* in that half.
13539     if (IncomingInputs.size() == 1) {
13540       if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
13541         int InputFixed = find(SourceHalfMask, -1) - std::begin(SourceHalfMask) +
13542                          SourceOffset;
13543         SourceHalfMask[InputFixed - SourceOffset] =
13544             IncomingInputs[0] - SourceOffset;
13545         std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0],
13546                      InputFixed);
13547         IncomingInputs[0] = InputFixed;
13548       }
13549     } else if (IncomingInputs.size() == 2) {
13550       if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 ||
13551           isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
13552         // We have two non-adjacent or clobbered inputs we need to extract from
13553         // the source half. To do this, we need to map them into some adjacent
13554         // dword slot in the source mask.
13555         int InputsFixed[2] = {IncomingInputs[0] - SourceOffset,
13556                               IncomingInputs[1] - SourceOffset};
13557 
13558         // If there is a free slot in the source half mask adjacent to one of
13559         // the inputs, place the other input in it. We use (Index XOR 1) to
13560         // compute an adjacent index.
13561         if (!isWordClobbered(SourceHalfMask, InputsFixed[0]) &&
13562             SourceHalfMask[InputsFixed[0] ^ 1] < 0) {
13563           SourceHalfMask[InputsFixed[0]] = InputsFixed[0];
13564           SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
13565           InputsFixed[1] = InputsFixed[0] ^ 1;
13566         } else if (!isWordClobbered(SourceHalfMask, InputsFixed[1]) &&
13567                    SourceHalfMask[InputsFixed[1] ^ 1] < 0) {
13568           SourceHalfMask[InputsFixed[1]] = InputsFixed[1];
13569           SourceHalfMask[InputsFixed[1] ^ 1] = InputsFixed[0];
13570           InputsFixed[0] = InputsFixed[1] ^ 1;
13571         } else if (SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] < 0 &&
13572                    SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] < 0) {
13573           // The two inputs are in the same DWord but it is clobbered and the
13574           // adjacent DWord isn't used at all. Move both inputs to the free
13575           // slot.
13576           SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] = InputsFixed[0];
13577           SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] = InputsFixed[1];
13578           InputsFixed[0] = 2 * ((InputsFixed[0] / 2) ^ 1);
13579           InputsFixed[1] = 2 * ((InputsFixed[0] / 2) ^ 1) + 1;
13580         } else {
13581           // The only way we hit this point is if there is no clobbering
13582           // (because there are no off-half inputs to this half) and there is no
13583           // free slot adjacent to one of the inputs. In this case, we have to
13584           // swap an input with a non-input.
13585           for (int i = 0; i < 4; ++i)
13586             assert((SourceHalfMask[i] < 0 || SourceHalfMask[i] == i) &&
13587                    "We can't handle any clobbers here!");
13588           assert(InputsFixed[1] != (InputsFixed[0] ^ 1) &&
13589                  "Cannot have adjacent inputs here!");
13590 
13591           SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
13592           SourceHalfMask[InputsFixed[1]] = InputsFixed[0] ^ 1;
13593 
13594           // We also have to update the final source mask in this case because
13595           // it may need to undo the above swap.
13596           for (int &M : FinalSourceHalfMask)
13597             if (M == (InputsFixed[0] ^ 1) + SourceOffset)
13598               M = InputsFixed[1] + SourceOffset;
13599             else if (M == InputsFixed[1] + SourceOffset)
13600               M = (InputsFixed[0] ^ 1) + SourceOffset;
13601 
13602           InputsFixed[1] = InputsFixed[0] ^ 1;
13603         }
13604 
13605         // Point everything at the fixed inputs.
13606         for (int &M : HalfMask)
13607           if (M == IncomingInputs[0])
13608             M = InputsFixed[0] + SourceOffset;
13609           else if (M == IncomingInputs[1])
13610             M = InputsFixed[1] + SourceOffset;
13611 
13612         IncomingInputs[0] = InputsFixed[0] + SourceOffset;
13613         IncomingInputs[1] = InputsFixed[1] + SourceOffset;
13614       }
13615     } else {
13616       llvm_unreachable("Unhandled input size!");
13617     }
13618 
13619     // Now hoist the DWord down to the right half.
13620     int FreeDWord = (PSHUFDMask[DestOffset / 2] < 0 ? 0 : 1) + DestOffset / 2;
13621     assert(PSHUFDMask[FreeDWord] < 0 && "DWord not free");
13622     PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2;
13623     for (int &M : HalfMask)
13624       for (int Input : IncomingInputs)
13625         if (M == Input)
13626           M = FreeDWord * 2 + Input % 2;
13627   };
13628   moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask, HiMask,
13629                         /*SourceOffset*/ 4, /*DestOffset*/ 0);
13630   moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask, LoMask,
13631                         /*SourceOffset*/ 0, /*DestOffset*/ 4);
13632 
13633   // Now enact all the shuffles we've computed to move the inputs into their
13634   // target half.
13635   if (!isNoopShuffleMask(PSHUFLMask))
13636     V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13637                     getV4X86ShuffleImm8ForMask(PSHUFLMask, DL, DAG));
13638   if (!isNoopShuffleMask(PSHUFHMask))
13639     V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13640                     getV4X86ShuffleImm8ForMask(PSHUFHMask, DL, DAG));
13641   if (!isNoopShuffleMask(PSHUFDMask))
13642     V = DAG.getBitcast(
13643         VT,
13644         DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
13645                     getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
13646 
13647   // At this point, each half should contain all its inputs, and we can then
13648   // just shuffle them into their final position.
13649   assert(count_if(LoMask, [](int M) { return M >= 4; }) == 0 &&
13650          "Failed to lift all the high half inputs to the low mask!");
13651   assert(count_if(HiMask, [](int M) { return M >= 0 && M < 4; }) == 0 &&
13652          "Failed to lift all the low half inputs to the high mask!");
13653 
13654   // Do a half shuffle for the low mask.
13655   if (!isNoopShuffleMask(LoMask))
13656     V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13657                     getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
13658 
13659   // Do a half shuffle with the high mask after shifting its values down.
13660   for (int &M : HiMask)
13661     if (M >= 0)
13662       M -= 4;
13663   if (!isNoopShuffleMask(HiMask))
13664     V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13665                     getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
13666 
13667   return V;
13668 }
13669 
13670 /// Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the
13671 /// blend if only one input is used.
13672 static SDValue lowerShuffleAsBlendOfPSHUFBs(
13673     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13674     const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
13675   assert(!is128BitLaneCrossingShuffleMask(VT, Mask) &&
13676          "Lane crossing shuffle masks not supported");
13677 
13678   int NumBytes = VT.getSizeInBits() / 8;
13679   int Size = Mask.size();
13680   int Scale = NumBytes / Size;
13681 
13682   SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));
13683   SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));
13684   V1InUse = false;
13685   V2InUse = false;
13686 
13687   for (int i = 0; i < NumBytes; ++i) {
13688     int M = Mask[i / Scale];
13689     if (M < 0)
13690       continue;
13691 
13692     const int ZeroMask = 0x80;
13693     int V1Idx = M < Size ? M * Scale + i % Scale : ZeroMask;
13694     int V2Idx = M < Size ? ZeroMask : (M - Size) * Scale + i % Scale;
13695     if (Zeroable[i / Scale])
13696       V1Idx = V2Idx = ZeroMask;
13697 
13698     V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8);
13699     V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8);
13700     V1InUse |= (ZeroMask != V1Idx);
13701     V2InUse |= (ZeroMask != V2Idx);
13702   }
13703 
13704   MVT ShufVT = MVT::getVectorVT(MVT::i8, NumBytes);
13705   if (V1InUse)
13706     V1 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V1),
13707                      DAG.getBuildVector(ShufVT, DL, V1Mask));
13708   if (V2InUse)
13709     V2 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V2),
13710                      DAG.getBuildVector(ShufVT, DL, V2Mask));
13711 
13712   // If we need shuffled inputs from both, blend the two.
13713   SDValue V;
13714   if (V1InUse && V2InUse)
13715     V = DAG.getNode(ISD::OR, DL, ShufVT, V1, V2);
13716   else
13717     V = V1InUse ? V1 : V2;
13718 
13719   // Cast the result back to the correct type.
13720   return DAG.getBitcast(VT, V);
13721 }
13722 
13723 /// Generic lowering of 8-lane i16 shuffles.
13724 ///
13725 /// This handles both single-input shuffles and combined shuffle/blends with
13726 /// two inputs. The single input shuffles are immediately delegated to
13727 /// a dedicated lowering routine.
13728 ///
13729 /// The blends are lowered in one of three fundamental ways. If there are few
13730 /// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle
13731 /// of the input is significantly cheaper when lowered as an interleaving of
13732 /// the two inputs, try to interleave them. Otherwise, blend the low and high
13733 /// halves of the inputs separately (making them have relatively few inputs)
13734 /// and then concatenate them.
13735 static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13736                                  const APInt &Zeroable, SDValue V1, SDValue V2,
13737                                  const X86Subtarget &Subtarget,
13738                                  SelectionDAG &DAG) {
13739   assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
13740   assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
13741   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
13742 
13743   // Whenever we can lower this as a zext, that instruction is strictly faster
13744   // than any alternative.
13745   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
13746                                                    Zeroable, Subtarget, DAG))
13747     return ZExt;
13748 
13749   // Try to use lower using a truncation.
13750   if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
13751                                         Subtarget, DAG))
13752     return V;
13753 
13754   int NumV2Inputs = count_if(Mask, [](int M) { return M >= 8; });
13755 
13756   if (NumV2Inputs == 0) {
13757     // Try to use shift instructions.
13758     if (SDValue Shift =
13759             lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask, Zeroable,
13760                                 Subtarget, DAG, /*BitwiseOnly*/ false))
13761       return Shift;
13762 
13763     // Check for being able to broadcast a single element.
13764     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
13765                                                     Mask, Subtarget, DAG))
13766       return Broadcast;
13767 
13768     // Try to use bit rotation instructions.
13769     if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,
13770                                                  Subtarget, DAG))
13771       return Rotate;
13772 
13773     // Use dedicated unpack instructions for masks that match their pattern.
13774     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
13775       return V;
13776 
13777     // Use dedicated pack instructions for masks that match their pattern.
13778     if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
13779                                          Subtarget))
13780       return V;
13781 
13782     // Try to use byte rotation instructions.
13783     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
13784                                                   Subtarget, DAG))
13785       return Rotate;
13786 
13787     // Make a copy of the mask so it can be modified.
13788     SmallVector<int, 8> MutableMask(Mask);
13789     return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v8i16, V1, MutableMask,
13790                                                Subtarget, DAG);
13791   }
13792 
13793   assert(llvm::any_of(Mask, [](int M) { return M >= 0 && M < 8; }) &&
13794          "All single-input shuffles should be canonicalized to be V1-input "
13795          "shuffles.");
13796 
13797   // Try to use shift instructions.
13798   if (SDValue Shift =
13799           lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget,
13800                               DAG, /*BitwiseOnly*/ false))
13801     return Shift;
13802 
13803   // See if we can use SSE4A Extraction / Insertion.
13804   if (Subtarget.hasSSE4A())
13805     if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
13806                                           Zeroable, DAG))
13807       return V;
13808 
13809   // There are special ways we can lower some single-element blends.
13810   if (NumV2Inputs == 1)
13811     if (SDValue V = lowerShuffleAsElementInsertion(
13812             DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
13813       return V;
13814 
13815   // We have different paths for blend lowering, but they all must use the
13816   // *exact* same predicate.
13817   bool IsBlendSupported = Subtarget.hasSSE41();
13818   if (IsBlendSupported)
13819     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
13820                                             Zeroable, Subtarget, DAG))
13821       return Blend;
13822 
13823   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
13824                                              Zeroable, Subtarget, DAG))
13825     return Masked;
13826 
13827   // Use dedicated unpack instructions for masks that match their pattern.
13828   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
13829     return V;
13830 
13831   // Use dedicated pack instructions for masks that match their pattern.
13832   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
13833                                        Subtarget))
13834     return V;
13835 
13836   // Try to use lower using a truncation.
13837   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
13838                                        Subtarget, DAG))
13839     return V;
13840 
13841   // Try to use byte rotation instructions.
13842   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
13843                                                 Subtarget, DAG))
13844     return Rotate;
13845 
13846   if (SDValue BitBlend =
13847           lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
13848     return BitBlend;
13849 
13850   // Try to use byte shift instructions to mask.
13851   if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v8i16, V1, V2, Mask,
13852                                               Zeroable, Subtarget, DAG))
13853     return V;
13854 
13855   // Attempt to lower using compaction, SSE41 is necessary for PACKUSDW.
13856   int NumEvenDrops = canLowerByDroppingElements(Mask, true, false);
13857   if ((NumEvenDrops == 1 || (NumEvenDrops == 2 && Subtarget.hasSSE41())) &&
13858       !Subtarget.hasVLX()) {
13859     // Check if this is part of a 256-bit vector truncation.
13860     unsigned PackOpc = 0;
13861     if (NumEvenDrops == 2 && Subtarget.hasAVX2() &&
13862         peekThroughBitcasts(V1).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
13863         peekThroughBitcasts(V2).getOpcode() == ISD::EXTRACT_SUBVECTOR) {
13864       SDValue V1V2 = concatSubVectors(V1, V2, DAG, DL);
13865       V1V2 = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1V2,
13866                          getZeroVector(MVT::v16i16, Subtarget, DAG, DL),
13867                          DAG.getTargetConstant(0xEE, DL, MVT::i8));
13868       V1V2 = DAG.getBitcast(MVT::v8i32, V1V2);
13869       V1 = extract128BitVector(V1V2, 0, DAG, DL);
13870       V2 = extract128BitVector(V1V2, 4, DAG, DL);
13871       PackOpc = X86ISD::PACKUS;
13872     } else if (Subtarget.hasSSE41()) {
13873       SmallVector<SDValue, 4> DWordClearOps(4,
13874                                             DAG.getConstant(0, DL, MVT::i32));
13875       for (unsigned i = 0; i != 4; i += 1 << (NumEvenDrops - 1))
13876         DWordClearOps[i] = DAG.getConstant(0xFFFF, DL, MVT::i32);
13877       SDValue DWordClearMask =
13878           DAG.getBuildVector(MVT::v4i32, DL, DWordClearOps);
13879       V1 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V1),
13880                        DWordClearMask);
13881       V2 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V2),
13882                        DWordClearMask);
13883       PackOpc = X86ISD::PACKUS;
13884     } else if (!Subtarget.hasSSSE3()) {
13885       SDValue ShAmt = DAG.getTargetConstant(16, DL, MVT::i8);
13886       V1 = DAG.getBitcast(MVT::v4i32, V1);
13887       V2 = DAG.getBitcast(MVT::v4i32, V2);
13888       V1 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V1, ShAmt);
13889       V2 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V2, ShAmt);
13890       V1 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V1, ShAmt);
13891       V2 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V2, ShAmt);
13892       PackOpc = X86ISD::PACKSS;
13893     }
13894     if (PackOpc) {
13895       // Now pack things back together.
13896       SDValue Result = DAG.getNode(PackOpc, DL, MVT::v8i16, V1, V2);
13897       if (NumEvenDrops == 2) {
13898         Result = DAG.getBitcast(MVT::v4i32, Result);
13899         Result = DAG.getNode(PackOpc, DL, MVT::v8i16, Result, Result);
13900       }
13901       return Result;
13902     }
13903   }
13904 
13905   // When compacting odd (upper) elements, use PACKSS pre-SSE41.
13906   int NumOddDrops = canLowerByDroppingElements(Mask, false, false);
13907   if (NumOddDrops == 1) {
13908     bool HasSSE41 = Subtarget.hasSSE41();
13909     V1 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,
13910                      DAG.getBitcast(MVT::v4i32, V1),
13911                      DAG.getTargetConstant(16, DL, MVT::i8));
13912     V2 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,
13913                      DAG.getBitcast(MVT::v4i32, V2),
13914                      DAG.getTargetConstant(16, DL, MVT::i8));
13915     return DAG.getNode(HasSSE41 ? X86ISD::PACKUS : X86ISD::PACKSS, DL,
13916                        MVT::v8i16, V1, V2);
13917   }
13918 
13919   // Try to lower by permuting the inputs into an unpack instruction.
13920   if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
13921                                                       Mask, Subtarget, DAG))
13922     return Unpack;
13923 
13924   // If we can't directly blend but can use PSHUFB, that will be better as it
13925   // can both shuffle and set up the inefficient blend.
13926   if (!IsBlendSupported && Subtarget.hasSSSE3()) {
13927     bool V1InUse, V2InUse;
13928     return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
13929                                         Zeroable, DAG, V1InUse, V2InUse);
13930   }
13931 
13932   // We can always bit-blend if we have to so the fallback strategy is to
13933   // decompose into single-input permutes and blends/unpacks.
13934   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2,
13935                                               Mask, Subtarget, DAG);
13936 }
13937 
13938 /// Lower 8-lane 16-bit floating point shuffles.
13939 static SDValue lowerV8F16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13940                                  const APInt &Zeroable, SDValue V1, SDValue V2,
13941                                  const X86Subtarget &Subtarget,
13942                                  SelectionDAG &DAG) {
13943   assert(V1.getSimpleValueType() == MVT::v8f16 && "Bad operand type!");
13944   assert(V2.getSimpleValueType() == MVT::v8f16 && "Bad operand type!");
13945   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
13946   int NumV2Elements = count_if(Mask, [](int M) { return M >= 8; });
13947 
13948   if (Subtarget.hasFP16()) {
13949     if (NumV2Elements == 0) {
13950       // Check for being able to broadcast a single element.
13951       if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f16, V1, V2,
13952                                                       Mask, Subtarget, DAG))
13953         return Broadcast;
13954     }
13955     if (NumV2Elements == 1 && Mask[0] >= 8)
13956       if (SDValue V = lowerShuffleAsElementInsertion(
13957               DL, MVT::v8f16, V1, V2, Mask, Zeroable, Subtarget, DAG))
13958         return V;
13959   }
13960 
13961   V1 = DAG.getBitcast(MVT::v8i16, V1);
13962   V2 = DAG.getBitcast(MVT::v8i16, V2);
13963   return DAG.getBitcast(MVT::v8f16,
13964                         DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));
13965 }
13966 
13967 // Lowers unary/binary shuffle as VPERMV/VPERMV3, for non-VLX targets,
13968 // sub-512-bit shuffles are padded to 512-bits for the shuffle and then
13969 // the active subvector is extracted.
13970 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
13971                                      ArrayRef<int> Mask, SDValue V1, SDValue V2,
13972                                      const X86Subtarget &Subtarget,
13973                                      SelectionDAG &DAG) {
13974   MVT MaskVT = VT.changeTypeToInteger();
13975   SDValue MaskNode;
13976   MVT ShuffleVT = VT;
13977   if (!VT.is512BitVector() && !Subtarget.hasVLX()) {
13978     V1 = widenSubVector(V1, false, Subtarget, DAG, DL, 512);
13979     V2 = widenSubVector(V2, false, Subtarget, DAG, DL, 512);
13980     ShuffleVT = V1.getSimpleValueType();
13981 
13982     // Adjust mask to correct indices for the second input.
13983     int NumElts = VT.getVectorNumElements();
13984     unsigned Scale = 512 / VT.getSizeInBits();
13985     SmallVector<int, 32> AdjustedMask(Mask);
13986     for (int &M : AdjustedMask)
13987       if (NumElts <= M)
13988         M += (Scale - 1) * NumElts;
13989     MaskNode = getConstVector(AdjustedMask, MaskVT, DAG, DL, true);
13990     MaskNode = widenSubVector(MaskNode, false, Subtarget, DAG, DL, 512);
13991   } else {
13992     MaskNode = getConstVector(Mask, MaskVT, DAG, DL, true);
13993   }
13994 
13995   SDValue Result;
13996   if (V2.isUndef())
13997     Result = DAG.getNode(X86ISD::VPERMV, DL, ShuffleVT, MaskNode, V1);
13998   else
13999     Result = DAG.getNode(X86ISD::VPERMV3, DL, ShuffleVT, V1, MaskNode, V2);
14000 
14001   if (VT != ShuffleVT)
14002     Result = extractSubVector(Result, 0, DAG, DL, VT.getSizeInBits());
14003 
14004   return Result;
14005 }
14006 
14007 /// Generic lowering of v16i8 shuffles.
14008 ///
14009 /// This is a hybrid strategy to lower v16i8 vectors. It first attempts to
14010 /// detect any complexity reducing interleaving. If that doesn't help, it uses
14011 /// UNPCK to spread the i8 elements across two i16-element vectors, and uses
14012 /// the existing lowering for v8i16 blends on each half, finally PACK-ing them
14013 /// back together.
14014 static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14015                                  const APInt &Zeroable, SDValue V1, SDValue V2,
14016                                  const X86Subtarget &Subtarget,
14017                                  SelectionDAG &DAG) {
14018   assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
14019   assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
14020   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
14021 
14022   // Try to use shift instructions.
14023   if (SDValue Shift =
14024           lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget,
14025                               DAG, /*BitwiseOnly*/ false))
14026     return Shift;
14027 
14028   // Try to use byte rotation instructions.
14029   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
14030                                                 Subtarget, DAG))
14031     return Rotate;
14032 
14033   // Use dedicated pack instructions for masks that match their pattern.
14034   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
14035                                        Subtarget))
14036     return V;
14037 
14038   // Try to use a zext lowering.
14039   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
14040                                                    Zeroable, Subtarget, DAG))
14041     return ZExt;
14042 
14043   // Try to use lower using a truncation.
14044   if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14045                                         Subtarget, DAG))
14046     return V;
14047 
14048   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14049                                        Subtarget, DAG))
14050     return V;
14051 
14052   // See if we can use SSE4A Extraction / Insertion.
14053   if (Subtarget.hasSSE4A())
14054     if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
14055                                           Zeroable, DAG))
14056       return V;
14057 
14058   int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
14059 
14060   // For single-input shuffles, there are some nicer lowering tricks we can use.
14061   if (NumV2Elements == 0) {
14062     // Check for being able to broadcast a single element.
14063     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
14064                                                     Mask, Subtarget, DAG))
14065       return Broadcast;
14066 
14067     // Try to use bit rotation instructions.
14068     if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,
14069                                                  Subtarget, DAG))
14070       return Rotate;
14071 
14072     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14073       return V;
14074 
14075     // Check whether we can widen this to an i16 shuffle by duplicating bytes.
14076     // Notably, this handles splat and partial-splat shuffles more efficiently.
14077     // However, it only makes sense if the pre-duplication shuffle simplifies
14078     // things significantly. Currently, this means we need to be able to
14079     // express the pre-duplication shuffle as an i16 shuffle.
14080     //
14081     // FIXME: We should check for other patterns which can be widened into an
14082     // i16 shuffle as well.
14083     auto canWidenViaDuplication = [](ArrayRef<int> Mask) {
14084       for (int i = 0; i < 16; i += 2)
14085         if (Mask[i] >= 0 && Mask[i + 1] >= 0 && Mask[i] != Mask[i + 1])
14086           return false;
14087 
14088       return true;
14089     };
14090     auto tryToWidenViaDuplication = [&]() -> SDValue {
14091       if (!canWidenViaDuplication(Mask))
14092         return SDValue();
14093       SmallVector<int, 4> LoInputs;
14094       copy_if(Mask, std::back_inserter(LoInputs),
14095               [](int M) { return M >= 0 && M < 8; });
14096       array_pod_sort(LoInputs.begin(), LoInputs.end());
14097       LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()),
14098                      LoInputs.end());
14099       SmallVector<int, 4> HiInputs;
14100       copy_if(Mask, std::back_inserter(HiInputs), [](int M) { return M >= 8; });
14101       array_pod_sort(HiInputs.begin(), HiInputs.end());
14102       HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()),
14103                      HiInputs.end());
14104 
14105       bool TargetLo = LoInputs.size() >= HiInputs.size();
14106       ArrayRef<int> InPlaceInputs = TargetLo ? LoInputs : HiInputs;
14107       ArrayRef<int> MovingInputs = TargetLo ? HiInputs : LoInputs;
14108 
14109       int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
14110       SmallDenseMap<int, int, 8> LaneMap;
14111       for (int I : InPlaceInputs) {
14112         PreDupI16Shuffle[I/2] = I/2;
14113         LaneMap[I] = I;
14114       }
14115       int j = TargetLo ? 0 : 4, je = j + 4;
14116       for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) {
14117         // Check if j is already a shuffle of this input. This happens when
14118         // there are two adjacent bytes after we move the low one.
14119         if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) {
14120           // If we haven't yet mapped the input, search for a slot into which
14121           // we can map it.
14122           while (j < je && PreDupI16Shuffle[j] >= 0)
14123             ++j;
14124 
14125           if (j == je)
14126             // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
14127             return SDValue();
14128 
14129           // Map this input with the i16 shuffle.
14130           PreDupI16Shuffle[j] = MovingInputs[i] / 2;
14131         }
14132 
14133         // Update the lane map based on the mapping we ended up with.
14134         LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2;
14135       }
14136       V1 = DAG.getBitcast(
14137           MVT::v16i8,
14138           DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14139                                DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
14140 
14141       // Unpack the bytes to form the i16s that will be shuffled into place.
14142       bool EvenInUse = false, OddInUse = false;
14143       for (int i = 0; i < 16; i += 2) {
14144         EvenInUse |= (Mask[i + 0] >= 0);
14145         OddInUse |= (Mask[i + 1] >= 0);
14146         if (EvenInUse && OddInUse)
14147           break;
14148       }
14149       V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
14150                        MVT::v16i8, EvenInUse ? V1 : DAG.getUNDEF(MVT::v16i8),
14151                        OddInUse ? V1 : DAG.getUNDEF(MVT::v16i8));
14152 
14153       int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
14154       for (int i = 0; i < 16; ++i)
14155         if (Mask[i] >= 0) {
14156           int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8);
14157           assert(MappedMask < 8 && "Invalid v8 shuffle mask!");
14158           if (PostDupI16Shuffle[i / 2] < 0)
14159             PostDupI16Shuffle[i / 2] = MappedMask;
14160           else
14161             assert(PostDupI16Shuffle[i / 2] == MappedMask &&
14162                    "Conflicting entries in the original shuffle!");
14163         }
14164       return DAG.getBitcast(
14165           MVT::v16i8,
14166           DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14167                                DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
14168     };
14169     if (SDValue V = tryToWidenViaDuplication())
14170       return V;
14171   }
14172 
14173   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
14174                                              Zeroable, Subtarget, DAG))
14175     return Masked;
14176 
14177   // Use dedicated unpack instructions for masks that match their pattern.
14178   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14179     return V;
14180 
14181   // Try to use byte shift instructions to mask.
14182   if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v16i8, V1, V2, Mask,
14183                                               Zeroable, Subtarget, DAG))
14184     return V;
14185 
14186   // Check for compaction patterns.
14187   bool IsSingleInput = V2.isUndef();
14188   int NumEvenDrops = canLowerByDroppingElements(Mask, true, IsSingleInput);
14189 
14190   // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly
14191   // with PSHUFB. It is important to do this before we attempt to generate any
14192   // blends but after all of the single-input lowerings. If the single input
14193   // lowerings can find an instruction sequence that is faster than a PSHUFB, we
14194   // want to preserve that and we can DAG combine any longer sequences into
14195   // a PSHUFB in the end. But once we start blending from multiple inputs,
14196   // the complexity of DAG combining bad patterns back into PSHUFB is too high,
14197   // and there are *very* few patterns that would actually be faster than the
14198   // PSHUFB approach because of its ability to zero lanes.
14199   //
14200   // If the mask is a binary compaction, we can more efficiently perform this
14201   // as a PACKUS(AND(),AND()) - which is quicker than UNPACK(PSHUFB(),PSHUFB()).
14202   //
14203   // FIXME: The only exceptions to the above are blends which are exact
14204   // interleavings with direct instructions supporting them. We currently don't
14205   // handle those well here.
14206   if (Subtarget.hasSSSE3() && (IsSingleInput || NumEvenDrops != 1)) {
14207     bool V1InUse = false;
14208     bool V2InUse = false;
14209 
14210     SDValue PSHUFB = lowerShuffleAsBlendOfPSHUFBs(
14211         DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
14212 
14213     // If both V1 and V2 are in use and we can use a direct blend or an unpack,
14214     // do so. This avoids using them to handle blends-with-zero which is
14215     // important as a single pshufb is significantly faster for that.
14216     if (V1InUse && V2InUse) {
14217       if (Subtarget.hasSSE41())
14218         if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
14219                                                 Zeroable, Subtarget, DAG))
14220           return Blend;
14221 
14222       // We can use an unpack to do the blending rather than an or in some
14223       // cases. Even though the or may be (very minorly) more efficient, we
14224       // preference this lowering because there are common cases where part of
14225       // the complexity of the shuffles goes away when we do the final blend as
14226       // an unpack.
14227       // FIXME: It might be worth trying to detect if the unpack-feeding
14228       // shuffles will both be pshufb, in which case we shouldn't bother with
14229       // this.
14230       if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(
14231               DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14232         return Unpack;
14233 
14234       // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8).
14235       if (Subtarget.hasVBMI())
14236         return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, Subtarget,
14237                                      DAG);
14238 
14239       // If we have XOP we can use one VPPERM instead of multiple PSHUFBs.
14240       if (Subtarget.hasXOP()) {
14241         SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true);
14242         return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode);
14243       }
14244 
14245       // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
14246       // PALIGNR will be cheaper than the second PSHUFB+OR.
14247       if (SDValue V = lowerShuffleAsByteRotateAndPermute(
14248               DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14249         return V;
14250     }
14251 
14252     return PSHUFB;
14253   }
14254 
14255   // There are special ways we can lower some single-element blends.
14256   if (NumV2Elements == 1)
14257     if (SDValue V = lowerShuffleAsElementInsertion(
14258             DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
14259       return V;
14260 
14261   if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
14262     return Blend;
14263 
14264   // Check whether a compaction lowering can be done. This handles shuffles
14265   // which take every Nth element for some even N. See the helper function for
14266   // details.
14267   //
14268   // We special case these as they can be particularly efficiently handled with
14269   // the PACKUSB instruction on x86 and they show up in common patterns of
14270   // rearranging bytes to truncate wide elements.
14271   if (NumEvenDrops) {
14272     // NumEvenDrops is the power of two stride of the elements. Another way of
14273     // thinking about it is that we need to drop the even elements this many
14274     // times to get the original input.
14275 
14276     // First we need to zero all the dropped bytes.
14277     assert(NumEvenDrops <= 3 &&
14278            "No support for dropping even elements more than 3 times.");
14279     SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));
14280     for (unsigned i = 0; i != 8; i += 1 << (NumEvenDrops - 1))
14281       WordClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i16);
14282     SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);
14283     V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1),
14284                      WordClearMask);
14285     if (!IsSingleInput)
14286       V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2),
14287                        WordClearMask);
14288 
14289     // Now pack things back together.
14290     SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14291                                  IsSingleInput ? V1 : V2);
14292     for (int i = 1; i < NumEvenDrops; ++i) {
14293       Result = DAG.getBitcast(MVT::v8i16, Result);
14294       Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
14295     }
14296     return Result;
14297   }
14298 
14299   int NumOddDrops = canLowerByDroppingElements(Mask, false, IsSingleInput);
14300   if (NumOddDrops == 1) {
14301     V1 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,
14302                      DAG.getBitcast(MVT::v8i16, V1),
14303                      DAG.getTargetConstant(8, DL, MVT::i8));
14304     if (!IsSingleInput)
14305       V2 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,
14306                        DAG.getBitcast(MVT::v8i16, V2),
14307                        DAG.getTargetConstant(8, DL, MVT::i8));
14308     return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14309                        IsSingleInput ? V1 : V2);
14310   }
14311 
14312   // Handle multi-input cases by blending/unpacking single-input shuffles.
14313   if (NumV2Elements > 0)
14314     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v16i8, V1, V2, Mask,
14315                                                 Subtarget, DAG);
14316 
14317   // The fallback path for single-input shuffles widens this into two v8i16
14318   // vectors with unpacks, shuffles those, and then pulls them back together
14319   // with a pack.
14320   SDValue V = V1;
14321 
14322   std::array<int, 8> LoBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}};
14323   std::array<int, 8> HiBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}};
14324   for (int i = 0; i < 16; ++i)
14325     if (Mask[i] >= 0)
14326       (i < 8 ? LoBlendMask[i] : HiBlendMask[i % 8]) = Mask[i];
14327 
14328   SDValue VLoHalf, VHiHalf;
14329   // Check if any of the odd lanes in the v16i8 are used. If not, we can mask
14330   // them out and avoid using UNPCK{L,H} to extract the elements of V as
14331   // i16s.
14332   if (none_of(LoBlendMask, [](int M) { return M >= 0 && M % 2 == 1; }) &&
14333       none_of(HiBlendMask, [](int M) { return M >= 0 && M % 2 == 1; })) {
14334     // Use a mask to drop the high bytes.
14335     VLoHalf = DAG.getBitcast(MVT::v8i16, V);
14336     VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf,
14337                           DAG.getConstant(0x00FF, DL, MVT::v8i16));
14338 
14339     // This will be a single vector shuffle instead of a blend so nuke VHiHalf.
14340     VHiHalf = DAG.getUNDEF(MVT::v8i16);
14341 
14342     // Squash the masks to point directly into VLoHalf.
14343     for (int &M : LoBlendMask)
14344       if (M >= 0)
14345         M /= 2;
14346     for (int &M : HiBlendMask)
14347       if (M >= 0)
14348         M /= 2;
14349   } else {
14350     // Otherwise just unpack the low half of V into VLoHalf and the high half into
14351     // VHiHalf so that we can blend them as i16s.
14352     SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
14353 
14354     VLoHalf = DAG.getBitcast(
14355         MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
14356     VHiHalf = DAG.getBitcast(
14357         MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
14358   }
14359 
14360   SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
14361   SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
14362 
14363   return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
14364 }
14365 
14366 /// Dispatching routine to lower various 128-bit x86 vector shuffles.
14367 ///
14368 /// This routine breaks down the specific type of 128-bit shuffle and
14369 /// dispatches to the lowering routines accordingly.
14370 static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
14371                                   MVT VT, SDValue V1, SDValue V2,
14372                                   const APInt &Zeroable,
14373                                   const X86Subtarget &Subtarget,
14374                                   SelectionDAG &DAG) {
14375   if (VT == MVT::v8bf16) {
14376     V1 = DAG.getBitcast(MVT::v8i16, V1);
14377     V2 = DAG.getBitcast(MVT::v8i16, V2);
14378     return DAG.getBitcast(VT,
14379                           DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));
14380   }
14381 
14382   switch (VT.SimpleTy) {
14383   case MVT::v2i64:
14384     return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14385   case MVT::v2f64:
14386     return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14387   case MVT::v4i32:
14388     return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14389   case MVT::v4f32:
14390     return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14391   case MVT::v8i16:
14392     return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14393   case MVT::v8f16:
14394     return lowerV8F16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14395   case MVT::v16i8:
14396     return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14397 
14398   default:
14399     llvm_unreachable("Unimplemented!");
14400   }
14401 }
14402 
14403 /// Generic routine to split vector shuffle into half-sized shuffles.
14404 ///
14405 /// This routine just extracts two subvectors, shuffles them independently, and
14406 /// then concatenates them back together. This should work effectively with all
14407 /// AVX vector shuffle types.
14408 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
14409                                     SDValue V2, ArrayRef<int> Mask,
14410                                     SelectionDAG &DAG, bool SimpleOnly) {
14411   assert(VT.getSizeInBits() >= 256 &&
14412          "Only for 256-bit or wider vector shuffles!");
14413   assert(V1.getSimpleValueType() == VT && "Bad operand type!");
14414   assert(V2.getSimpleValueType() == VT && "Bad operand type!");
14415 
14416   ArrayRef<int> LoMask = Mask.slice(0, Mask.size() / 2);
14417   ArrayRef<int> HiMask = Mask.slice(Mask.size() / 2);
14418 
14419   int NumElements = VT.getVectorNumElements();
14420   int SplitNumElements = NumElements / 2;
14421   MVT ScalarVT = VT.getVectorElementType();
14422   MVT SplitVT = MVT::getVectorVT(ScalarVT, SplitNumElements);
14423 
14424   // Use splitVector/extractSubVector so that split build-vectors just build two
14425   // narrower build vectors. This helps shuffling with splats and zeros.
14426   auto SplitVector = [&](SDValue V) {
14427     SDValue LoV, HiV;
14428     std::tie(LoV, HiV) = splitVector(peekThroughBitcasts(V), DAG, DL);
14429     return std::make_pair(DAG.getBitcast(SplitVT, LoV),
14430                           DAG.getBitcast(SplitVT, HiV));
14431   };
14432 
14433   SDValue LoV1, HiV1, LoV2, HiV2;
14434   std::tie(LoV1, HiV1) = SplitVector(V1);
14435   std::tie(LoV2, HiV2) = SplitVector(V2);
14436 
14437   // Now create two 4-way blends of these half-width vectors.
14438   auto GetHalfBlendPiecesReq = [&](const ArrayRef<int> &HalfMask, bool &UseLoV1,
14439                                    bool &UseHiV1, bool &UseLoV2,
14440                                    bool &UseHiV2) {
14441     UseLoV1 = UseHiV1 = UseLoV2 = UseHiV2 = false;
14442     for (int i = 0; i < SplitNumElements; ++i) {
14443       int M = HalfMask[i];
14444       if (M >= NumElements) {
14445         if (M >= NumElements + SplitNumElements)
14446           UseHiV2 = true;
14447         else
14448           UseLoV2 = true;
14449       } else if (M >= 0) {
14450         if (M >= SplitNumElements)
14451           UseHiV1 = true;
14452         else
14453           UseLoV1 = true;
14454       }
14455     }
14456   };
14457 
14458   auto CheckHalfBlendUsable = [&](const ArrayRef<int> &HalfMask) -> bool {
14459     if (!SimpleOnly)
14460       return true;
14461 
14462     bool UseLoV1, UseHiV1, UseLoV2, UseHiV2;
14463     GetHalfBlendPiecesReq(HalfMask, UseLoV1, UseHiV1, UseLoV2, UseHiV2);
14464 
14465     return !(UseHiV1 || UseHiV2);
14466   };
14467 
14468   auto HalfBlend = [&](ArrayRef<int> HalfMask) {
14469     SmallVector<int, 32> V1BlendMask((unsigned)SplitNumElements, -1);
14470     SmallVector<int, 32> V2BlendMask((unsigned)SplitNumElements, -1);
14471     SmallVector<int, 32> BlendMask((unsigned)SplitNumElements, -1);
14472     for (int i = 0; i < SplitNumElements; ++i) {
14473       int M = HalfMask[i];
14474       if (M >= NumElements) {
14475         V2BlendMask[i] = M - NumElements;
14476         BlendMask[i] = SplitNumElements + i;
14477       } else if (M >= 0) {
14478         V1BlendMask[i] = M;
14479         BlendMask[i] = i;
14480       }
14481     }
14482 
14483     bool UseLoV1, UseHiV1, UseLoV2, UseHiV2;
14484     GetHalfBlendPiecesReq(HalfMask, UseLoV1, UseHiV1, UseLoV2, UseHiV2);
14485 
14486     // Because the lowering happens after all combining takes place, we need to
14487     // manually combine these blend masks as much as possible so that we create
14488     // a minimal number of high-level vector shuffle nodes.
14489     assert((!SimpleOnly || (!UseHiV1 && !UseHiV2)) && "Shuffle isn't simple");
14490 
14491     // First try just blending the halves of V1 or V2.
14492     if (!UseLoV1 && !UseHiV1 && !UseLoV2 && !UseHiV2)
14493       return DAG.getUNDEF(SplitVT);
14494     if (!UseLoV2 && !UseHiV2)
14495       return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14496     if (!UseLoV1 && !UseHiV1)
14497       return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14498 
14499     SDValue V1Blend, V2Blend;
14500     if (UseLoV1 && UseHiV1) {
14501       V1Blend = DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14502     } else {
14503       // We only use half of V1 so map the usage down into the final blend mask.
14504       V1Blend = UseLoV1 ? LoV1 : HiV1;
14505       for (int i = 0; i < SplitNumElements; ++i)
14506         if (BlendMask[i] >= 0 && BlendMask[i] < SplitNumElements)
14507           BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements);
14508     }
14509     if (UseLoV2 && UseHiV2) {
14510       V2Blend = DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14511     } else {
14512       // We only use half of V2 so map the usage down into the final blend mask.
14513       V2Blend = UseLoV2 ? LoV2 : HiV2;
14514       for (int i = 0; i < SplitNumElements; ++i)
14515         if (BlendMask[i] >= SplitNumElements)
14516           BlendMask[i] = V2BlendMask[i] + (UseLoV2 ? SplitNumElements : 0);
14517     }
14518     return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask);
14519   };
14520 
14521   if (!CheckHalfBlendUsable(LoMask) || !CheckHalfBlendUsable(HiMask))
14522     return SDValue();
14523 
14524   SDValue Lo = HalfBlend(LoMask);
14525   SDValue Hi = HalfBlend(HiMask);
14526   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
14527 }
14528 
14529 /// Either split a vector in halves or decompose the shuffles and the
14530 /// blend/unpack.
14531 ///
14532 /// This is provided as a good fallback for many lowerings of non-single-input
14533 /// shuffles with more than one 128-bit lane. In those cases, we want to select
14534 /// between splitting the shuffle into 128-bit components and stitching those
14535 /// back together vs. extracting the single-input shuffles and blending those
14536 /// results.
14537 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
14538                                           SDValue V2, ArrayRef<int> Mask,
14539                                           const X86Subtarget &Subtarget,
14540                                           SelectionDAG &DAG) {
14541   assert(!V2.isUndef() && "This routine must not be used to lower single-input "
14542          "shuffles as it could then recurse on itself.");
14543   int Size = Mask.size();
14544 
14545   // If this can be modeled as a broadcast of two elements followed by a blend,
14546   // prefer that lowering. This is especially important because broadcasts can
14547   // often fold with memory operands.
14548   auto DoBothBroadcast = [&] {
14549     int V1BroadcastIdx = -1, V2BroadcastIdx = -1;
14550     for (int M : Mask)
14551       if (M >= Size) {
14552         if (V2BroadcastIdx < 0)
14553           V2BroadcastIdx = M - Size;
14554         else if (M - Size != V2BroadcastIdx)
14555           return false;
14556       } else if (M >= 0) {
14557         if (V1BroadcastIdx < 0)
14558           V1BroadcastIdx = M;
14559         else if (M != V1BroadcastIdx)
14560           return false;
14561       }
14562     return true;
14563   };
14564   if (DoBothBroadcast())
14565     return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget,
14566                                                 DAG);
14567 
14568   // If the inputs all stem from a single 128-bit lane of each input, then we
14569   // split them rather than blending because the split will decompose to
14570   // unusually few instructions.
14571   int LaneCount = VT.getSizeInBits() / 128;
14572   int LaneSize = Size / LaneCount;
14573   SmallBitVector LaneInputs[2];
14574   LaneInputs[0].resize(LaneCount, false);
14575   LaneInputs[1].resize(LaneCount, false);
14576   for (int i = 0; i < Size; ++i)
14577     if (Mask[i] >= 0)
14578       LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true;
14579   if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1)
14580     return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
14581                                 /*SimpleOnly*/ false);
14582 
14583   // Otherwise, just fall back to decomposed shuffles and a blend/unpack. This
14584   // requires that the decomposed single-input shuffles don't end up here.
14585   return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget,
14586                                               DAG);
14587 }
14588 
14589 // Lower as SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
14590 // TODO: Extend to support v8f32 (+ 512-bit shuffles).
14591 static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT,
14592                                                  SDValue V1, SDValue V2,
14593                                                  ArrayRef<int> Mask,
14594                                                  SelectionDAG &DAG) {
14595   assert(VT == MVT::v4f64 && "Only for v4f64 shuffles");
14596 
14597   int LHSMask[4] = {-1, -1, -1, -1};
14598   int RHSMask[4] = {-1, -1, -1, -1};
14599   unsigned SHUFPMask = 0;
14600 
14601   // As SHUFPD uses a single LHS/RHS element per lane, we can always
14602   // perform the shuffle once the lanes have been shuffled in place.
14603   for (int i = 0; i != 4; ++i) {
14604     int M = Mask[i];
14605     if (M < 0)
14606       continue;
14607     int LaneBase = i & ~1;
14608     auto &LaneMask = (i & 1) ? RHSMask : LHSMask;
14609     LaneMask[LaneBase + (M & 1)] = M;
14610     SHUFPMask |= (M & 1) << i;
14611   }
14612 
14613   SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask);
14614   SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask);
14615   return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS,
14616                      DAG.getTargetConstant(SHUFPMask, DL, MVT::i8));
14617 }
14618 
14619 /// Lower a vector shuffle crossing multiple 128-bit lanes as
14620 /// a lane permutation followed by a per-lane permutation.
14621 ///
14622 /// This is mainly for cases where we can have non-repeating permutes
14623 /// in each lane.
14624 ///
14625 /// TODO: This is very similar to lowerShuffleAsLanePermuteAndRepeatedMask,
14626 /// we should investigate merging them.
14627 static SDValue lowerShuffleAsLanePermuteAndPermute(
14628     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14629     SelectionDAG &DAG, const X86Subtarget &Subtarget) {
14630   int NumElts = VT.getVectorNumElements();
14631   int NumLanes = VT.getSizeInBits() / 128;
14632   int NumEltsPerLane = NumElts / NumLanes;
14633   bool CanUseSublanes = Subtarget.hasAVX2() && V2.isUndef();
14634 
14635   /// Attempts to find a sublane permute with the given size
14636   /// that gets all elements into their target lanes.
14637   ///
14638   /// If successful, fills CrossLaneMask and InLaneMask and returns true.
14639   /// If unsuccessful, returns false and may overwrite InLaneMask.
14640   auto getSublanePermute = [&](int NumSublanes) -> SDValue {
14641     int NumSublanesPerLane = NumSublanes / NumLanes;
14642     int NumEltsPerSublane = NumElts / NumSublanes;
14643 
14644     SmallVector<int, 16> CrossLaneMask;
14645     SmallVector<int, 16> InLaneMask(NumElts, SM_SentinelUndef);
14646     // CrossLaneMask but one entry == one sublane.
14647     SmallVector<int, 16> CrossLaneMaskLarge(NumSublanes, SM_SentinelUndef);
14648 
14649     for (int i = 0; i != NumElts; ++i) {
14650       int M = Mask[i];
14651       if (M < 0)
14652         continue;
14653 
14654       int SrcSublane = M / NumEltsPerSublane;
14655       int DstLane = i / NumEltsPerLane;
14656 
14657       // We only need to get the elements into the right lane, not sublane.
14658       // So search all sublanes that make up the destination lane.
14659       bool Found = false;
14660       int DstSubStart = DstLane * NumSublanesPerLane;
14661       int DstSubEnd = DstSubStart + NumSublanesPerLane;
14662       for (int DstSublane = DstSubStart; DstSublane < DstSubEnd; ++DstSublane) {
14663         if (!isUndefOrEqual(CrossLaneMaskLarge[DstSublane], SrcSublane))
14664           continue;
14665 
14666         Found = true;
14667         CrossLaneMaskLarge[DstSublane] = SrcSublane;
14668         int DstSublaneOffset = DstSublane * NumEltsPerSublane;
14669         InLaneMask[i] = DstSublaneOffset + M % NumEltsPerSublane;
14670         break;
14671       }
14672       if (!Found)
14673         return SDValue();
14674     }
14675 
14676     // Fill CrossLaneMask using CrossLaneMaskLarge.
14677     narrowShuffleMaskElts(NumEltsPerSublane, CrossLaneMaskLarge, CrossLaneMask);
14678 
14679     if (!CanUseSublanes) {
14680       // If we're only shuffling a single lowest lane and the rest are identity
14681       // then don't bother.
14682       // TODO - isShuffleMaskInputInPlace could be extended to something like
14683       // this.
14684       int NumIdentityLanes = 0;
14685       bool OnlyShuffleLowestLane = true;
14686       for (int i = 0; i != NumLanes; ++i) {
14687         int LaneOffset = i * NumEltsPerLane;
14688         if (isSequentialOrUndefInRange(InLaneMask, LaneOffset, NumEltsPerLane,
14689                                        i * NumEltsPerLane))
14690           NumIdentityLanes++;
14691         else if (CrossLaneMask[LaneOffset] != 0)
14692           OnlyShuffleLowestLane = false;
14693       }
14694       if (OnlyShuffleLowestLane && NumIdentityLanes == (NumLanes - 1))
14695         return SDValue();
14696     }
14697 
14698     // Avoid returning the same shuffle operation. For example,
14699     // t7: v16i16 = vector_shuffle<8,9,10,11,4,5,6,7,0,1,2,3,12,13,14,15> t5,
14700     //                             undef:v16i16
14701     if (CrossLaneMask == Mask || InLaneMask == Mask)
14702       return SDValue();
14703 
14704     SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask);
14705     return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT),
14706                                 InLaneMask);
14707   };
14708 
14709   // First attempt a solution with full lanes.
14710   if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes))
14711     return V;
14712 
14713   // The rest of the solutions use sublanes.
14714   if (!CanUseSublanes)
14715     return SDValue();
14716 
14717   // Then attempt a solution with 64-bit sublanes (vpermq).
14718   if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes * 2))
14719     return V;
14720 
14721   // If that doesn't work and we have fast variable cross-lane shuffle,
14722   // attempt 32-bit sublanes (vpermd).
14723   if (!Subtarget.hasFastVariableCrossLaneShuffle())
14724     return SDValue();
14725 
14726   return getSublanePermute(/*NumSublanes=*/NumLanes * 4);
14727 }
14728 
14729 /// Helper to get compute inlane shuffle mask for a complete shuffle mask.
14730 static void computeInLaneShuffleMask(const ArrayRef<int> &Mask, int LaneSize,
14731                                      SmallVector<int> &InLaneMask) {
14732   int Size = Mask.size();
14733   InLaneMask.assign(Mask.begin(), Mask.end());
14734   for (int i = 0; i < Size; ++i) {
14735     int &M = InLaneMask[i];
14736     if (M < 0)
14737       continue;
14738     if (((M % Size) / LaneSize) != (i / LaneSize))
14739       M = (M % LaneSize) + ((i / LaneSize) * LaneSize) + Size;
14740   }
14741 }
14742 
14743 /// Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one
14744 /// source with a lane permutation.
14745 ///
14746 /// This lowering strategy results in four instructions in the worst case for a
14747 /// single-input cross lane shuffle which is lower than any other fully general
14748 /// cross-lane shuffle strategy I'm aware of. Special cases for each particular
14749 /// shuffle pattern should be handled prior to trying this lowering.
14750 static SDValue lowerShuffleAsLanePermuteAndShuffle(
14751     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14752     SelectionDAG &DAG, const X86Subtarget &Subtarget) {
14753   // FIXME: This should probably be generalized for 512-bit vectors as well.
14754   assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
14755   int Size = Mask.size();
14756   int LaneSize = Size / 2;
14757 
14758   // Fold to SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
14759   // Only do this if the elements aren't all from the lower lane,
14760   // otherwise we're (probably) better off doing a split.
14761   if (VT == MVT::v4f64 &&
14762       !all_of(Mask, [LaneSize](int M) { return M < LaneSize; }))
14763     return lowerShuffleAsLanePermuteAndSHUFP(DL, VT, V1, V2, Mask, DAG);
14764 
14765   // If there are only inputs from one 128-bit lane, splitting will in fact be
14766   // less expensive. The flags track whether the given lane contains an element
14767   // that crosses to another lane.
14768   bool AllLanes;
14769   if (!Subtarget.hasAVX2()) {
14770     bool LaneCrossing[2] = {false, false};
14771     for (int i = 0; i < Size; ++i)
14772       if (Mask[i] >= 0 && ((Mask[i] % Size) / LaneSize) != (i / LaneSize))
14773         LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
14774     AllLanes = LaneCrossing[0] && LaneCrossing[1];
14775   } else {
14776     bool LaneUsed[2] = {false, false};
14777     for (int i = 0; i < Size; ++i)
14778       if (Mask[i] >= 0)
14779         LaneUsed[(Mask[i] % Size) / LaneSize] = true;
14780     AllLanes = LaneUsed[0] && LaneUsed[1];
14781   }
14782 
14783   // TODO - we could support shuffling V2 in the Flipped input.
14784   assert(V2.isUndef() &&
14785          "This last part of this routine only works on single input shuffles");
14786 
14787   SmallVector<int> InLaneMask;
14788   computeInLaneShuffleMask(Mask, Mask.size() / 2, InLaneMask);
14789 
14790   assert(!is128BitLaneCrossingShuffleMask(VT, InLaneMask) &&
14791          "In-lane shuffle mask expected");
14792 
14793   // If we're not using both lanes in each lane and the inlane mask is not
14794   // repeating, then we're better off splitting.
14795   if (!AllLanes && !is128BitLaneRepeatedShuffleMask(VT, InLaneMask))
14796     return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
14797                                 /*SimpleOnly*/ false);
14798 
14799   // Flip the lanes, and shuffle the results which should now be in-lane.
14800   MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;
14801   SDValue Flipped = DAG.getBitcast(PVT, V1);
14802   Flipped =
14803       DAG.getVectorShuffle(PVT, DL, Flipped, DAG.getUNDEF(PVT), {2, 3, 0, 1});
14804   Flipped = DAG.getBitcast(VT, Flipped);
14805   return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask);
14806 }
14807 
14808 /// Handle lowering 2-lane 128-bit shuffles.
14809 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
14810                                   SDValue V2, ArrayRef<int> Mask,
14811                                   const APInt &Zeroable,
14812                                   const X86Subtarget &Subtarget,
14813                                   SelectionDAG &DAG) {
14814   if (V2.isUndef()) {
14815     // Attempt to match VBROADCAST*128 subvector broadcast load.
14816     bool SplatLo = isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1);
14817     bool SplatHi = isShuffleEquivalent(Mask, {2, 3, 2, 3}, V1);
14818     if ((SplatLo || SplatHi) && !Subtarget.hasAVX512() && V1.hasOneUse() &&
14819         X86::mayFoldLoad(peekThroughOneUseBitcasts(V1), Subtarget)) {
14820       MVT MemVT = VT.getHalfNumVectorElementsVT();
14821       unsigned Ofs = SplatLo ? 0 : MemVT.getStoreSize();
14822       auto *Ld = cast<LoadSDNode>(peekThroughOneUseBitcasts(V1));
14823       if (SDValue BcstLd = getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, DL,
14824                                              VT, MemVT, Ld, Ofs, DAG))
14825         return BcstLd;
14826     }
14827 
14828     // With AVX2, use VPERMQ/VPERMPD for unary shuffles to allow memory folding.
14829     if (Subtarget.hasAVX2())
14830       return SDValue();
14831   }
14832 
14833   bool V2IsZero = !V2.isUndef() && ISD::isBuildVectorAllZeros(V2.getNode());
14834 
14835   SmallVector<int, 4> WidenedMask;
14836   if (!canWidenShuffleElements(Mask, Zeroable, V2IsZero, WidenedMask))
14837     return SDValue();
14838 
14839   bool IsLowZero = (Zeroable & 0x3) == 0x3;
14840   bool IsHighZero = (Zeroable & 0xc) == 0xc;
14841 
14842   // Try to use an insert into a zero vector.
14843   if (WidenedMask[0] == 0 && IsHighZero) {
14844     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
14845     SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
14846                               DAG.getIntPtrConstant(0, DL));
14847     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
14848                        getZeroVector(VT, Subtarget, DAG, DL), LoV,
14849                        DAG.getIntPtrConstant(0, DL));
14850   }
14851 
14852   // TODO: If minimizing size and one of the inputs is a zero vector and the
14853   // the zero vector has only one use, we could use a VPERM2X128 to save the
14854   // instruction bytes needed to explicitly generate the zero vector.
14855 
14856   // Blends are faster and handle all the non-lane-crossing cases.
14857   if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
14858                                           Subtarget, DAG))
14859     return Blend;
14860 
14861   // If either input operand is a zero vector, use VPERM2X128 because its mask
14862   // allows us to replace the zero input with an implicit zero.
14863   if (!IsLowZero && !IsHighZero) {
14864     // Check for patterns which can be matched with a single insert of a 128-bit
14865     // subvector.
14866     bool OnlyUsesV1 = isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1, V2);
14867     if (OnlyUsesV1 || isShuffleEquivalent(Mask, {0, 1, 4, 5}, V1, V2)) {
14868 
14869       // With AVX1, use vperm2f128 (below) to allow load folding. Otherwise,
14870       // this will likely become vinsertf128 which can't fold a 256-bit memop.
14871       if (!isa<LoadSDNode>(peekThroughBitcasts(V1))) {
14872         MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
14873         SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
14874                                      OnlyUsesV1 ? V1 : V2,
14875                                      DAG.getIntPtrConstant(0, DL));
14876         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
14877                            DAG.getIntPtrConstant(2, DL));
14878       }
14879     }
14880 
14881     // Try to use SHUF128 if possible.
14882     if (Subtarget.hasVLX()) {
14883       if (WidenedMask[0] < 2 && WidenedMask[1] >= 2) {
14884         unsigned PermMask = ((WidenedMask[0] % 2) << 0) |
14885                             ((WidenedMask[1] % 2) << 1);
14886         return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
14887                            DAG.getTargetConstant(PermMask, DL, MVT::i8));
14888       }
14889     }
14890   }
14891 
14892   // Otherwise form a 128-bit permutation. After accounting for undefs,
14893   // convert the 64-bit shuffle mask selection values into 128-bit
14894   // selection bits by dividing the indexes by 2 and shifting into positions
14895   // defined by a vperm2*128 instruction's immediate control byte.
14896 
14897   // The immediate permute control byte looks like this:
14898   //    [1:0] - select 128 bits from sources for low half of destination
14899   //    [2]   - ignore
14900   //    [3]   - zero low half of destination
14901   //    [5:4] - select 128 bits from sources for high half of destination
14902   //    [6]   - ignore
14903   //    [7]   - zero high half of destination
14904 
14905   assert((WidenedMask[0] >= 0 || IsLowZero) &&
14906          (WidenedMask[1] >= 0 || IsHighZero) && "Undef half?");
14907 
14908   unsigned PermMask = 0;
14909   PermMask |= IsLowZero  ? 0x08 : (WidenedMask[0] << 0);
14910   PermMask |= IsHighZero ? 0x80 : (WidenedMask[1] << 4);
14911 
14912   // Check the immediate mask and replace unused sources with undef.
14913   if ((PermMask & 0x0a) != 0x00 && (PermMask & 0xa0) != 0x00)
14914     V1 = DAG.getUNDEF(VT);
14915   if ((PermMask & 0x0a) != 0x02 && (PermMask & 0xa0) != 0x20)
14916     V2 = DAG.getUNDEF(VT);
14917 
14918   return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
14919                      DAG.getTargetConstant(PermMask, DL, MVT::i8));
14920 }
14921 
14922 /// Lower a vector shuffle by first fixing the 128-bit lanes and then
14923 /// shuffling each lane.
14924 ///
14925 /// This attempts to create a repeated lane shuffle where each lane uses one
14926 /// or two of the lanes of the inputs. The lanes of the input vectors are
14927 /// shuffled in one or two independent shuffles to get the lanes into the
14928 /// position needed by the final shuffle.
14929 static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
14930     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14931     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
14932   assert(!V2.isUndef() && "This is only useful with multiple inputs.");
14933 
14934   if (is128BitLaneRepeatedShuffleMask(VT, Mask))
14935     return SDValue();
14936 
14937   int NumElts = Mask.size();
14938   int NumLanes = VT.getSizeInBits() / 128;
14939   int NumLaneElts = 128 / VT.getScalarSizeInBits();
14940   SmallVector<int, 16> RepeatMask(NumLaneElts, -1);
14941   SmallVector<std::array<int, 2>, 2> LaneSrcs(NumLanes, {{-1, -1}});
14942 
14943   // First pass will try to fill in the RepeatMask from lanes that need two
14944   // sources.
14945   for (int Lane = 0; Lane != NumLanes; ++Lane) {
14946     int Srcs[2] = {-1, -1};
14947     SmallVector<int, 16> InLaneMask(NumLaneElts, -1);
14948     for (int i = 0; i != NumLaneElts; ++i) {
14949       int M = Mask[(Lane * NumLaneElts) + i];
14950       if (M < 0)
14951         continue;
14952       // Determine which of the possible input lanes (NumLanes from each source)
14953       // this element comes from. Assign that as one of the sources for this
14954       // lane. We can assign up to 2 sources for this lane. If we run out
14955       // sources we can't do anything.
14956       int LaneSrc = M / NumLaneElts;
14957       int Src;
14958       if (Srcs[0] < 0 || Srcs[0] == LaneSrc)
14959         Src = 0;
14960       else if (Srcs[1] < 0 || Srcs[1] == LaneSrc)
14961         Src = 1;
14962       else
14963         return SDValue();
14964 
14965       Srcs[Src] = LaneSrc;
14966       InLaneMask[i] = (M % NumLaneElts) + Src * NumElts;
14967     }
14968 
14969     // If this lane has two sources, see if it fits with the repeat mask so far.
14970     if (Srcs[1] < 0)
14971       continue;
14972 
14973     LaneSrcs[Lane][0] = Srcs[0];
14974     LaneSrcs[Lane][1] = Srcs[1];
14975 
14976     auto MatchMasks = [](ArrayRef<int> M1, ArrayRef<int> M2) {
14977       assert(M1.size() == M2.size() && "Unexpected mask size");
14978       for (int i = 0, e = M1.size(); i != e; ++i)
14979         if (M1[i] >= 0 && M2[i] >= 0 && M1[i] != M2[i])
14980           return false;
14981       return true;
14982     };
14983 
14984     auto MergeMasks = [](ArrayRef<int> Mask, MutableArrayRef<int> MergedMask) {
14985       assert(Mask.size() == MergedMask.size() && "Unexpected mask size");
14986       for (int i = 0, e = MergedMask.size(); i != e; ++i) {
14987         int M = Mask[i];
14988         if (M < 0)
14989           continue;
14990         assert((MergedMask[i] < 0 || MergedMask[i] == M) &&
14991                "Unexpected mask element");
14992         MergedMask[i] = M;
14993       }
14994     };
14995 
14996     if (MatchMasks(InLaneMask, RepeatMask)) {
14997       // Merge this lane mask into the final repeat mask.
14998       MergeMasks(InLaneMask, RepeatMask);
14999       continue;
15000     }
15001 
15002     // Didn't find a match. Swap the operands and try again.
15003     std::swap(LaneSrcs[Lane][0], LaneSrcs[Lane][1]);
15004     ShuffleVectorSDNode::commuteMask(InLaneMask);
15005 
15006     if (MatchMasks(InLaneMask, RepeatMask)) {
15007       // Merge this lane mask into the final repeat mask.
15008       MergeMasks(InLaneMask, RepeatMask);
15009       continue;
15010     }
15011 
15012     // Couldn't find a match with the operands in either order.
15013     return SDValue();
15014   }
15015 
15016   // Now handle any lanes with only one source.
15017   for (int Lane = 0; Lane != NumLanes; ++Lane) {
15018     // If this lane has already been processed, skip it.
15019     if (LaneSrcs[Lane][0] >= 0)
15020       continue;
15021 
15022     for (int i = 0; i != NumLaneElts; ++i) {
15023       int M = Mask[(Lane * NumLaneElts) + i];
15024       if (M < 0)
15025         continue;
15026 
15027       // If RepeatMask isn't defined yet we can define it ourself.
15028       if (RepeatMask[i] < 0)
15029         RepeatMask[i] = M % NumLaneElts;
15030 
15031       if (RepeatMask[i] < NumElts) {
15032         if (RepeatMask[i] != M % NumLaneElts)
15033           return SDValue();
15034         LaneSrcs[Lane][0] = M / NumLaneElts;
15035       } else {
15036         if (RepeatMask[i] != ((M % NumLaneElts) + NumElts))
15037           return SDValue();
15038         LaneSrcs[Lane][1] = M / NumLaneElts;
15039       }
15040     }
15041 
15042     if (LaneSrcs[Lane][0] < 0 && LaneSrcs[Lane][1] < 0)
15043       return SDValue();
15044   }
15045 
15046   SmallVector<int, 16> NewMask(NumElts, -1);
15047   for (int Lane = 0; Lane != NumLanes; ++Lane) {
15048     int Src = LaneSrcs[Lane][0];
15049     for (int i = 0; i != NumLaneElts; ++i) {
15050       int M = -1;
15051       if (Src >= 0)
15052         M = Src * NumLaneElts + i;
15053       NewMask[Lane * NumLaneElts + i] = M;
15054     }
15055   }
15056   SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15057   // Ensure we didn't get back the shuffle we started with.
15058   // FIXME: This is a hack to make up for some splat handling code in
15059   // getVectorShuffle.
15060   if (isa<ShuffleVectorSDNode>(NewV1) &&
15061       cast<ShuffleVectorSDNode>(NewV1)->getMask() == Mask)
15062     return SDValue();
15063 
15064   for (int Lane = 0; Lane != NumLanes; ++Lane) {
15065     int Src = LaneSrcs[Lane][1];
15066     for (int i = 0; i != NumLaneElts; ++i) {
15067       int M = -1;
15068       if (Src >= 0)
15069         M = Src * NumLaneElts + i;
15070       NewMask[Lane * NumLaneElts + i] = M;
15071     }
15072   }
15073   SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15074   // Ensure we didn't get back the shuffle we started with.
15075   // FIXME: This is a hack to make up for some splat handling code in
15076   // getVectorShuffle.
15077   if (isa<ShuffleVectorSDNode>(NewV2) &&
15078       cast<ShuffleVectorSDNode>(NewV2)->getMask() == Mask)
15079     return SDValue();
15080 
15081   for (int i = 0; i != NumElts; ++i) {
15082     if (Mask[i] < 0) {
15083       NewMask[i] = -1;
15084       continue;
15085     }
15086     NewMask[i] = RepeatMask[i % NumLaneElts];
15087     if (NewMask[i] < 0)
15088       continue;
15089 
15090     NewMask[i] += (i / NumLaneElts) * NumLaneElts;
15091   }
15092   return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask);
15093 }
15094 
15095 /// If the input shuffle mask results in a vector that is undefined in all upper
15096 /// or lower half elements and that mask accesses only 2 halves of the
15097 /// shuffle's operands, return true. A mask of half the width with mask indexes
15098 /// adjusted to access the extracted halves of the original shuffle operands is
15099 /// returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or
15100 /// lower half of each input operand is accessed.
15101 static bool
15102 getHalfShuffleMask(ArrayRef<int> Mask, MutableArrayRef<int> HalfMask,
15103                    int &HalfIdx1, int &HalfIdx2) {
15104   assert((Mask.size() == HalfMask.size() * 2) &&
15105          "Expected input mask to be twice as long as output");
15106 
15107   // Exactly one half of the result must be undef to allow narrowing.
15108   bool UndefLower = isUndefLowerHalf(Mask);
15109   bool UndefUpper = isUndefUpperHalf(Mask);
15110   if (UndefLower == UndefUpper)
15111     return false;
15112 
15113   unsigned HalfNumElts = HalfMask.size();
15114   unsigned MaskIndexOffset = UndefLower ? HalfNumElts : 0;
15115   HalfIdx1 = -1;
15116   HalfIdx2 = -1;
15117   for (unsigned i = 0; i != HalfNumElts; ++i) {
15118     int M = Mask[i + MaskIndexOffset];
15119     if (M < 0) {
15120       HalfMask[i] = M;
15121       continue;
15122     }
15123 
15124     // Determine which of the 4 half vectors this element is from.
15125     // i.e. 0 = Lower V1, 1 = Upper V1, 2 = Lower V2, 3 = Upper V2.
15126     int HalfIdx = M / HalfNumElts;
15127 
15128     // Determine the element index into its half vector source.
15129     int HalfElt = M % HalfNumElts;
15130 
15131     // We can shuffle with up to 2 half vectors, set the new 'half'
15132     // shuffle mask accordingly.
15133     if (HalfIdx1 < 0 || HalfIdx1 == HalfIdx) {
15134       HalfMask[i] = HalfElt;
15135       HalfIdx1 = HalfIdx;
15136       continue;
15137     }
15138     if (HalfIdx2 < 0 || HalfIdx2 == HalfIdx) {
15139       HalfMask[i] = HalfElt + HalfNumElts;
15140       HalfIdx2 = HalfIdx;
15141       continue;
15142     }
15143 
15144     // Too many half vectors referenced.
15145     return false;
15146   }
15147 
15148   return true;
15149 }
15150 
15151 /// Given the output values from getHalfShuffleMask(), create a half width
15152 /// shuffle of extracted vectors followed by an insert back to full width.
15153 static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15154                                      ArrayRef<int> HalfMask, int HalfIdx1,
15155                                      int HalfIdx2, bool UndefLower,
15156                                      SelectionDAG &DAG, bool UseConcat = false) {
15157   assert(V1.getValueType() == V2.getValueType() && "Different sized vectors?");
15158   assert(V1.getValueType().isSimple() && "Expecting only simple types");
15159 
15160   MVT VT = V1.getSimpleValueType();
15161   MVT HalfVT = VT.getHalfNumVectorElementsVT();
15162   unsigned HalfNumElts = HalfVT.getVectorNumElements();
15163 
15164   auto getHalfVector = [&](int HalfIdx) {
15165     if (HalfIdx < 0)
15166       return DAG.getUNDEF(HalfVT);
15167     SDValue V = (HalfIdx < 2 ? V1 : V2);
15168     HalfIdx = (HalfIdx % 2) * HalfNumElts;
15169     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V,
15170                        DAG.getIntPtrConstant(HalfIdx, DL));
15171   };
15172 
15173   // ins undef, (shuf (ext V1, HalfIdx1), (ext V2, HalfIdx2), HalfMask), Offset
15174   SDValue Half1 = getHalfVector(HalfIdx1);
15175   SDValue Half2 = getHalfVector(HalfIdx2);
15176   SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15177   if (UseConcat) {
15178     SDValue Op0 = V;
15179     SDValue Op1 = DAG.getUNDEF(HalfVT);
15180     if (UndefLower)
15181       std::swap(Op0, Op1);
15182     return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1);
15183   }
15184 
15185   unsigned Offset = UndefLower ? HalfNumElts : 0;
15186   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V,
15187                      DAG.getIntPtrConstant(Offset, DL));
15188 }
15189 
15190 /// Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
15191 /// This allows for fast cases such as subvector extraction/insertion
15192 /// or shuffling smaller vector types which can lower more efficiently.
15193 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15194                                          SDValue V2, ArrayRef<int> Mask,
15195                                          const X86Subtarget &Subtarget,
15196                                          SelectionDAG &DAG) {
15197   assert((VT.is256BitVector() || VT.is512BitVector()) &&
15198          "Expected 256-bit or 512-bit vector");
15199 
15200   bool UndefLower = isUndefLowerHalf(Mask);
15201   if (!UndefLower && !isUndefUpperHalf(Mask))
15202     return SDValue();
15203 
15204   assert((!UndefLower || !isUndefUpperHalf(Mask)) &&
15205          "Completely undef shuffle mask should have been simplified already");
15206 
15207   // Upper half is undef and lower half is whole upper subvector.
15208   // e.g. vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
15209   MVT HalfVT = VT.getHalfNumVectorElementsVT();
15210   unsigned HalfNumElts = HalfVT.getVectorNumElements();
15211   if (!UndefLower &&
15212       isSequentialOrUndefInRange(Mask, 0, HalfNumElts, HalfNumElts)) {
15213     SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15214                              DAG.getIntPtrConstant(HalfNumElts, DL));
15215     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15216                        DAG.getIntPtrConstant(0, DL));
15217   }
15218 
15219   // Lower half is undef and upper half is whole lower subvector.
15220   // e.g. vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
15221   if (UndefLower &&
15222       isSequentialOrUndefInRange(Mask, HalfNumElts, HalfNumElts, 0)) {
15223     SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15224                              DAG.getIntPtrConstant(0, DL));
15225     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15226                        DAG.getIntPtrConstant(HalfNumElts, DL));
15227   }
15228 
15229   int HalfIdx1, HalfIdx2;
15230   SmallVector<int, 8> HalfMask(HalfNumElts);
15231   if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2))
15232     return SDValue();
15233 
15234   assert(HalfMask.size() == HalfNumElts && "Unexpected shuffle mask length");
15235 
15236   // Only shuffle the halves of the inputs when useful.
15237   unsigned NumLowerHalves =
15238       (HalfIdx1 == 0 || HalfIdx1 == 2) + (HalfIdx2 == 0 || HalfIdx2 == 2);
15239   unsigned NumUpperHalves =
15240       (HalfIdx1 == 1 || HalfIdx1 == 3) + (HalfIdx2 == 1 || HalfIdx2 == 3);
15241   assert(NumLowerHalves + NumUpperHalves <= 2 && "Only 1 or 2 halves allowed");
15242 
15243   // Determine the larger pattern of undef/halves, then decide if it's worth
15244   // splitting the shuffle based on subtarget capabilities and types.
15245   unsigned EltWidth = VT.getVectorElementType().getSizeInBits();
15246   if (!UndefLower) {
15247     // XXXXuuuu: no insert is needed.
15248     // Always extract lowers when setting lower - these are all free subreg ops.
15249     if (NumUpperHalves == 0)
15250       return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15251                                    UndefLower, DAG);
15252 
15253     if (NumUpperHalves == 1) {
15254       // AVX2 has efficient 32/64-bit element cross-lane shuffles.
15255       if (Subtarget.hasAVX2()) {
15256         // extract128 + vunpckhps/vshufps, is better than vblend + vpermps.
15257         if (EltWidth == 32 && NumLowerHalves && HalfVT.is128BitVector() &&
15258             !is128BitUnpackShuffleMask(HalfMask, DAG) &&
15259             (!isSingleSHUFPSMask(HalfMask) ||
15260              Subtarget.hasFastVariableCrossLaneShuffle()))
15261           return SDValue();
15262         // If this is a unary shuffle (assume that the 2nd operand is
15263         // canonicalized to undef), then we can use vpermpd. Otherwise, we
15264         // are better off extracting the upper half of 1 operand and using a
15265         // narrow shuffle.
15266         if (EltWidth == 64 && V2.isUndef())
15267           return SDValue();
15268       }
15269       // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
15270       if (Subtarget.hasAVX512() && VT.is512BitVector())
15271         return SDValue();
15272       // Extract + narrow shuffle is better than the wide alternative.
15273       return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15274                                    UndefLower, DAG);
15275     }
15276 
15277     // Don't extract both uppers, instead shuffle and then extract.
15278     assert(NumUpperHalves == 2 && "Half vector count went wrong");
15279     return SDValue();
15280   }
15281 
15282   // UndefLower - uuuuXXXX: an insert to high half is required if we split this.
15283   if (NumUpperHalves == 0) {
15284     // AVX2 has efficient 64-bit element cross-lane shuffles.
15285     // TODO: Refine to account for unary shuffle, splat, and other masks?
15286     if (Subtarget.hasAVX2() && EltWidth == 64)
15287       return SDValue();
15288     // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
15289     if (Subtarget.hasAVX512() && VT.is512BitVector())
15290       return SDValue();
15291     // Narrow shuffle + insert is better than the wide alternative.
15292     return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15293                                  UndefLower, DAG);
15294   }
15295 
15296   // NumUpperHalves != 0: don't bother with extract, shuffle, and then insert.
15297   return SDValue();
15298 }
15299 
15300 /// Handle case where shuffle sources are coming from the same 128-bit lane and
15301 /// every lane can be represented as the same repeating mask - allowing us to
15302 /// shuffle the sources with the repeating shuffle and then permute the result
15303 /// to the destination lanes.
15304 static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
15305     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15306     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
15307   int NumElts = VT.getVectorNumElements();
15308   int NumLanes = VT.getSizeInBits() / 128;
15309   int NumLaneElts = NumElts / NumLanes;
15310 
15311   // On AVX2 we may be able to just shuffle the lowest elements and then
15312   // broadcast the result.
15313   if (Subtarget.hasAVX2()) {
15314     for (unsigned BroadcastSize : {16, 32, 64}) {
15315       if (BroadcastSize <= VT.getScalarSizeInBits())
15316         continue;
15317       int NumBroadcastElts = BroadcastSize / VT.getScalarSizeInBits();
15318 
15319       // Attempt to match a repeating pattern every NumBroadcastElts,
15320       // accounting for UNDEFs but only references the lowest 128-bit
15321       // lane of the inputs.
15322       auto FindRepeatingBroadcastMask = [&](SmallVectorImpl<int> &RepeatMask) {
15323         for (int i = 0; i != NumElts; i += NumBroadcastElts)
15324           for (int j = 0; j != NumBroadcastElts; ++j) {
15325             int M = Mask[i + j];
15326             if (M < 0)
15327               continue;
15328             int &R = RepeatMask[j];
15329             if (0 != ((M % NumElts) / NumLaneElts))
15330               return false;
15331             if (0 <= R && R != M)
15332               return false;
15333             R = M;
15334           }
15335         return true;
15336       };
15337 
15338       SmallVector<int, 8> RepeatMask((unsigned)NumElts, -1);
15339       if (!FindRepeatingBroadcastMask(RepeatMask))
15340         continue;
15341 
15342       // Shuffle the (lowest) repeated elements in place for broadcast.
15343       SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
15344 
15345       // Shuffle the actual broadcast.
15346       SmallVector<int, 8> BroadcastMask((unsigned)NumElts, -1);
15347       for (int i = 0; i != NumElts; i += NumBroadcastElts)
15348         for (int j = 0; j != NumBroadcastElts; ++j)
15349           BroadcastMask[i + j] = j;
15350 
15351       // Avoid returning the same shuffle operation. For example,
15352       // v8i32 = vector_shuffle<0,1,0,1,0,1,0,1> t5, undef:v8i32
15353       if (BroadcastMask == Mask)
15354         return SDValue();
15355 
15356       return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT),
15357                                   BroadcastMask);
15358     }
15359   }
15360 
15361   // Bail if the shuffle mask doesn't cross 128-bit lanes.
15362   if (!is128BitLaneCrossingShuffleMask(VT, Mask))
15363     return SDValue();
15364 
15365   // Bail if we already have a repeated lane shuffle mask.
15366   if (is128BitLaneRepeatedShuffleMask(VT, Mask))
15367     return SDValue();
15368 
15369   // Helper to look for repeated mask in each split sublane, and that those
15370   // sublanes can then be permuted into place.
15371   auto ShuffleSubLanes = [&](int SubLaneScale) {
15372     int NumSubLanes = NumLanes * SubLaneScale;
15373     int NumSubLaneElts = NumLaneElts / SubLaneScale;
15374 
15375     // Check that all the sources are coming from the same lane and see if we
15376     // can form a repeating shuffle mask (local to each sub-lane). At the same
15377     // time, determine the source sub-lane for each destination sub-lane.
15378     int TopSrcSubLane = -1;
15379     SmallVector<int, 8> Dst2SrcSubLanes((unsigned)NumSubLanes, -1);
15380     SmallVector<SmallVector<int, 8>> RepeatedSubLaneMasks(
15381         SubLaneScale,
15382         SmallVector<int, 8>((unsigned)NumSubLaneElts, SM_SentinelUndef));
15383 
15384     for (int DstSubLane = 0; DstSubLane != NumSubLanes; ++DstSubLane) {
15385       // Extract the sub-lane mask, check that it all comes from the same lane
15386       // and normalize the mask entries to come from the first lane.
15387       int SrcLane = -1;
15388       SmallVector<int, 8> SubLaneMask((unsigned)NumSubLaneElts, -1);
15389       for (int Elt = 0; Elt != NumSubLaneElts; ++Elt) {
15390         int M = Mask[(DstSubLane * NumSubLaneElts) + Elt];
15391         if (M < 0)
15392           continue;
15393         int Lane = (M % NumElts) / NumLaneElts;
15394         if ((0 <= SrcLane) && (SrcLane != Lane))
15395           return SDValue();
15396         SrcLane = Lane;
15397         int LocalM = (M % NumLaneElts) + (M < NumElts ? 0 : NumElts);
15398         SubLaneMask[Elt] = LocalM;
15399       }
15400 
15401       // Whole sub-lane is UNDEF.
15402       if (SrcLane < 0)
15403         continue;
15404 
15405       // Attempt to match against the candidate repeated sub-lane masks.
15406       for (int SubLane = 0; SubLane != SubLaneScale; ++SubLane) {
15407         auto MatchMasks = [NumSubLaneElts](ArrayRef<int> M1, ArrayRef<int> M2) {
15408           for (int i = 0; i != NumSubLaneElts; ++i) {
15409             if (M1[i] < 0 || M2[i] < 0)
15410               continue;
15411             if (M1[i] != M2[i])
15412               return false;
15413           }
15414           return true;
15415         };
15416 
15417         auto &RepeatedSubLaneMask = RepeatedSubLaneMasks[SubLane];
15418         if (!MatchMasks(SubLaneMask, RepeatedSubLaneMask))
15419           continue;
15420 
15421         // Merge the sub-lane mask into the matching repeated sub-lane mask.
15422         for (int i = 0; i != NumSubLaneElts; ++i) {
15423           int M = SubLaneMask[i];
15424           if (M < 0)
15425             continue;
15426           assert((RepeatedSubLaneMask[i] < 0 || RepeatedSubLaneMask[i] == M) &&
15427                  "Unexpected mask element");
15428           RepeatedSubLaneMask[i] = M;
15429         }
15430 
15431         // Track the top most source sub-lane - by setting the remaining to
15432         // UNDEF we can greatly simplify shuffle matching.
15433         int SrcSubLane = (SrcLane * SubLaneScale) + SubLane;
15434         TopSrcSubLane = std::max(TopSrcSubLane, SrcSubLane);
15435         Dst2SrcSubLanes[DstSubLane] = SrcSubLane;
15436         break;
15437       }
15438 
15439       // Bail if we failed to find a matching repeated sub-lane mask.
15440       if (Dst2SrcSubLanes[DstSubLane] < 0)
15441         return SDValue();
15442     }
15443     assert(0 <= TopSrcSubLane && TopSrcSubLane < NumSubLanes &&
15444            "Unexpected source lane");
15445 
15446     // Create a repeating shuffle mask for the entire vector.
15447     SmallVector<int, 8> RepeatedMask((unsigned)NumElts, -1);
15448     for (int SubLane = 0; SubLane <= TopSrcSubLane; ++SubLane) {
15449       int Lane = SubLane / SubLaneScale;
15450       auto &RepeatedSubLaneMask = RepeatedSubLaneMasks[SubLane % SubLaneScale];
15451       for (int Elt = 0; Elt != NumSubLaneElts; ++Elt) {
15452         int M = RepeatedSubLaneMask[Elt];
15453         if (M < 0)
15454           continue;
15455         int Idx = (SubLane * NumSubLaneElts) + Elt;
15456         RepeatedMask[Idx] = M + (Lane * NumLaneElts);
15457       }
15458     }
15459 
15460     // Shuffle each source sub-lane to its destination.
15461     SmallVector<int, 8> SubLaneMask((unsigned)NumElts, -1);
15462     for (int i = 0; i != NumElts; i += NumSubLaneElts) {
15463       int SrcSubLane = Dst2SrcSubLanes[i / NumSubLaneElts];
15464       if (SrcSubLane < 0)
15465         continue;
15466       for (int j = 0; j != NumSubLaneElts; ++j)
15467         SubLaneMask[i + j] = j + (SrcSubLane * NumSubLaneElts);
15468     }
15469 
15470     // Avoid returning the same shuffle operation.
15471     // v8i32 = vector_shuffle<0,1,4,5,2,3,6,7> t5, undef:v8i32
15472     if (RepeatedMask == Mask || SubLaneMask == Mask)
15473       return SDValue();
15474 
15475     SDValue RepeatedShuffle =
15476         DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
15477 
15478     return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT),
15479                                 SubLaneMask);
15480   };
15481 
15482   // On AVX2 targets we can permute 256-bit vectors as 64-bit sub-lanes
15483   // (with PERMQ/PERMPD). On AVX2/AVX512BW targets, permuting 32-bit sub-lanes,
15484   // even with a variable shuffle, can be worth it for v32i8/v64i8 vectors.
15485   // Otherwise we can only permute whole 128-bit lanes.
15486   int MinSubLaneScale = 1, MaxSubLaneScale = 1;
15487   if (Subtarget.hasAVX2() && VT.is256BitVector()) {
15488     bool OnlyLowestElts = isUndefOrInRange(Mask, 0, NumLaneElts);
15489     MinSubLaneScale = 2;
15490     MaxSubLaneScale =
15491         (!OnlyLowestElts && V2.isUndef() && VT == MVT::v32i8) ? 4 : 2;
15492   }
15493   if (Subtarget.hasBWI() && VT == MVT::v64i8)
15494     MinSubLaneScale = MaxSubLaneScale = 4;
15495 
15496   for (int Scale = MinSubLaneScale; Scale <= MaxSubLaneScale; Scale *= 2)
15497     if (SDValue Shuffle = ShuffleSubLanes(Scale))
15498       return Shuffle;
15499 
15500   return SDValue();
15501 }
15502 
15503 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
15504                                    bool &ForceV1Zero, bool &ForceV2Zero,
15505                                    unsigned &ShuffleImm, ArrayRef<int> Mask,
15506                                    const APInt &Zeroable) {
15507   int NumElts = VT.getVectorNumElements();
15508   assert(VT.getScalarSizeInBits() == 64 &&
15509          (NumElts == 2 || NumElts == 4 || NumElts == 8) &&
15510          "Unexpected data type for VSHUFPD");
15511   assert(isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) &&
15512          "Illegal shuffle mask");
15513 
15514   bool ZeroLane[2] = { true, true };
15515   for (int i = 0; i < NumElts; ++i)
15516     ZeroLane[i & 1] &= Zeroable[i];
15517 
15518   // Mask for V8F64: 0/1,  8/9,  2/3,  10/11, 4/5, ..
15519   // Mask for V4F64; 0/1,  4/5,  2/3,  6/7..
15520   ShuffleImm = 0;
15521   bool ShufpdMask = true;
15522   bool CommutableMask = true;
15523   for (int i = 0; i < NumElts; ++i) {
15524     if (Mask[i] == SM_SentinelUndef || ZeroLane[i & 1])
15525       continue;
15526     if (Mask[i] < 0)
15527       return false;
15528     int Val = (i & 6) + NumElts * (i & 1);
15529     int CommutVal = (i & 0xe) + NumElts * ((i & 1) ^ 1);
15530     if (Mask[i] < Val || Mask[i] > Val + 1)
15531       ShufpdMask = false;
15532     if (Mask[i] < CommutVal || Mask[i] > CommutVal + 1)
15533       CommutableMask = false;
15534     ShuffleImm |= (Mask[i] % 2) << i;
15535   }
15536 
15537   if (!ShufpdMask && !CommutableMask)
15538     return false;
15539 
15540   if (!ShufpdMask && CommutableMask)
15541     std::swap(V1, V2);
15542 
15543   ForceV1Zero = ZeroLane[0];
15544   ForceV2Zero = ZeroLane[1];
15545   return true;
15546 }
15547 
15548 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
15549                                       SDValue V2, ArrayRef<int> Mask,
15550                                       const APInt &Zeroable,
15551                                       const X86Subtarget &Subtarget,
15552                                       SelectionDAG &DAG) {
15553   assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) &&
15554          "Unexpected data type for VSHUFPD");
15555 
15556   unsigned Immediate = 0;
15557   bool ForceV1Zero = false, ForceV2Zero = false;
15558   if (!matchShuffleWithSHUFPD(VT, V1, V2, ForceV1Zero, ForceV2Zero, Immediate,
15559                               Mask, Zeroable))
15560     return SDValue();
15561 
15562   // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
15563   if (ForceV1Zero)
15564     V1 = getZeroVector(VT, Subtarget, DAG, DL);
15565   if (ForceV2Zero)
15566     V2 = getZeroVector(VT, Subtarget, DAG, DL);
15567 
15568   return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
15569                      DAG.getTargetConstant(Immediate, DL, MVT::i8));
15570 }
15571 
15572 // Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
15573 // by zeroable elements in the remaining 24 elements. Turn this into two
15574 // vmovqb instructions shuffled together.
15575 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
15576                                              SDValue V1, SDValue V2,
15577                                              ArrayRef<int> Mask,
15578                                              const APInt &Zeroable,
15579                                              SelectionDAG &DAG) {
15580   assert(VT == MVT::v32i8 && "Unexpected type!");
15581 
15582   // The first 8 indices should be every 8th element.
15583   if (!isSequentialOrUndefInRange(Mask, 0, 8, 0, 8))
15584     return SDValue();
15585 
15586   // Remaining elements need to be zeroable.
15587   if (Zeroable.countl_one() < (Mask.size() - 8))
15588     return SDValue();
15589 
15590   V1 = DAG.getBitcast(MVT::v4i64, V1);
15591   V2 = DAG.getBitcast(MVT::v4i64, V2);
15592 
15593   V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1);
15594   V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2);
15595 
15596   // The VTRUNCs will put 0s in the upper 12 bytes. Use them to put zeroes in
15597   // the upper bits of the result using an unpckldq.
15598   SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
15599                                         { 0, 1, 2, 3, 16, 17, 18, 19,
15600                                           4, 5, 6, 7, 20, 21, 22, 23 });
15601   // Insert the unpckldq into a zero vector to widen to v32i8.
15602   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,
15603                      DAG.getConstant(0, DL, MVT::v32i8), Unpack,
15604                      DAG.getIntPtrConstant(0, DL));
15605 }
15606 
15607 // a = shuffle v1, v2, mask1    ; interleaving lower lanes of v1 and v2
15608 // b = shuffle v1, v2, mask2    ; interleaving higher lanes of v1 and v2
15609 //     =>
15610 // ul = unpckl v1, v2
15611 // uh = unpckh v1, v2
15612 // a = vperm ul, uh
15613 // b = vperm ul, uh
15614 //
15615 // Pattern-match interleave(256b v1, 256b v2) -> 512b v3 and lower it into unpck
15616 // and permute. We cannot directly match v3 because it is split into two
15617 // 256-bit vectors in earlier isel stages. Therefore, this function matches a
15618 // pair of 256-bit shuffles and makes sure the masks are consecutive.
15619 //
15620 // Once unpck and permute nodes are created, the permute corresponding to this
15621 // shuffle is returned, while the other permute replaces the other half of the
15622 // shuffle in the selection dag.
15623 static SDValue lowerShufflePairAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
15624                                                  SDValue V1, SDValue V2,
15625                                                  ArrayRef<int> Mask,
15626                                                  SelectionDAG &DAG) {
15627   if (VT != MVT::v8f32 && VT != MVT::v8i32 && VT != MVT::v16i16 &&
15628       VT != MVT::v32i8)
15629     return SDValue();
15630   // <B0, B1, B0+1, B1+1, ..., >
15631   auto IsInterleavingPattern = [&](ArrayRef<int> Mask, unsigned Begin0,
15632                                    unsigned Begin1) {
15633     size_t Size = Mask.size();
15634     assert(Size % 2 == 0 && "Expected even mask size");
15635     for (unsigned I = 0; I < Size; I += 2) {
15636       if (Mask[I] != (int)(Begin0 + I / 2) ||
15637           Mask[I + 1] != (int)(Begin1 + I / 2))
15638         return false;
15639     }
15640     return true;
15641   };
15642   // Check which half is this shuffle node
15643   int NumElts = VT.getVectorNumElements();
15644   size_t FirstQtr = NumElts / 2;
15645   size_t ThirdQtr = NumElts + NumElts / 2;
15646   bool IsFirstHalf = IsInterleavingPattern(Mask, 0, NumElts);
15647   bool IsSecondHalf = IsInterleavingPattern(Mask, FirstQtr, ThirdQtr);
15648   if (!IsFirstHalf && !IsSecondHalf)
15649     return SDValue();
15650 
15651   // Find the intersection between shuffle users of V1 and V2.
15652   SmallVector<SDNode *, 2> Shuffles;
15653   for (SDNode *User : V1->uses())
15654     if (User->getOpcode() == ISD::VECTOR_SHUFFLE && User->getOperand(0) == V1 &&
15655         User->getOperand(1) == V2)
15656       Shuffles.push_back(User);
15657   // Limit user size to two for now.
15658   if (Shuffles.size() != 2)
15659     return SDValue();
15660   // Find out which half of the 512-bit shuffles is each smaller shuffle
15661   auto *SVN1 = cast<ShuffleVectorSDNode>(Shuffles[0]);
15662   auto *SVN2 = cast<ShuffleVectorSDNode>(Shuffles[1]);
15663   SDNode *FirstHalf;
15664   SDNode *SecondHalf;
15665   if (IsInterleavingPattern(SVN1->getMask(), 0, NumElts) &&
15666       IsInterleavingPattern(SVN2->getMask(), FirstQtr, ThirdQtr)) {
15667     FirstHalf = Shuffles[0];
15668     SecondHalf = Shuffles[1];
15669   } else if (IsInterleavingPattern(SVN1->getMask(), FirstQtr, ThirdQtr) &&
15670              IsInterleavingPattern(SVN2->getMask(), 0, NumElts)) {
15671     FirstHalf = Shuffles[1];
15672     SecondHalf = Shuffles[0];
15673   } else {
15674     return SDValue();
15675   }
15676   // Lower into unpck and perm. Return the perm of this shuffle and replace
15677   // the other.
15678   SDValue Unpckl = DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
15679   SDValue Unpckh = DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
15680   SDValue Perm1 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
15681                               DAG.getTargetConstant(0x20, DL, MVT::i8));
15682   SDValue Perm2 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
15683                               DAG.getTargetConstant(0x31, DL, MVT::i8));
15684   if (IsFirstHalf) {
15685     DAG.ReplaceAllUsesWith(SecondHalf, &Perm2);
15686     return Perm1;
15687   }
15688   DAG.ReplaceAllUsesWith(FirstHalf, &Perm1);
15689   return Perm2;
15690 }
15691 
15692 /// Handle lowering of 4-lane 64-bit floating point shuffles.
15693 ///
15694 /// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
15695 /// isn't available.
15696 static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15697                                  const APInt &Zeroable, SDValue V1, SDValue V2,
15698                                  const X86Subtarget &Subtarget,
15699                                  SelectionDAG &DAG) {
15700   assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
15701   assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
15702   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
15703 
15704   if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
15705                                      Subtarget, DAG))
15706     return V;
15707 
15708   if (V2.isUndef()) {
15709     // Check for being able to broadcast a single element.
15710     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
15711                                                     Mask, Subtarget, DAG))
15712       return Broadcast;
15713 
15714     // Use low duplicate instructions for masks that match their pattern.
15715     if (isShuffleEquivalent(Mask, {0, 0, 2, 2}, V1, V2))
15716       return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
15717 
15718     if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) {
15719       // Non-half-crossing single input shuffles can be lowered with an
15720       // interleaved permutation.
15721       unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
15722                               ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3);
15723       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
15724                          DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
15725     }
15726 
15727     // With AVX2 we have direct support for this permutation.
15728     if (Subtarget.hasAVX2())
15729       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
15730                          getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
15731 
15732     // Try to create an in-lane repeating shuffle mask and then shuffle the
15733     // results into the target lanes.
15734     if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15735             DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15736       return V;
15737 
15738     // Try to permute the lanes and then use a per-lane permute.
15739     if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
15740                                                         Mask, DAG, Subtarget))
15741       return V;
15742 
15743     // Otherwise, fall back.
15744     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v4f64, V1, V2, Mask,
15745                                                DAG, Subtarget);
15746   }
15747 
15748   // Use dedicated unpack instructions for masks that match their pattern.
15749   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
15750     return V;
15751 
15752   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
15753                                           Zeroable, Subtarget, DAG))
15754     return Blend;
15755 
15756   // Check if the blend happens to exactly fit that of SHUFPD.
15757   if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
15758                                           Zeroable, Subtarget, DAG))
15759     return Op;
15760 
15761   bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
15762   bool V2IsInPlace = isShuffleMaskInputInPlace(1, Mask);
15763 
15764   // If we have lane crossing shuffles AND they don't all come from the lower
15765   // lane elements, lower to SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
15766   // TODO: Handle BUILD_VECTOR sources which getVectorShuffle currently
15767   // canonicalize to a blend of splat which isn't necessary for this combine.
15768   if (is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask) &&
15769       !all_of(Mask, [](int M) { return M < 2 || (4 <= M && M < 6); }) &&
15770       (V1.getOpcode() != ISD::BUILD_VECTOR) &&
15771       (V2.getOpcode() != ISD::BUILD_VECTOR))
15772     return lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2, Mask, DAG);
15773 
15774   // If we have one input in place, then we can permute the other input and
15775   // blend the result.
15776   if (V1IsInPlace || V2IsInPlace)
15777     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
15778                                                 Subtarget, DAG);
15779 
15780   // Try to create an in-lane repeating shuffle mask and then shuffle the
15781   // results into the target lanes.
15782   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15783           DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15784     return V;
15785 
15786   // Try to simplify this by merging 128-bit lanes to enable a lane-based
15787   // shuffle. However, if we have AVX2 and either inputs are already in place,
15788   // we will be able to shuffle even across lanes the other input in a single
15789   // instruction so skip this pattern.
15790   if (!(Subtarget.hasAVX2() && (V1IsInPlace || V2IsInPlace)))
15791     if (SDValue V = lowerShuffleAsLanePermuteAndRepeatedMask(
15792             DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15793       return V;
15794 
15795   // If we have VLX support, we can use VEXPAND.
15796   if (Subtarget.hasVLX())
15797     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,
15798                                          DAG, Subtarget))
15799       return V;
15800 
15801   // If we have AVX2 then we always want to lower with a blend because an v4 we
15802   // can fully permute the elements.
15803   if (Subtarget.hasAVX2())
15804     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
15805                                                 Subtarget, DAG);
15806 
15807   // Otherwise fall back on generic lowering.
15808   return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,
15809                                     Subtarget, DAG);
15810 }
15811 
15812 /// Handle lowering of 4-lane 64-bit integer shuffles.
15813 ///
15814 /// This routine is only called when we have AVX2 and thus a reasonable
15815 /// instruction set for v4i64 shuffling..
15816 static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15817                                  const APInt &Zeroable, SDValue V1, SDValue V2,
15818                                  const X86Subtarget &Subtarget,
15819                                  SelectionDAG &DAG) {
15820   assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
15821   assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
15822   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
15823   assert(Subtarget.hasAVX2() && "We can only lower v4i64 with AVX2!");
15824 
15825   if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
15826                                      Subtarget, DAG))
15827     return V;
15828 
15829   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
15830                                           Zeroable, Subtarget, DAG))
15831     return Blend;
15832 
15833   // Check for being able to broadcast a single element.
15834   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
15835                                                   Subtarget, DAG))
15836     return Broadcast;
15837 
15838   // Try to use shift instructions if fast.
15839   if (Subtarget.preferLowerShuffleAsShift())
15840     if (SDValue Shift =
15841             lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
15842                                 Subtarget, DAG, /*BitwiseOnly*/ true))
15843       return Shift;
15844 
15845   if (V2.isUndef()) {
15846     // When the shuffle is mirrored between the 128-bit lanes of the unit, we
15847     // can use lower latency instructions that will operate on both lanes.
15848     SmallVector<int, 2> RepeatedMask;
15849     if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) {
15850       SmallVector<int, 4> PSHUFDMask;
15851       narrowShuffleMaskElts(2, RepeatedMask, PSHUFDMask);
15852       return DAG.getBitcast(
15853           MVT::v4i64,
15854           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
15855                       DAG.getBitcast(MVT::v8i32, V1),
15856                       getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
15857     }
15858 
15859     // AVX2 provides a direct instruction for permuting a single input across
15860     // lanes.
15861     return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
15862                        getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
15863   }
15864 
15865   // Try to use shift instructions.
15866   if (SDValue Shift =
15867           lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, Zeroable, Subtarget,
15868                               DAG, /*BitwiseOnly*/ false))
15869     return Shift;
15870 
15871   // If we have VLX support, we can use VALIGN or VEXPAND.
15872   if (Subtarget.hasVLX()) {
15873     if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i64, V1, V2, Mask,
15874                                               Zeroable, Subtarget, DAG))
15875       return Rotate;
15876 
15877     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,
15878                                          DAG, Subtarget))
15879       return V;
15880   }
15881 
15882   // Try to use PALIGNR.
15883   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
15884                                                 Subtarget, DAG))
15885     return Rotate;
15886 
15887   // Use dedicated unpack instructions for masks that match their pattern.
15888   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
15889     return V;
15890 
15891   bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
15892   bool V2IsInPlace = isShuffleMaskInputInPlace(1, Mask);
15893 
15894   // If we have one input in place, then we can permute the other input and
15895   // blend the result.
15896   if (V1IsInPlace || V2IsInPlace)
15897     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
15898                                                 Subtarget, DAG);
15899 
15900   // Try to create an in-lane repeating shuffle mask and then shuffle the
15901   // results into the target lanes.
15902   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15903           DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
15904     return V;
15905 
15906   // Try to lower to PERMQ(BLENDD(V1,V2)).
15907   if (SDValue V =
15908           lowerShuffleAsBlendAndPermute(DL, MVT::v4i64, V1, V2, Mask, DAG))
15909     return V;
15910 
15911   // Try to simplify this by merging 128-bit lanes to enable a lane-based
15912   // shuffle. However, if we have AVX2 and either inputs are already in place,
15913   // we will be able to shuffle even across lanes the other input in a single
15914   // instruction so skip this pattern.
15915   if (!V1IsInPlace && !V2IsInPlace)
15916     if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
15917             DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
15918       return Result;
15919 
15920   // Otherwise fall back on generic blend lowering.
15921   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
15922                                               Subtarget, DAG);
15923 }
15924 
15925 /// Handle lowering of 8-lane 32-bit floating point shuffles.
15926 ///
15927 /// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2
15928 /// isn't available.
15929 static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15930                                  const APInt &Zeroable, SDValue V1, SDValue V2,
15931                                  const X86Subtarget &Subtarget,
15932                                  SelectionDAG &DAG) {
15933   assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
15934   assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
15935   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
15936 
15937   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
15938                                           Zeroable, Subtarget, DAG))
15939     return Blend;
15940 
15941   // Check for being able to broadcast a single element.
15942   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
15943                                                   Subtarget, DAG))
15944     return Broadcast;
15945 
15946   if (!Subtarget.hasAVX2()) {
15947     SmallVector<int> InLaneMask;
15948     computeInLaneShuffleMask(Mask, Mask.size() / 2, InLaneMask);
15949 
15950     if (!is128BitLaneRepeatedShuffleMask(MVT::v8f32, InLaneMask))
15951       if (SDValue R = splitAndLowerShuffle(DL, MVT::v8f32, V1, V2, Mask, DAG,
15952                                            /*SimpleOnly*/ true))
15953         return R;
15954   }
15955   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
15956                                                    Zeroable, Subtarget, DAG))
15957     return DAG.getBitcast(MVT::v8f32, ZExt);
15958 
15959   // If the shuffle mask is repeated in each 128-bit lane, we have many more
15960   // options to efficiently lower the shuffle.
15961   SmallVector<int, 4> RepeatedMask;
15962   if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) {
15963     assert(RepeatedMask.size() == 4 &&
15964            "Repeated masks must be half the mask width!");
15965 
15966     // Use even/odd duplicate instructions for masks that match their pattern.
15967     if (isShuffleEquivalent(RepeatedMask, {0, 0, 2, 2}, V1, V2))
15968       return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
15969     if (isShuffleEquivalent(RepeatedMask, {1, 1, 3, 3}, V1, V2))
15970       return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
15971 
15972     if (V2.isUndef())
15973       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
15974                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
15975 
15976     // Use dedicated unpack instructions for masks that match their pattern.
15977     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
15978       return V;
15979 
15980     // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
15981     // have already handled any direct blends.
15982     return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
15983   }
15984 
15985   // Try to create an in-lane repeating shuffle mask and then shuffle the
15986   // results into the target lanes.
15987   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15988           DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
15989     return V;
15990 
15991   // If we have a single input shuffle with different shuffle patterns in the
15992   // two 128-bit lanes use the variable mask to VPERMILPS.
15993   if (V2.isUndef()) {
15994     if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) {
15995       SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
15996       return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask);
15997     }
15998     if (Subtarget.hasAVX2()) {
15999       SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16000       return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);
16001     }
16002     // Otherwise, fall back.
16003     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v8f32, V1, V2, Mask,
16004                                                DAG, Subtarget);
16005   }
16006 
16007   // Try to simplify this by merging 128-bit lanes to enable a lane-based
16008   // shuffle.
16009   if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16010           DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16011     return Result;
16012 
16013   // If we have VLX support, we can use VEXPAND.
16014   if (Subtarget.hasVLX())
16015     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,
16016                                          DAG, Subtarget))
16017       return V;
16018 
16019   // Try to match an interleave of two v8f32s and lower them as unpck and
16020   // permutes using ymms. This needs to go before we try to split the vectors.
16021   //
16022   // TODO: Expand this to AVX1. Currently v8i32 is casted to v8f32 and hits
16023   // this path inadvertently.
16024   if (Subtarget.hasAVX2() && !Subtarget.hasAVX512())
16025     if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v8f32, V1, V2,
16026                                                       Mask, DAG))
16027       return V;
16028 
16029   // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
16030   // since after split we get a more efficient code using vpunpcklwd and
16031   // vpunpckhwd instrs than vblend.
16032   if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG))
16033     return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Subtarget,
16034                                       DAG);
16035 
16036   // If we have AVX2 then we always want to lower with a blend because at v8 we
16037   // can fully permute the elements.
16038   if (Subtarget.hasAVX2())
16039     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8f32, V1, V2, Mask,
16040                                                 Subtarget, DAG);
16041 
16042   // Otherwise fall back on generic lowering.
16043   return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
16044                                     Subtarget, DAG);
16045 }
16046 
16047 /// Handle lowering of 8-lane 32-bit integer shuffles.
16048 ///
16049 /// This routine is only called when we have AVX2 and thus a reasonable
16050 /// instruction set for v8i32 shuffling..
16051 static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16052                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16053                                  const X86Subtarget &Subtarget,
16054                                  SelectionDAG &DAG) {
16055   assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
16056   assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
16057   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16058   assert(Subtarget.hasAVX2() && "We can only lower v8i32 with AVX2!");
16059 
16060   int NumV2Elements = count_if(Mask, [](int M) { return M >= 8; });
16061 
16062   // Whenever we can lower this as a zext, that instruction is strictly faster
16063   // than any alternative. It also allows us to fold memory operands into the
16064   // shuffle in many cases.
16065   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16066                                                    Zeroable, Subtarget, DAG))
16067     return ZExt;
16068 
16069   // Try to match an interleave of two v8i32s and lower them as unpck and
16070   // permutes using ymms. This needs to go before we try to split the vectors.
16071   if (!Subtarget.hasAVX512())
16072     if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v8i32, V1, V2,
16073                                                       Mask, DAG))
16074       return V;
16075 
16076   // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
16077   // since after split we get a more efficient code than vblend by using
16078   // vpunpcklwd and vpunpckhwd instrs.
16079   if (isUnpackWdShuffleMask(Mask, MVT::v8i32, DAG) && !V2.isUndef() &&
16080       !Subtarget.hasAVX512())
16081     return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Subtarget,
16082                                       DAG);
16083 
16084   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
16085                                           Zeroable, Subtarget, DAG))
16086     return Blend;
16087 
16088   // Check for being able to broadcast a single element.
16089   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
16090                                                   Subtarget, DAG))
16091     return Broadcast;
16092 
16093   // Try to use shift instructions if fast.
16094   if (Subtarget.preferLowerShuffleAsShift()) {
16095     if (SDValue Shift =
16096             lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, Zeroable,
16097                                 Subtarget, DAG, /*BitwiseOnly*/ true))
16098       return Shift;
16099     if (NumV2Elements == 0)
16100       if (SDValue Rotate =
16101               lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG))
16102         return Rotate;
16103   }
16104 
16105   // If the shuffle mask is repeated in each 128-bit lane we can use more
16106   // efficient instructions that mirror the shuffles across the two 128-bit
16107   // lanes.
16108   SmallVector<int, 4> RepeatedMask;
16109   bool Is128BitLaneRepeatedShuffle =
16110       is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask);
16111   if (Is128BitLaneRepeatedShuffle) {
16112     assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16113     if (V2.isUndef())
16114       return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
16115                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16116 
16117     // Use dedicated unpack instructions for masks that match their pattern.
16118     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
16119       return V;
16120   }
16121 
16122   // Try to use shift instructions.
16123   if (SDValue Shift =
16124           lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget,
16125                               DAG, /*BitwiseOnly*/ false))
16126     return Shift;
16127 
16128   if (!Subtarget.preferLowerShuffleAsShift() && NumV2Elements == 0)
16129     if (SDValue Rotate =
16130             lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG))
16131       return Rotate;
16132 
16133   // If we have VLX support, we can use VALIGN or EXPAND.
16134   if (Subtarget.hasVLX()) {
16135     if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i32, V1, V2, Mask,
16136                                               Zeroable, Subtarget, DAG))
16137       return Rotate;
16138 
16139     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,
16140                                          DAG, Subtarget))
16141       return V;
16142   }
16143 
16144   // Try to use byte rotation instructions.
16145   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
16146                                                 Subtarget, DAG))
16147     return Rotate;
16148 
16149   // Try to create an in-lane repeating shuffle mask and then shuffle the
16150   // results into the target lanes.
16151   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16152           DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16153     return V;
16154 
16155   if (V2.isUndef()) {
16156     // Try to produce a fixed cross-128-bit lane permute followed by unpack
16157     // because that should be faster than the variable permute alternatives.
16158     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG))
16159       return V;
16160 
16161     // If the shuffle patterns aren't repeated but it's a single input, directly
16162     // generate a cross-lane VPERMD instruction.
16163     SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16164     return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1);
16165   }
16166 
16167   // Assume that a single SHUFPS is faster than an alternative sequence of
16168   // multiple instructions (even if the CPU has a domain penalty).
16169   // If some CPU is harmed by the domain switch, we can fix it in a later pass.
16170   if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
16171     SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
16172     SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
16173     SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
16174                                             CastV1, CastV2, DAG);
16175     return DAG.getBitcast(MVT::v8i32, ShufPS);
16176   }
16177 
16178   // Try to simplify this by merging 128-bit lanes to enable a lane-based
16179   // shuffle.
16180   if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16181           DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16182     return Result;
16183 
16184   // Otherwise fall back on generic blend lowering.
16185   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i32, V1, V2, Mask,
16186                                               Subtarget, DAG);
16187 }
16188 
16189 /// Handle lowering of 16-lane 16-bit integer shuffles.
16190 ///
16191 /// This routine is only called when we have AVX2 and thus a reasonable
16192 /// instruction set for v16i16 shuffling..
16193 static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16194                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16195                                   const X86Subtarget &Subtarget,
16196                                   SelectionDAG &DAG) {
16197   assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
16198   assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
16199   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16200   assert(Subtarget.hasAVX2() && "We can only lower v16i16 with AVX2!");
16201 
16202   // Whenever we can lower this as a zext, that instruction is strictly faster
16203   // than any alternative. It also allows us to fold memory operands into the
16204   // shuffle in many cases.
16205   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16206           DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16207     return ZExt;
16208 
16209   // Check for being able to broadcast a single element.
16210   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
16211                                                   Subtarget, DAG))
16212     return Broadcast;
16213 
16214   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
16215                                           Zeroable, Subtarget, DAG))
16216     return Blend;
16217 
16218   // Use dedicated unpack instructions for masks that match their pattern.
16219   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
16220     return V;
16221 
16222   // Use dedicated pack instructions for masks that match their pattern.
16223   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
16224                                        Subtarget))
16225     return V;
16226 
16227   // Try to use lower using a truncation.
16228   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16229                                        Subtarget, DAG))
16230     return V;
16231 
16232   // Try to use shift instructions.
16233   if (SDValue Shift =
16234           lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16235                               Subtarget, DAG, /*BitwiseOnly*/ false))
16236     return Shift;
16237 
16238   // Try to use byte rotation instructions.
16239   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
16240                                                 Subtarget, DAG))
16241     return Rotate;
16242 
16243   // Try to create an in-lane repeating shuffle mask and then shuffle the
16244   // results into the target lanes.
16245   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16246           DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16247     return V;
16248 
16249   if (V2.isUndef()) {
16250     // Try to use bit rotation instructions.
16251     if (SDValue Rotate =
16252             lowerShuffleAsBitRotate(DL, MVT::v16i16, V1, Mask, Subtarget, DAG))
16253       return Rotate;
16254 
16255     // Try to produce a fixed cross-128-bit lane permute followed by unpack
16256     // because that should be faster than the variable permute alternatives.
16257     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG))
16258       return V;
16259 
16260     // There are no generalized cross-lane shuffle operations available on i16
16261     // element types.
16262     if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) {
16263       if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16264               DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16265         return V;
16266 
16267       return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v16i16, V1, V2, Mask,
16268                                                  DAG, Subtarget);
16269     }
16270 
16271     SmallVector<int, 8> RepeatedMask;
16272     if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
16273       // As this is a single-input shuffle, the repeated mask should be
16274       // a strictly valid v8i16 mask that we can pass through to the v8i16
16275       // lowering to handle even the v16 case.
16276       return lowerV8I16GeneralSingleInputShuffle(
16277           DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
16278     }
16279   }
16280 
16281   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
16282                                               Zeroable, Subtarget, DAG))
16283     return PSHUFB;
16284 
16285   // AVX512BW can lower to VPERMW (non-VLX will pad to v32i16).
16286   if (Subtarget.hasBWI())
16287     return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, Subtarget, DAG);
16288 
16289   // Try to simplify this by merging 128-bit lanes to enable a lane-based
16290   // shuffle.
16291   if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16292           DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16293     return Result;
16294 
16295   // Try to permute the lanes and then use a per-lane permute.
16296   if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16297           DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16298     return V;
16299 
16300   // Try to match an interleave of two v16i16s and lower them as unpck and
16301   // permutes using ymms.
16302   if (!Subtarget.hasAVX512())
16303     if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v16i16, V1, V2,
16304                                                       Mask, DAG))
16305       return V;
16306 
16307   // Otherwise fall back on generic lowering.
16308   return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,
16309                                     Subtarget, DAG);
16310 }
16311 
16312 /// Handle lowering of 32-lane 8-bit integer shuffles.
16313 ///
16314 /// This routine is only called when we have AVX2 and thus a reasonable
16315 /// instruction set for v32i8 shuffling..
16316 static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16317                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16318                                  const X86Subtarget &Subtarget,
16319                                  SelectionDAG &DAG) {
16320   assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
16321   assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
16322   assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
16323   assert(Subtarget.hasAVX2() && "We can only lower v32i8 with AVX2!");
16324 
16325   // Whenever we can lower this as a zext, that instruction is strictly faster
16326   // than any alternative. It also allows us to fold memory operands into the
16327   // shuffle in many cases.
16328   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
16329                                                    Zeroable, Subtarget, DAG))
16330     return ZExt;
16331 
16332   // Check for being able to broadcast a single element.
16333   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
16334                                                   Subtarget, DAG))
16335     return Broadcast;
16336 
16337   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
16338                                           Zeroable, Subtarget, DAG))
16339     return Blend;
16340 
16341   // Use dedicated unpack instructions for masks that match their pattern.
16342   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
16343     return V;
16344 
16345   // Use dedicated pack instructions for masks that match their pattern.
16346   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
16347                                        Subtarget))
16348     return V;
16349 
16350   // Try to use lower using a truncation.
16351   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
16352                                        Subtarget, DAG))
16353     return V;
16354 
16355   // Try to use shift instructions.
16356   if (SDValue Shift =
16357           lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget,
16358                               DAG, /*BitwiseOnly*/ false))
16359     return Shift;
16360 
16361   // Try to use byte rotation instructions.
16362   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
16363                                                 Subtarget, DAG))
16364     return Rotate;
16365 
16366   // Try to use bit rotation instructions.
16367   if (V2.isUndef())
16368     if (SDValue Rotate =
16369             lowerShuffleAsBitRotate(DL, MVT::v32i8, V1, Mask, Subtarget, DAG))
16370       return Rotate;
16371 
16372   // Try to create an in-lane repeating shuffle mask and then shuffle the
16373   // results into the target lanes.
16374   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16375           DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16376     return V;
16377 
16378   // There are no generalized cross-lane shuffle operations available on i8
16379   // element types.
16380   if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {
16381     // Try to produce a fixed cross-128-bit lane permute followed by unpack
16382     // because that should be faster than the variable permute alternatives.
16383     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG))
16384       return V;
16385 
16386     if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16387             DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16388       return V;
16389 
16390     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v32i8, V1, V2, Mask,
16391                                                DAG, Subtarget);
16392   }
16393 
16394   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
16395                                               Zeroable, Subtarget, DAG))
16396     return PSHUFB;
16397 
16398   // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8).
16399   if (Subtarget.hasVBMI())
16400     return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, Subtarget, DAG);
16401 
16402   // Try to simplify this by merging 128-bit lanes to enable a lane-based
16403   // shuffle.
16404   if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16405           DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16406     return Result;
16407 
16408   // Try to permute the lanes and then use a per-lane permute.
16409   if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16410           DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16411     return V;
16412 
16413   // Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
16414   // by zeroable elements in the remaining 24 elements. Turn this into two
16415   // vmovqb instructions shuffled together.
16416   if (Subtarget.hasVLX())
16417     if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
16418                                                   Mask, Zeroable, DAG))
16419       return V;
16420 
16421   // Try to match an interleave of two v32i8s and lower them as unpck and
16422   // permutes using ymms.
16423   if (!Subtarget.hasAVX512())
16424     if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v32i8, V1, V2,
16425                                                       Mask, DAG))
16426       return V;
16427 
16428   // Otherwise fall back on generic lowering.
16429   return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,
16430                                     Subtarget, DAG);
16431 }
16432 
16433 /// High-level routine to lower various 256-bit x86 vector shuffles.
16434 ///
16435 /// This routine either breaks down the specific type of a 256-bit x86 vector
16436 /// shuffle or splits it into two 128-bit shuffles and fuses the results back
16437 /// together based on the available instructions.
16438 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
16439                                   SDValue V1, SDValue V2, const APInt &Zeroable,
16440                                   const X86Subtarget &Subtarget,
16441                                   SelectionDAG &DAG) {
16442   // If we have a single input to the zero element, insert that into V1 if we
16443   // can do so cheaply.
16444   int NumElts = VT.getVectorNumElements();
16445   int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
16446 
16447   if (NumV2Elements == 1 && Mask[0] >= NumElts)
16448     if (SDValue Insertion = lowerShuffleAsElementInsertion(
16449             DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
16450       return Insertion;
16451 
16452   // Handle special cases where the lower or upper half is UNDEF.
16453   if (SDValue V =
16454           lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
16455     return V;
16456 
16457   // There is a really nice hard cut-over between AVX1 and AVX2 that means we
16458   // can check for those subtargets here and avoid much of the subtarget
16459   // querying in the per-vector-type lowering routines. With AVX1 we have
16460   // essentially *zero* ability to manipulate a 256-bit vector with integer
16461   // types. Since we'll use floating point types there eventually, just
16462   // immediately cast everything to a float and operate entirely in that domain.
16463   if (VT.isInteger() && !Subtarget.hasAVX2()) {
16464     int ElementBits = VT.getScalarSizeInBits();
16465     if (ElementBits < 32) {
16466       // No floating point type available, if we can't use the bit operations
16467       // for masking/blending then decompose into 128-bit vectors.
16468       if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16469                                             Subtarget, DAG))
16470         return V;
16471       if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
16472         return V;
16473       return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
16474     }
16475 
16476     MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
16477                                 VT.getVectorNumElements());
16478     V1 = DAG.getBitcast(FpVT, V1);
16479     V2 = DAG.getBitcast(FpVT, V2);
16480     return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
16481   }
16482 
16483   if (VT == MVT::v16f16 || VT == MVT::v16bf16) {
16484     V1 = DAG.getBitcast(MVT::v16i16, V1);
16485     V2 = DAG.getBitcast(MVT::v16i16, V2);
16486     return DAG.getBitcast(VT,
16487                           DAG.getVectorShuffle(MVT::v16i16, DL, V1, V2, Mask));
16488   }
16489 
16490   switch (VT.SimpleTy) {
16491   case MVT::v4f64:
16492     return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16493   case MVT::v4i64:
16494     return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16495   case MVT::v8f32:
16496     return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16497   case MVT::v8i32:
16498     return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16499   case MVT::v16i16:
16500     return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16501   case MVT::v32i8:
16502     return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16503 
16504   default:
16505     llvm_unreachable("Not a valid 256-bit x86 vector type!");
16506   }
16507 }
16508 
16509 /// Try to lower a vector shuffle as a 128-bit shuffles.
16510 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
16511                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16512                                   const X86Subtarget &Subtarget,
16513                                   SelectionDAG &DAG) {
16514   assert(VT.getScalarSizeInBits() == 64 &&
16515          "Unexpected element type size for 128bit shuffle.");
16516 
16517   // To handle 256 bit vector requires VLX and most probably
16518   // function lowerV2X128VectorShuffle() is better solution.
16519   assert(VT.is512BitVector() && "Unexpected vector size for 512bit shuffle.");
16520 
16521   // TODO - use Zeroable like we do for lowerV2X128VectorShuffle?
16522   SmallVector<int, 4> Widened128Mask;
16523   if (!canWidenShuffleElements(Mask, Widened128Mask))
16524     return SDValue();
16525   assert(Widened128Mask.size() == 4 && "Shuffle widening mismatch");
16526 
16527   // Try to use an insert into a zero vector.
16528   if (Widened128Mask[0] == 0 && (Zeroable & 0xf0) == 0xf0 &&
16529       (Widened128Mask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) {
16530     unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4;
16531     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
16532     SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
16533                               DAG.getIntPtrConstant(0, DL));
16534     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
16535                        getZeroVector(VT, Subtarget, DAG, DL), LoV,
16536                        DAG.getIntPtrConstant(0, DL));
16537   }
16538 
16539   // Check for patterns which can be matched with a single insert of a 256-bit
16540   // subvector.
16541   bool OnlyUsesV1 = isShuffleEquivalent(Mask, {0, 1, 2, 3, 0, 1, 2, 3}, V1, V2);
16542   if (OnlyUsesV1 ||
16543       isShuffleEquivalent(Mask, {0, 1, 2, 3, 8, 9, 10, 11}, V1, V2)) {
16544     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4);
16545     SDValue SubVec =
16546         DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2,
16547                     DAG.getIntPtrConstant(0, DL));
16548     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
16549                        DAG.getIntPtrConstant(4, DL));
16550   }
16551 
16552   // See if this is an insertion of the lower 128-bits of V2 into V1.
16553   bool IsInsert = true;
16554   int V2Index = -1;
16555   for (int i = 0; i < 4; ++i) {
16556     assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value");
16557     if (Widened128Mask[i] < 0)
16558       continue;
16559 
16560     // Make sure all V1 subvectors are in place.
16561     if (Widened128Mask[i] < 4) {
16562       if (Widened128Mask[i] != i) {
16563         IsInsert = false;
16564         break;
16565       }
16566     } else {
16567       // Make sure we only have a single V2 index and its the lowest 128-bits.
16568       if (V2Index >= 0 || Widened128Mask[i] != 4) {
16569         IsInsert = false;
16570         break;
16571       }
16572       V2Index = i;
16573     }
16574   }
16575   if (IsInsert && V2Index >= 0) {
16576     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
16577     SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
16578                                  DAG.getIntPtrConstant(0, DL));
16579     return insert128BitVector(V1, Subvec, V2Index * 2, DAG, DL);
16580   }
16581 
16582   // See if we can widen to a 256-bit lane shuffle, we're going to lose 128-lane
16583   // UNDEF info by lowering to X86ISD::SHUF128 anyway, so by widening where
16584   // possible we at least ensure the lanes stay sequential to help later
16585   // combines.
16586   SmallVector<int, 2> Widened256Mask;
16587   if (canWidenShuffleElements(Widened128Mask, Widened256Mask)) {
16588     Widened128Mask.clear();
16589     narrowShuffleMaskElts(2, Widened256Mask, Widened128Mask);
16590   }
16591 
16592   // Try to lower to vshuf64x2/vshuf32x4.
16593   SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
16594   int PermMask[4] = {-1, -1, -1, -1};
16595   // Ensure elements came from the same Op.
16596   for (int i = 0; i < 4; ++i) {
16597     assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value");
16598     if (Widened128Mask[i] < 0)
16599       continue;
16600 
16601     SDValue Op = Widened128Mask[i] >= 4 ? V2 : V1;
16602     unsigned OpIndex = i / 2;
16603     if (Ops[OpIndex].isUndef())
16604       Ops[OpIndex] = Op;
16605     else if (Ops[OpIndex] != Op)
16606       return SDValue();
16607 
16608     PermMask[i] = Widened128Mask[i] % 4;
16609   }
16610 
16611   return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
16612                      getV4X86ShuffleImm8ForMask(PermMask, DL, DAG));
16613 }
16614 
16615 /// Handle lowering of 8-lane 64-bit floating point shuffles.
16616 static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16617                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16618                                  const X86Subtarget &Subtarget,
16619                                  SelectionDAG &DAG) {
16620   assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
16621   assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
16622   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16623 
16624   if (V2.isUndef()) {
16625     // Use low duplicate instructions for masks that match their pattern.
16626     if (isShuffleEquivalent(Mask, {0, 0, 2, 2, 4, 4, 6, 6}, V1, V2))
16627       return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);
16628 
16629     if (!is128BitLaneCrossingShuffleMask(MVT::v8f64, Mask)) {
16630       // Non-half-crossing single input shuffles can be lowered with an
16631       // interleaved permutation.
16632       unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
16633                               ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3) |
16634                               ((Mask[4] == 5) << 4) | ((Mask[5] == 5) << 5) |
16635                               ((Mask[6] == 7) << 6) | ((Mask[7] == 7) << 7);
16636       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
16637                          DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
16638     }
16639 
16640     SmallVector<int, 4> RepeatedMask;
16641     if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask))
16642       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,
16643                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16644   }
16645 
16646   if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
16647                                            V2, Subtarget, DAG))
16648     return Shuf128;
16649 
16650   if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
16651     return Unpck;
16652 
16653   // Check if the blend happens to exactly fit that of SHUFPD.
16654   if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
16655                                           Zeroable, Subtarget, DAG))
16656     return Op;
16657 
16658   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
16659                                        DAG, Subtarget))
16660     return V;
16661 
16662   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
16663                                           Zeroable, Subtarget, DAG))
16664     return Blend;
16665 
16666   return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG);
16667 }
16668 
16669 /// Handle lowering of 16-lane 32-bit floating point shuffles.
16670 static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16671                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16672                                   const X86Subtarget &Subtarget,
16673                                   SelectionDAG &DAG) {
16674   assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
16675   assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
16676   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16677 
16678   // If the shuffle mask is repeated in each 128-bit lane, we have many more
16679   // options to efficiently lower the shuffle.
16680   SmallVector<int, 4> RepeatedMask;
16681   if (is128BitLaneRepeatedShuffleMask(MVT::v16f32, Mask, RepeatedMask)) {
16682     assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16683 
16684     // Use even/odd duplicate instructions for masks that match their pattern.
16685     if (isShuffleEquivalent(RepeatedMask, {0, 0, 2, 2}, V1, V2))
16686       return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);
16687     if (isShuffleEquivalent(RepeatedMask, {1, 1, 3, 3}, V1, V2))
16688       return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);
16689 
16690     if (V2.isUndef())
16691       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
16692                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16693 
16694     // Use dedicated unpack instructions for masks that match their pattern.
16695     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
16696       return V;
16697 
16698     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
16699                                             Zeroable, Subtarget, DAG))
16700       return Blend;
16701 
16702     // Otherwise, fall back to a SHUFPS sequence.
16703     return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
16704   }
16705 
16706   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
16707                                           Zeroable, Subtarget, DAG))
16708     return Blend;
16709 
16710   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16711           DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
16712     return DAG.getBitcast(MVT::v16f32, ZExt);
16713 
16714   // Try to create an in-lane repeating shuffle mask and then shuffle the
16715   // results into the target lanes.
16716   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16717           DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG))
16718     return V;
16719 
16720   // If we have a single input shuffle with different shuffle patterns in the
16721   // 128-bit lanes and don't lane cross, use variable mask VPERMILPS.
16722   if (V2.isUndef() &&
16723       !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {
16724     SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
16725     return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
16726   }
16727 
16728   // If we have AVX512F support, we can use VEXPAND.
16729   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
16730                                              V1, V2, DAG, Subtarget))
16731     return V;
16732 
16733   return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG);
16734 }
16735 
16736 /// Handle lowering of 8-lane 64-bit integer shuffles.
16737 static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16738                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16739                                  const X86Subtarget &Subtarget,
16740                                  SelectionDAG &DAG) {
16741   assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
16742   assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
16743   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16744 
16745   // Try to use shift instructions if fast.
16746   if (Subtarget.preferLowerShuffleAsShift())
16747     if (SDValue Shift =
16748             lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, Zeroable,
16749                                 Subtarget, DAG, /*BitwiseOnly*/ true))
16750       return Shift;
16751 
16752   if (V2.isUndef()) {
16753     // When the shuffle is mirrored between the 128-bit lanes of the unit, we
16754     // can use lower latency instructions that will operate on all four
16755     // 128-bit lanes.
16756     SmallVector<int, 2> Repeated128Mask;
16757     if (is128BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated128Mask)) {
16758       SmallVector<int, 4> PSHUFDMask;
16759       narrowShuffleMaskElts(2, Repeated128Mask, PSHUFDMask);
16760       return DAG.getBitcast(
16761           MVT::v8i64,
16762           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
16763                       DAG.getBitcast(MVT::v16i32, V1),
16764                       getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
16765     }
16766 
16767     SmallVector<int, 4> Repeated256Mask;
16768     if (is256BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated256Mask))
16769       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,
16770                          getV4X86ShuffleImm8ForMask(Repeated256Mask, DL, DAG));
16771   }
16772 
16773   if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
16774                                            V2, Subtarget, DAG))
16775     return Shuf128;
16776 
16777   // Try to use shift instructions.
16778   if (SDValue Shift =
16779           lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, Zeroable, Subtarget,
16780                               DAG, /*BitwiseOnly*/ false))
16781     return Shift;
16782 
16783   // Try to use VALIGN.
16784   if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i64, V1, V2, Mask,
16785                                             Zeroable, Subtarget, DAG))
16786     return Rotate;
16787 
16788   // Try to use PALIGNR.
16789   if (Subtarget.hasBWI())
16790     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
16791                                                   Subtarget, DAG))
16792       return Rotate;
16793 
16794   if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
16795     return Unpck;
16796 
16797   // If we have AVX512F support, we can use VEXPAND.
16798   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
16799                                        DAG, Subtarget))
16800     return V;
16801 
16802   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
16803                                           Zeroable, Subtarget, DAG))
16804     return Blend;
16805 
16806   return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, Subtarget, DAG);
16807 }
16808 
16809 /// Handle lowering of 16-lane 32-bit integer shuffles.
16810 static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16811                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16812                                   const X86Subtarget &Subtarget,
16813                                   SelectionDAG &DAG) {
16814   assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
16815   assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
16816   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16817 
16818   int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
16819 
16820   // Whenever we can lower this as a zext, that instruction is strictly faster
16821   // than any alternative. It also allows us to fold memory operands into the
16822   // shuffle in many cases.
16823   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16824           DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
16825     return ZExt;
16826 
16827   // Try to use shift instructions if fast.
16828   if (Subtarget.preferLowerShuffleAsShift()) {
16829     if (SDValue Shift =
16830             lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, Zeroable,
16831                                 Subtarget, DAG, /*BitwiseOnly*/ true))
16832       return Shift;
16833     if (NumV2Elements == 0)
16834       if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask,
16835                                                    Subtarget, DAG))
16836         return Rotate;
16837   }
16838 
16839   // If the shuffle mask is repeated in each 128-bit lane we can use more
16840   // efficient instructions that mirror the shuffles across the four 128-bit
16841   // lanes.
16842   SmallVector<int, 4> RepeatedMask;
16843   bool Is128BitLaneRepeatedShuffle =
16844       is128BitLaneRepeatedShuffleMask(MVT::v16i32, Mask, RepeatedMask);
16845   if (Is128BitLaneRepeatedShuffle) {
16846     assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16847     if (V2.isUndef())
16848       return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,
16849                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16850 
16851     // Use dedicated unpack instructions for masks that match their pattern.
16852     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
16853       return V;
16854   }
16855 
16856   // Try to use shift instructions.
16857   if (SDValue Shift =
16858           lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, Zeroable,
16859                               Subtarget, DAG, /*BitwiseOnly*/ false))
16860     return Shift;
16861 
16862   if (!Subtarget.preferLowerShuffleAsShift() && NumV2Elements != 0)
16863     if (SDValue Rotate =
16864             lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask, Subtarget, DAG))
16865       return Rotate;
16866 
16867   // Try to use VALIGN.
16868   if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v16i32, V1, V2, Mask,
16869                                             Zeroable, Subtarget, DAG))
16870     return Rotate;
16871 
16872   // Try to use byte rotation instructions.
16873   if (Subtarget.hasBWI())
16874     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
16875                                                   Subtarget, DAG))
16876       return Rotate;
16877 
16878   // Assume that a single SHUFPS is faster than using a permv shuffle.
16879   // If some CPU is harmed by the domain switch, we can fix it in a later pass.
16880   if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
16881     SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
16882     SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
16883     SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
16884                                             CastV1, CastV2, DAG);
16885     return DAG.getBitcast(MVT::v16i32, ShufPS);
16886   }
16887 
16888   // Try to create an in-lane repeating shuffle mask and then shuffle the
16889   // results into the target lanes.
16890   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16891           DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))
16892     return V;
16893 
16894   // If we have AVX512F support, we can use VEXPAND.
16895   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
16896                                        DAG, Subtarget))
16897     return V;
16898 
16899   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
16900                                           Zeroable, Subtarget, DAG))
16901     return Blend;
16902 
16903   return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, Subtarget, DAG);
16904 }
16905 
16906 /// Handle lowering of 32-lane 16-bit integer shuffles.
16907 static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16908                                   const APInt &Zeroable, SDValue V1, SDValue V2,
16909                                   const X86Subtarget &Subtarget,
16910                                   SelectionDAG &DAG) {
16911   assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
16912   assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
16913   assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
16914   assert(Subtarget.hasBWI() && "We can only lower v32i16 with AVX-512-BWI!");
16915 
16916   // Whenever we can lower this as a zext, that instruction is strictly faster
16917   // than any alternative. It also allows us to fold memory operands into the
16918   // shuffle in many cases.
16919   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16920           DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16921     return ZExt;
16922 
16923   // Use dedicated unpack instructions for masks that match their pattern.
16924   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
16925     return V;
16926 
16927   // Use dedicated pack instructions for masks that match their pattern.
16928   if (SDValue V =
16929           lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget))
16930     return V;
16931 
16932   // Try to use shift instructions.
16933   if (SDValue Shift =
16934           lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask, Zeroable,
16935                               Subtarget, DAG, /*BitwiseOnly*/ false))
16936     return Shift;
16937 
16938   // Try to use byte rotation instructions.
16939   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
16940                                                 Subtarget, DAG))
16941     return Rotate;
16942 
16943   if (V2.isUndef()) {
16944     // Try to use bit rotation instructions.
16945     if (SDValue Rotate =
16946             lowerShuffleAsBitRotate(DL, MVT::v32i16, V1, Mask, Subtarget, DAG))
16947       return Rotate;
16948 
16949     SmallVector<int, 8> RepeatedMask;
16950     if (is128BitLaneRepeatedShuffleMask(MVT::v32i16, Mask, RepeatedMask)) {
16951       // As this is a single-input shuffle, the repeated mask should be
16952       // a strictly valid v8i16 mask that we can pass through to the v8i16
16953       // lowering to handle even the v32 case.
16954       return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v32i16, V1,
16955                                                  RepeatedMask, Subtarget, DAG);
16956     }
16957   }
16958 
16959   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
16960                                           Zeroable, Subtarget, DAG))
16961     return Blend;
16962 
16963   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
16964                                               Zeroable, Subtarget, DAG))
16965     return PSHUFB;
16966 
16967   return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG);
16968 }
16969 
16970 /// Handle lowering of 64-lane 8-bit integer shuffles.
16971 static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16972                                  const APInt &Zeroable, SDValue V1, SDValue V2,
16973                                  const X86Subtarget &Subtarget,
16974                                  SelectionDAG &DAG) {
16975   assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
16976   assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
16977   assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!");
16978   assert(Subtarget.hasBWI() && "We can only lower v64i8 with AVX-512-BWI!");
16979 
16980   // Whenever we can lower this as a zext, that instruction is strictly faster
16981   // than any alternative. It also allows us to fold memory operands into the
16982   // shuffle in many cases.
16983   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16984           DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
16985     return ZExt;
16986 
16987   // Use dedicated unpack instructions for masks that match their pattern.
16988   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
16989     return V;
16990 
16991   // Use dedicated pack instructions for masks that match their pattern.
16992   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
16993                                        Subtarget))
16994     return V;
16995 
16996   // Try to use shift instructions.
16997   if (SDValue Shift =
16998           lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget,
16999                               DAG, /*BitwiseOnly*/ false))
17000     return Shift;
17001 
17002   // Try to use byte rotation instructions.
17003   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
17004                                                 Subtarget, DAG))
17005     return Rotate;
17006 
17007   // Try to use bit rotation instructions.
17008   if (V2.isUndef())
17009     if (SDValue Rotate =
17010             lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG))
17011       return Rotate;
17012 
17013   // Lower as AND if possible.
17014   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v64i8, V1, V2, Mask,
17015                                              Zeroable, Subtarget, DAG))
17016     return Masked;
17017 
17018   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
17019                                               Zeroable, Subtarget, DAG))
17020     return PSHUFB;
17021 
17022   // Try to create an in-lane repeating shuffle mask and then shuffle the
17023   // results into the target lanes.
17024   if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17025           DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17026     return V;
17027 
17028   if (SDValue Result = lowerShuffleAsLanePermuteAndPermute(
17029           DL, MVT::v64i8, V1, V2, Mask, DAG, Subtarget))
17030     return Result;
17031 
17032   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
17033                                           Zeroable, Subtarget, DAG))
17034     return Blend;
17035 
17036   if (!is128BitLaneCrossingShuffleMask(MVT::v64i8, Mask)) {
17037     // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
17038     // PALIGNR will be cheaper than the second PSHUFB+OR.
17039     if (SDValue V = lowerShuffleAsByteRotateAndPermute(DL, MVT::v64i8, V1, V2,
17040                                                        Mask, Subtarget, DAG))
17041       return V;
17042 
17043     // If we can't directly blend but can use PSHUFB, that will be better as it
17044     // can both shuffle and set up the inefficient blend.
17045     bool V1InUse, V2InUse;
17046     return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v64i8, V1, V2, Mask, Zeroable,
17047                                         DAG, V1InUse, V2InUse);
17048   }
17049 
17050   // Try to simplify this by merging 128-bit lanes to enable a lane-based
17051   // shuffle.
17052   if (!V2.isUndef())
17053     if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
17054             DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17055       return Result;
17056 
17057   // VBMI can use VPERMV/VPERMV3 byte shuffles.
17058   if (Subtarget.hasVBMI())
17059     return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, Subtarget, DAG);
17060 
17061   return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
17062 }
17063 
17064 /// High-level routine to lower various 512-bit x86 vector shuffles.
17065 ///
17066 /// This routine either breaks down the specific type of a 512-bit x86 vector
17067 /// shuffle or splits it into two 256-bit shuffles and fuses the results back
17068 /// together based on the available instructions.
17069 static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17070                                   MVT VT, SDValue V1, SDValue V2,
17071                                   const APInt &Zeroable,
17072                                   const X86Subtarget &Subtarget,
17073                                   SelectionDAG &DAG) {
17074   assert(Subtarget.hasAVX512() &&
17075          "Cannot lower 512-bit vectors w/ basic ISA!");
17076 
17077   // If we have a single input to the zero element, insert that into V1 if we
17078   // can do so cheaply.
17079   int NumElts = Mask.size();
17080   int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
17081 
17082   if (NumV2Elements == 1 && Mask[0] >= NumElts)
17083     if (SDValue Insertion = lowerShuffleAsElementInsertion(
17084             DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
17085       return Insertion;
17086 
17087   // Handle special cases where the lower or upper half is UNDEF.
17088   if (SDValue V =
17089           lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
17090     return V;
17091 
17092   // Check for being able to broadcast a single element.
17093   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
17094                                                   Subtarget, DAG))
17095     return Broadcast;
17096 
17097   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) {
17098     // Try using bit ops for masking and blending before falling back to
17099     // splitting.
17100     if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
17101                                           Subtarget, DAG))
17102       return V;
17103     if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
17104       return V;
17105 
17106     return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
17107   }
17108 
17109   if (VT == MVT::v32f16 || VT == MVT::v32bf16) {
17110     if (!Subtarget.hasBWI())
17111       return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
17112                                   /*SimpleOnly*/ false);
17113 
17114     V1 = DAG.getBitcast(MVT::v32i16, V1);
17115     V2 = DAG.getBitcast(MVT::v32i16, V2);
17116     return DAG.getBitcast(VT,
17117                           DAG.getVectorShuffle(MVT::v32i16, DL, V1, V2, Mask));
17118   }
17119 
17120   // Dispatch to each element type for lowering. If we don't have support for
17121   // specific element type shuffles at 512 bits, immediately split them and
17122   // lower them. Each lowering routine of a given type is allowed to assume that
17123   // the requisite ISA extensions for that element type are available.
17124   switch (VT.SimpleTy) {
17125   case MVT::v8f64:
17126     return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17127   case MVT::v16f32:
17128     return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17129   case MVT::v8i64:
17130     return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17131   case MVT::v16i32:
17132     return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17133   case MVT::v32i16:
17134     return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17135   case MVT::v64i8:
17136     return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17137 
17138   default:
17139     llvm_unreachable("Not a valid 512-bit x86 vector type!");
17140   }
17141 }
17142 
17143 static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
17144                                          MVT VT, SDValue V1, SDValue V2,
17145                                          const X86Subtarget &Subtarget,
17146                                          SelectionDAG &DAG) {
17147   // Shuffle should be unary.
17148   if (!V2.isUndef())
17149     return SDValue();
17150 
17151   int ShiftAmt = -1;
17152   int NumElts = Mask.size();
17153   for (int i = 0; i != NumElts; ++i) {
17154     int M = Mask[i];
17155     assert((M == SM_SentinelUndef || (0 <= M && M < NumElts)) &&
17156            "Unexpected mask index.");
17157     if (M < 0)
17158       continue;
17159 
17160     // The first non-undef element determines our shift amount.
17161     if (ShiftAmt < 0) {
17162       ShiftAmt = M - i;
17163       // Need to be shifting right.
17164       if (ShiftAmt <= 0)
17165         return SDValue();
17166     }
17167     // All non-undef elements must shift by the same amount.
17168     if (ShiftAmt != M - i)
17169       return SDValue();
17170   }
17171   assert(ShiftAmt >= 0 && "All undef?");
17172 
17173   // Great we found a shift right.
17174   SDValue Res = widenMaskVector(V1, false, Subtarget, DAG, DL);
17175   Res = DAG.getNode(X86ISD::KSHIFTR, DL, Res.getValueType(), Res,
17176                     DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17177   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17178                      DAG.getIntPtrConstant(0, DL));
17179 }
17180 
17181 // Determine if this shuffle can be implemented with a KSHIFT instruction.
17182 // Returns the shift amount if possible or -1 if not. This is a simplified
17183 // version of matchShuffleAsShift.
17184 static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef<int> Mask,
17185                                     int MaskOffset, const APInt &Zeroable) {
17186   int Size = Mask.size();
17187 
17188   auto CheckZeros = [&](int Shift, bool Left) {
17189     for (int j = 0; j < Shift; ++j)
17190       if (!Zeroable[j + (Left ? 0 : (Size - Shift))])
17191         return false;
17192 
17193     return true;
17194   };
17195 
17196   auto MatchShift = [&](int Shift, bool Left) {
17197     unsigned Pos = Left ? Shift : 0;
17198     unsigned Low = Left ? 0 : Shift;
17199     unsigned Len = Size - Shift;
17200     return isSequentialOrUndefInRange(Mask, Pos, Len, Low + MaskOffset);
17201   };
17202 
17203   for (int Shift = 1; Shift != Size; ++Shift)
17204     for (bool Left : {true, false})
17205       if (CheckZeros(Shift, Left) && MatchShift(Shift, Left)) {
17206         Opcode = Left ? X86ISD::KSHIFTL : X86ISD::KSHIFTR;
17207         return Shift;
17208       }
17209 
17210   return -1;
17211 }
17212 
17213 
17214 // Lower vXi1 vector shuffles.
17215 // There is no a dedicated instruction on AVX-512 that shuffles the masks.
17216 // The only way to shuffle bits is to sign-extend the mask vector to SIMD
17217 // vector, shuffle and then truncate it back.
17218 static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17219                                 MVT VT, SDValue V1, SDValue V2,
17220                                 const APInt &Zeroable,
17221                                 const X86Subtarget &Subtarget,
17222                                 SelectionDAG &DAG) {
17223   assert(Subtarget.hasAVX512() &&
17224          "Cannot lower 512-bit vectors w/o basic ISA!");
17225 
17226   int NumElts = Mask.size();
17227   int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
17228 
17229   // Try to recognize shuffles that are just padding a subvector with zeros.
17230   int SubvecElts = 0;
17231   int Src = -1;
17232   for (int i = 0; i != NumElts; ++i) {
17233     if (Mask[i] >= 0) {
17234       // Grab the source from the first valid mask. All subsequent elements need
17235       // to use this same source.
17236       if (Src < 0)
17237         Src = Mask[i] / NumElts;
17238       if (Src != (Mask[i] / NumElts) || (Mask[i] % NumElts) != i)
17239         break;
17240     }
17241 
17242     ++SubvecElts;
17243   }
17244   assert(SubvecElts != NumElts && "Identity shuffle?");
17245 
17246   // Clip to a power 2.
17247   SubvecElts = llvm::bit_floor<uint32_t>(SubvecElts);
17248 
17249   // Make sure the number of zeroable bits in the top at least covers the bits
17250   // not covered by the subvector.
17251   if ((int)Zeroable.countl_one() >= (NumElts - SubvecElts)) {
17252     assert(Src >= 0 && "Expected a source!");
17253     MVT ExtractVT = MVT::getVectorVT(MVT::i1, SubvecElts);
17254     SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT,
17255                                   Src == 0 ? V1 : V2,
17256                                   DAG.getIntPtrConstant(0, DL));
17257     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
17258                        DAG.getConstant(0, DL, VT),
17259                        Extract, DAG.getIntPtrConstant(0, DL));
17260   }
17261 
17262   // Try a simple shift right with undef elements. Later we'll try with zeros.
17263   if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget,
17264                                                 DAG))
17265     return Shift;
17266 
17267   // Try to match KSHIFTs.
17268   unsigned Offset = 0;
17269   for (SDValue V : { V1, V2 }) {
17270     unsigned Opcode;
17271     int ShiftAmt = match1BitShuffleAsKSHIFT(Opcode, Mask, Offset, Zeroable);
17272     if (ShiftAmt >= 0) {
17273       SDValue Res = widenMaskVector(V, false, Subtarget, DAG, DL);
17274       MVT WideVT = Res.getSimpleValueType();
17275       // Widened right shifts need two shifts to ensure we shift in zeroes.
17276       if (Opcode == X86ISD::KSHIFTR && WideVT != VT) {
17277         int WideElts = WideVT.getVectorNumElements();
17278         // Shift left to put the original vector in the MSBs of the new size.
17279         Res = DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res,
17280                           DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8));
17281         // Increase the shift amount to account for the left shift.
17282         ShiftAmt += WideElts - NumElts;
17283       }
17284 
17285       Res = DAG.getNode(Opcode, DL, WideVT, Res,
17286                         DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17287       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17288                          DAG.getIntPtrConstant(0, DL));
17289     }
17290     Offset += NumElts; // Increment for next iteration.
17291   }
17292 
17293   // If we're performing an unary shuffle on a SETCC result, try to shuffle the
17294   // ops instead.
17295   // TODO: What other unary shuffles would benefit from this?
17296   if (NumV2Elements == 0 && V1.getOpcode() == ISD::SETCC && V1->hasOneUse()) {
17297     SDValue Op0 = V1.getOperand(0);
17298     SDValue Op1 = V1.getOperand(1);
17299     ISD::CondCode CC = cast<CondCodeSDNode>(V1.getOperand(2))->get();
17300     EVT OpVT = Op0.getValueType();
17301     if (OpVT.getScalarSizeInBits() >= 32 || isBroadcastShuffleMask(Mask))
17302       return DAG.getSetCC(
17303           DL, VT, DAG.getVectorShuffle(OpVT, DL, Op0, DAG.getUNDEF(OpVT), Mask),
17304           DAG.getVectorShuffle(OpVT, DL, Op1, DAG.getUNDEF(OpVT), Mask), CC);
17305   }
17306 
17307   MVT ExtVT;
17308   switch (VT.SimpleTy) {
17309   default:
17310     llvm_unreachable("Expected a vector of i1 elements");
17311   case MVT::v2i1:
17312     ExtVT = MVT::v2i64;
17313     break;
17314   case MVT::v4i1:
17315     ExtVT = MVT::v4i32;
17316     break;
17317   case MVT::v8i1:
17318     // Take 512-bit type, more shuffles on KNL. If we have VLX use a 256-bit
17319     // shuffle.
17320     ExtVT = Subtarget.hasVLX() ? MVT::v8i32 : MVT::v8i64;
17321     break;
17322   case MVT::v16i1:
17323     // Take 512-bit type, unless we are avoiding 512-bit types and have the
17324     // 256-bit operation available.
17325     ExtVT = Subtarget.canExtendTo512DQ() ? MVT::v16i32 : MVT::v16i16;
17326     break;
17327   case MVT::v32i1:
17328     // Take 512-bit type, unless we are avoiding 512-bit types and have the
17329     // 256-bit operation available.
17330     assert(Subtarget.hasBWI() && "Expected AVX512BW support");
17331     ExtVT = Subtarget.canExtendTo512BW() ? MVT::v32i16 : MVT::v32i8;
17332     break;
17333   case MVT::v64i1:
17334     // Fall back to scalarization. FIXME: We can do better if the shuffle
17335     // can be partitioned cleanly.
17336     if (!Subtarget.useBWIRegs())
17337       return SDValue();
17338     ExtVT = MVT::v64i8;
17339     break;
17340   }
17341 
17342   V1 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V1);
17343   V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2);
17344 
17345   SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17346   // i1 was sign extended we can use X86ISD::CVT2MASK.
17347   int NumElems = VT.getVectorNumElements();
17348   if ((Subtarget.hasBWI() && (NumElems >= 32)) ||
17349       (Subtarget.hasDQI() && (NumElems < 32)))
17350     return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),
17351                        Shuffle, ISD::SETGT);
17352 
17353   return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);
17354 }
17355 
17356 /// Helper function that returns true if the shuffle mask should be
17357 /// commuted to improve canonicalization.
17358 static bool canonicalizeShuffleMaskWithCommute(ArrayRef<int> Mask) {
17359   int NumElements = Mask.size();
17360 
17361   int NumV1Elements = 0, NumV2Elements = 0;
17362   for (int M : Mask)
17363     if (M < 0)
17364       continue;
17365     else if (M < NumElements)
17366       ++NumV1Elements;
17367     else
17368       ++NumV2Elements;
17369 
17370   // Commute the shuffle as needed such that more elements come from V1 than
17371   // V2. This allows us to match the shuffle pattern strictly on how many
17372   // elements come from V1 without handling the symmetric cases.
17373   if (NumV2Elements > NumV1Elements)
17374     return true;
17375 
17376   assert(NumV1Elements > 0 && "No V1 indices");
17377 
17378   if (NumV2Elements == 0)
17379     return false;
17380 
17381   // When the number of V1 and V2 elements are the same, try to minimize the
17382   // number of uses of V2 in the low half of the vector. When that is tied,
17383   // ensure that the sum of indices for V1 is equal to or lower than the sum
17384   // indices for V2. When those are equal, try to ensure that the number of odd
17385   // indices for V1 is lower than the number of odd indices for V2.
17386   if (NumV1Elements == NumV2Elements) {
17387     int LowV1Elements = 0, LowV2Elements = 0;
17388     for (int M : Mask.slice(0, NumElements / 2))
17389       if (M >= NumElements)
17390         ++LowV2Elements;
17391       else if (M >= 0)
17392         ++LowV1Elements;
17393     if (LowV2Elements > LowV1Elements)
17394       return true;
17395     if (LowV2Elements == LowV1Elements) {
17396       int SumV1Indices = 0, SumV2Indices = 0;
17397       for (int i = 0, Size = Mask.size(); i < Size; ++i)
17398         if (Mask[i] >= NumElements)
17399           SumV2Indices += i;
17400         else if (Mask[i] >= 0)
17401           SumV1Indices += i;
17402       if (SumV2Indices < SumV1Indices)
17403         return true;
17404       if (SumV2Indices == SumV1Indices) {
17405         int NumV1OddIndices = 0, NumV2OddIndices = 0;
17406         for (int i = 0, Size = Mask.size(); i < Size; ++i)
17407           if (Mask[i] >= NumElements)
17408             NumV2OddIndices += i % 2;
17409           else if (Mask[i] >= 0)
17410             NumV1OddIndices += i % 2;
17411         if (NumV2OddIndices < NumV1OddIndices)
17412           return true;
17413       }
17414     }
17415   }
17416 
17417   return false;
17418 }
17419 
17420 static bool canCombineAsMaskOperation(SDValue V,
17421                                       const X86Subtarget &Subtarget) {
17422   if (!Subtarget.hasAVX512())
17423     return false;
17424 
17425   if (!V.getValueType().isSimple())
17426     return false;
17427 
17428   MVT VT = V.getSimpleValueType().getScalarType();
17429   if ((VT == MVT::i16 || VT == MVT::i8) && !Subtarget.hasBWI())
17430     return false;
17431 
17432   // If vec width < 512, widen i8/i16 even with BWI as blendd/blendps/blendpd
17433   // are preferable to blendw/blendvb/masked-mov.
17434   if ((VT == MVT::i16 || VT == MVT::i8) &&
17435       V.getSimpleValueType().getSizeInBits() < 512)
17436     return false;
17437 
17438   auto HasMaskOperation = [&](SDValue V) {
17439     // TODO: Currently we only check limited opcode. We probably extend
17440     // it to all binary operation by checking TLI.isBinOp().
17441     switch (V->getOpcode()) {
17442     default:
17443       return false;
17444     case ISD::ADD:
17445     case ISD::SUB:
17446     case ISD::AND:
17447     case ISD::XOR:
17448     case ISD::OR:
17449     case ISD::SMAX:
17450     case ISD::SMIN:
17451     case ISD::UMAX:
17452     case ISD::UMIN:
17453     case ISD::ABS:
17454     case ISD::SHL:
17455     case ISD::SRL:
17456     case ISD::SRA:
17457     case ISD::MUL:
17458       break;
17459     }
17460     if (!V->hasOneUse())
17461       return false;
17462 
17463     return true;
17464   };
17465 
17466   if (HasMaskOperation(V))
17467     return true;
17468 
17469   return false;
17470 }
17471 
17472 // Forward declaration.
17473 static SDValue canonicalizeShuffleMaskWithHorizOp(
17474     MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask,
17475     unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
17476     const X86Subtarget &Subtarget);
17477 
17478     /// Top-level lowering for x86 vector shuffles.
17479 ///
17480 /// This handles decomposition, canonicalization, and lowering of all x86
17481 /// vector shuffles. Most of the specific lowering strategies are encapsulated
17482 /// above in helper routines. The canonicalization attempts to widen shuffles
17483 /// to involve fewer lanes of wider elements, consolidate symmetric patterns
17484 /// s.t. only one of the two inputs needs to be tested, etc.
17485 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
17486                                    SelectionDAG &DAG) {
17487   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
17488   ArrayRef<int> OrigMask = SVOp->getMask();
17489   SDValue V1 = Op.getOperand(0);
17490   SDValue V2 = Op.getOperand(1);
17491   MVT VT = Op.getSimpleValueType();
17492   int NumElements = VT.getVectorNumElements();
17493   SDLoc DL(Op);
17494   bool Is1BitVector = (VT.getVectorElementType() == MVT::i1);
17495 
17496   assert((VT.getSizeInBits() != 64 || Is1BitVector) &&
17497          "Can't lower MMX shuffles");
17498 
17499   bool V1IsUndef = V1.isUndef();
17500   bool V2IsUndef = V2.isUndef();
17501   if (V1IsUndef && V2IsUndef)
17502     return DAG.getUNDEF(VT);
17503 
17504   // When we create a shuffle node we put the UNDEF node to second operand,
17505   // but in some cases the first operand may be transformed to UNDEF.
17506   // In this case we should just commute the node.
17507   if (V1IsUndef)
17508     return DAG.getCommutedVectorShuffle(*SVOp);
17509 
17510   // Check for non-undef masks pointing at an undef vector and make the masks
17511   // undef as well. This makes it easier to match the shuffle based solely on
17512   // the mask.
17513   if (V2IsUndef &&
17514       any_of(OrigMask, [NumElements](int M) { return M >= NumElements; })) {
17515     SmallVector<int, 8> NewMask(OrigMask);
17516     for (int &M : NewMask)
17517       if (M >= NumElements)
17518         M = -1;
17519     return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
17520   }
17521 
17522   // Check for illegal shuffle mask element index values.
17523   int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
17524   (void)MaskUpperLimit;
17525   assert(llvm::all_of(OrigMask,
17526                       [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&
17527          "Out of bounds shuffle index");
17528 
17529   // We actually see shuffles that are entirely re-arrangements of a set of
17530   // zero inputs. This mostly happens while decomposing complex shuffles into
17531   // simple ones. Directly lower these as a buildvector of zeros.
17532   APInt KnownUndef, KnownZero;
17533   computeZeroableShuffleElements(OrigMask, V1, V2, KnownUndef, KnownZero);
17534 
17535   APInt Zeroable = KnownUndef | KnownZero;
17536   if (Zeroable.isAllOnes())
17537     return getZeroVector(VT, Subtarget, DAG, DL);
17538 
17539   bool V2IsZero = !V2IsUndef && ISD::isBuildVectorAllZeros(V2.getNode());
17540 
17541   // Try to collapse shuffles into using a vector type with fewer elements but
17542   // wider element types. We cap this to not form integers or floating point
17543   // elements wider than 64 bits. It does not seem beneficial to form i128
17544   // integers to handle flipping the low and high halves of AVX 256-bit vectors.
17545   SmallVector<int, 16> WidenedMask;
17546   if (VT.getScalarSizeInBits() < 64 && !Is1BitVector &&
17547       !canCombineAsMaskOperation(V1, Subtarget) &&
17548       !canCombineAsMaskOperation(V2, Subtarget) &&
17549       canWidenShuffleElements(OrigMask, Zeroable, V2IsZero, WidenedMask)) {
17550     // Shuffle mask widening should not interfere with a broadcast opportunity
17551     // by obfuscating the operands with bitcasts.
17552     // TODO: Avoid lowering directly from this top-level function: make this
17553     // a query (canLowerAsBroadcast) and defer lowering to the type-based calls.
17554     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
17555                                                     Subtarget, DAG))
17556       return Broadcast;
17557 
17558     MVT NewEltVT = VT.isFloatingPoint()
17559                        ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)
17560                        : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);
17561     int NewNumElts = NumElements / 2;
17562     MVT NewVT = MVT::getVectorVT(NewEltVT, NewNumElts);
17563     // Make sure that the new vector type is legal. For example, v2f64 isn't
17564     // legal on SSE1.
17565     if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
17566       if (V2IsZero) {
17567         // Modify the new Mask to take all zeros from the all-zero vector.
17568         // Choose indices that are blend-friendly.
17569         bool UsedZeroVector = false;
17570         assert(is_contained(WidenedMask, SM_SentinelZero) &&
17571                "V2's non-undef elements are used?!");
17572         for (int i = 0; i != NewNumElts; ++i)
17573           if (WidenedMask[i] == SM_SentinelZero) {
17574             WidenedMask[i] = i + NewNumElts;
17575             UsedZeroVector = true;
17576           }
17577         // Ensure all elements of V2 are zero - isBuildVectorAllZeros permits
17578         // some elements to be undef.
17579         if (UsedZeroVector)
17580           V2 = getZeroVector(NewVT, Subtarget, DAG, DL);
17581       }
17582       V1 = DAG.getBitcast(NewVT, V1);
17583       V2 = DAG.getBitcast(NewVT, V2);
17584       return DAG.getBitcast(
17585           VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask));
17586     }
17587   }
17588 
17589   SmallVector<SDValue> Ops = {V1, V2};
17590   SmallVector<int> Mask(OrigMask);
17591 
17592   // Canonicalize the shuffle with any horizontal ops inputs.
17593   // NOTE: This may update Ops and Mask.
17594   if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp(
17595           Ops, Mask, VT.getSizeInBits(), DL, DAG, Subtarget))
17596     return DAG.getBitcast(VT, HOp);
17597 
17598   V1 = DAG.getBitcast(VT, Ops[0]);
17599   V2 = DAG.getBitcast(VT, Ops[1]);
17600   assert(NumElements == (int)Mask.size() &&
17601          "canonicalizeShuffleMaskWithHorizOp "
17602          "shouldn't alter the shuffle mask size");
17603 
17604   // Commute the shuffle if it will improve canonicalization.
17605   if (canonicalizeShuffleMaskWithCommute(Mask)) {
17606     ShuffleVectorSDNode::commuteMask(Mask);
17607     std::swap(V1, V2);
17608   }
17609 
17610   // For each vector width, delegate to a specialized lowering routine.
17611   if (VT.is128BitVector())
17612     return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17613 
17614   if (VT.is256BitVector())
17615     return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17616 
17617   if (VT.is512BitVector())
17618     return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17619 
17620   if (Is1BitVector)
17621     return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17622 
17623   llvm_unreachable("Unimplemented!");
17624 }
17625 
17626 /// Try to lower a VSELECT instruction to a vector shuffle.
17627 static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
17628                                            const X86Subtarget &Subtarget,
17629                                            SelectionDAG &DAG) {
17630   SDValue Cond = Op.getOperand(0);
17631   SDValue LHS = Op.getOperand(1);
17632   SDValue RHS = Op.getOperand(2);
17633   MVT VT = Op.getSimpleValueType();
17634 
17635   // Only non-legal VSELECTs reach this lowering, convert those into generic
17636   // shuffles and re-use the shuffle lowering path for blends.
17637   if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) {
17638     SmallVector<int, 32> Mask;
17639     if (createShuffleMaskFromVSELECT(Mask, Cond))
17640       return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask);
17641   }
17642 
17643   return SDValue();
17644 }
17645 
17646 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
17647   SDValue Cond = Op.getOperand(0);
17648   SDValue LHS = Op.getOperand(1);
17649   SDValue RHS = Op.getOperand(2);
17650 
17651   SDLoc dl(Op);
17652   MVT VT = Op.getSimpleValueType();
17653   if (isSoftF16(VT, Subtarget)) {
17654     MVT NVT = VT.changeVectorElementTypeToInteger();
17655     return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, dl, NVT, Cond,
17656                                           DAG.getBitcast(NVT, LHS),
17657                                           DAG.getBitcast(NVT, RHS)));
17658   }
17659 
17660   // A vselect where all conditions and data are constants can be optimized into
17661   // a single vector load by SelectionDAGLegalize::ExpandBUILD_VECTOR().
17662   if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()) &&
17663       ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()) &&
17664       ISD::isBuildVectorOfConstantSDNodes(RHS.getNode()))
17665     return SDValue();
17666 
17667   // Try to lower this to a blend-style vector shuffle. This can handle all
17668   // constant condition cases.
17669   if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
17670     return BlendOp;
17671 
17672   // If this VSELECT has a vector if i1 as a mask, it will be directly matched
17673   // with patterns on the mask registers on AVX-512.
17674   MVT CondVT = Cond.getSimpleValueType();
17675   unsigned CondEltSize = Cond.getScalarValueSizeInBits();
17676   if (CondEltSize == 1)
17677     return Op;
17678 
17679   // Variable blends are only legal from SSE4.1 onward.
17680   if (!Subtarget.hasSSE41())
17681     return SDValue();
17682 
17683   unsigned EltSize = VT.getScalarSizeInBits();
17684   unsigned NumElts = VT.getVectorNumElements();
17685 
17686   // Expand v32i16/v64i8 without BWI.
17687   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
17688     return SDValue();
17689 
17690   // If the VSELECT is on a 512-bit type, we have to convert a non-i1 condition
17691   // into an i1 condition so that we can use the mask-based 512-bit blend
17692   // instructions.
17693   if (VT.getSizeInBits() == 512) {
17694     // Build a mask by testing the condition against zero.
17695     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
17696     SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
17697                                 DAG.getConstant(0, dl, CondVT),
17698                                 ISD::SETNE);
17699     // Now return a new VSELECT using the mask.
17700     return DAG.getSelect(dl, VT, Mask, LHS, RHS);
17701   }
17702 
17703   // SEXT/TRUNC cases where the mask doesn't match the destination size.
17704   if (CondEltSize != EltSize) {
17705     // If we don't have a sign splat, rely on the expansion.
17706     if (CondEltSize != DAG.ComputeNumSignBits(Cond))
17707       return SDValue();
17708 
17709     MVT NewCondSVT = MVT::getIntegerVT(EltSize);
17710     MVT NewCondVT = MVT::getVectorVT(NewCondSVT, NumElts);
17711     Cond = DAG.getSExtOrTrunc(Cond, dl, NewCondVT);
17712     return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS);
17713   }
17714 
17715   // Only some types will be legal on some subtargets. If we can emit a legal
17716   // VSELECT-matching blend, return Op, and but if we need to expand, return
17717   // a null value.
17718   switch (VT.SimpleTy) {
17719   default:
17720     // Most of the vector types have blends past SSE4.1.
17721     return Op;
17722 
17723   case MVT::v32i8:
17724     // The byte blends for AVX vectors were introduced only in AVX2.
17725     if (Subtarget.hasAVX2())
17726       return Op;
17727 
17728     return SDValue();
17729 
17730   case MVT::v8i16:
17731   case MVT::v16i16: {
17732     // Bitcast everything to the vXi8 type and use a vXi8 vselect.
17733     MVT CastVT = MVT::getVectorVT(MVT::i8, NumElts * 2);
17734     Cond = DAG.getBitcast(CastVT, Cond);
17735     LHS = DAG.getBitcast(CastVT, LHS);
17736     RHS = DAG.getBitcast(CastVT, RHS);
17737     SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS);
17738     return DAG.getBitcast(VT, Select);
17739   }
17740   }
17741 }
17742 
17743 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
17744   MVT VT = Op.getSimpleValueType();
17745   SDValue Vec = Op.getOperand(0);
17746   SDValue Idx = Op.getOperand(1);
17747   assert(isa<ConstantSDNode>(Idx) && "Constant index expected");
17748   SDLoc dl(Op);
17749 
17750   if (!Vec.getSimpleValueType().is128BitVector())
17751     return SDValue();
17752 
17753   if (VT.getSizeInBits() == 8) {
17754     // If IdxVal is 0, it's cheaper to do a move instead of a pextrb, unless
17755     // we're going to zero extend the register or fold the store.
17756     if (llvm::isNullConstant(Idx) && !X86::mayFoldIntoZeroExtend(Op) &&
17757         !X86::mayFoldIntoStore(Op))
17758       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
17759                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17760                                      DAG.getBitcast(MVT::v4i32, Vec), Idx));
17761 
17762     unsigned IdxVal = Idx->getAsZExtVal();
17763     SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec,
17764                                   DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17765     return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
17766   }
17767 
17768   if (VT == MVT::f32) {
17769     // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
17770     // the result back to FR32 register. It's only worth matching if the
17771     // result has a single use which is a store or a bitcast to i32.  And in
17772     // the case of a store, it's not worth it if the index is a constant 0,
17773     // because a MOVSSmr can be used instead, which is smaller and faster.
17774     if (!Op.hasOneUse())
17775       return SDValue();
17776     SDNode *User = *Op.getNode()->use_begin();
17777     if ((User->getOpcode() != ISD::STORE || isNullConstant(Idx)) &&
17778         (User->getOpcode() != ISD::BITCAST ||
17779          User->getValueType(0) != MVT::i32))
17780       return SDValue();
17781     SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17782                                   DAG.getBitcast(MVT::v4i32, Vec), Idx);
17783     return DAG.getBitcast(MVT::f32, Extract);
17784   }
17785 
17786   if (VT == MVT::i32 || VT == MVT::i64)
17787       return Op;
17788 
17789   return SDValue();
17790 }
17791 
17792 /// Extract one bit from mask vector, like v16i1 or v8i1.
17793 /// AVX-512 feature.
17794 static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
17795                                         const X86Subtarget &Subtarget) {
17796   SDValue Vec = Op.getOperand(0);
17797   SDLoc dl(Vec);
17798   MVT VecVT = Vec.getSimpleValueType();
17799   SDValue Idx = Op.getOperand(1);
17800   auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
17801   MVT EltVT = Op.getSimpleValueType();
17802 
17803   assert((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) &&
17804          "Unexpected vector type in ExtractBitFromMaskVector");
17805 
17806   // variable index can't be handled in mask registers,
17807   // extend vector to VR512/128
17808   if (!IdxC) {
17809     unsigned NumElts = VecVT.getVectorNumElements();
17810     // Extending v8i1/v16i1 to 512-bit get better performance on KNL
17811     // than extending to 128/256bit.
17812     if (NumElts == 1) {
17813       Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
17814       MVT IntVT = MVT::getIntegerVT(Vec.getValueType().getVectorNumElements());
17815       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, DAG.getBitcast(IntVT, Vec));
17816     }
17817     MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
17818     MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
17819     SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec);
17820     SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx);
17821     return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
17822   }
17823 
17824   unsigned IdxVal = IdxC->getZExtValue();
17825   if (IdxVal == 0) // the operation is legal
17826     return Op;
17827 
17828   // Extend to natively supported kshift.
17829   Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
17830 
17831   // Use kshiftr instruction to move to the lower element.
17832   Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec,
17833                     DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17834 
17835   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
17836                      DAG.getIntPtrConstant(0, dl));
17837 }
17838 
17839 // Helper to find all the extracted elements from a vector.
17840 static APInt getExtractedDemandedElts(SDNode *N) {
17841   MVT VT = N->getSimpleValueType(0);
17842   unsigned NumElts = VT.getVectorNumElements();
17843   APInt DemandedElts = APInt::getZero(NumElts);
17844   for (SDNode *User : N->uses()) {
17845     switch (User->getOpcode()) {
17846     case X86ISD::PEXTRB:
17847     case X86ISD::PEXTRW:
17848     case ISD::EXTRACT_VECTOR_ELT:
17849       if (!isa<ConstantSDNode>(User->getOperand(1))) {
17850         DemandedElts.setAllBits();
17851         return DemandedElts;
17852       }
17853       DemandedElts.setBit(User->getConstantOperandVal(1));
17854       break;
17855     case ISD::BITCAST: {
17856       if (!User->getValueType(0).isSimple() ||
17857           !User->getValueType(0).isVector()) {
17858         DemandedElts.setAllBits();
17859         return DemandedElts;
17860       }
17861       APInt DemandedSrcElts = getExtractedDemandedElts(User);
17862       DemandedElts |= APIntOps::ScaleBitMask(DemandedSrcElts, NumElts);
17863       break;
17864     }
17865     default:
17866       DemandedElts.setAllBits();
17867       return DemandedElts;
17868     }
17869   }
17870   return DemandedElts;
17871 }
17872 
17873 SDValue
17874 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
17875                                            SelectionDAG &DAG) const {
17876   SDLoc dl(Op);
17877   SDValue Vec = Op.getOperand(0);
17878   MVT VecVT = Vec.getSimpleValueType();
17879   SDValue Idx = Op.getOperand(1);
17880   auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
17881 
17882   if (VecVT.getVectorElementType() == MVT::i1)
17883     return ExtractBitFromMaskVector(Op, DAG, Subtarget);
17884 
17885   if (!IdxC) {
17886     // Its more profitable to go through memory (1 cycles throughput)
17887     // than using VMOVD + VPERMV/PSHUFB sequence (2/3 cycles throughput)
17888     // IACA tool was used to get performance estimation
17889     // (https://software.intel.com/en-us/articles/intel-architecture-code-analyzer)
17890     //
17891     // example : extractelement <16 x i8> %a, i32 %i
17892     //
17893     // Block Throughput: 3.00 Cycles
17894     // Throughput Bottleneck: Port5
17895     //
17896     // | Num Of |   Ports pressure in cycles  |    |
17897     // |  Uops  |  0  - DV  |  5  |  6  |  7  |    |
17898     // ---------------------------------------------
17899     // |   1    |           | 1.0 |     |     | CP | vmovd xmm1, edi
17900     // |   1    |           | 1.0 |     |     | CP | vpshufb xmm0, xmm0, xmm1
17901     // |   2    | 1.0       | 1.0 |     |     | CP | vpextrb eax, xmm0, 0x0
17902     // Total Num Of Uops: 4
17903     //
17904     //
17905     // Block Throughput: 1.00 Cycles
17906     // Throughput Bottleneck: PORT2_AGU, PORT3_AGU, Port4
17907     //
17908     // |    |  Ports pressure in cycles   |  |
17909     // |Uops| 1 | 2 - D  |3 -  D  | 4 | 5 |  |
17910     // ---------------------------------------------------------
17911     // |2^  |   | 0.5    | 0.5    |1.0|   |CP| vmovaps xmmword ptr [rsp-0x18], xmm0
17912     // |1   |0.5|        |        |   |0.5|  | lea rax, ptr [rsp-0x18]
17913     // |1   |   |0.5, 0.5|0.5, 0.5|   |   |CP| mov al, byte ptr [rdi+rax*1]
17914     // Total Num Of Uops: 4
17915 
17916     return SDValue();
17917   }
17918 
17919   unsigned IdxVal = IdxC->getZExtValue();
17920 
17921   // If this is a 256-bit vector result, first extract the 128-bit vector and
17922   // then extract the element from the 128-bit vector.
17923   if (VecVT.is256BitVector() || VecVT.is512BitVector()) {
17924     // Get the 128-bit vector.
17925     Vec = extract128BitVector(Vec, IdxVal, DAG, dl);
17926     MVT EltVT = VecVT.getVectorElementType();
17927 
17928     unsigned ElemsPerChunk = 128 / EltVT.getSizeInBits();
17929     assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
17930 
17931     // Find IdxVal modulo ElemsPerChunk. Since ElemsPerChunk is a power of 2
17932     // this can be done with a mask.
17933     IdxVal &= ElemsPerChunk - 1;
17934     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
17935                        DAG.getIntPtrConstant(IdxVal, dl));
17936   }
17937 
17938   assert(VecVT.is128BitVector() && "Unexpected vector length");
17939 
17940   MVT VT = Op.getSimpleValueType();
17941 
17942   if (VT == MVT::i16) {
17943     // If IdxVal is 0, it's cheaper to do a move instead of a pextrw, unless
17944     // we're going to zero extend the register or fold the store (SSE41 only).
17945     if (IdxVal == 0 && !X86::mayFoldIntoZeroExtend(Op) &&
17946         !(Subtarget.hasSSE41() && X86::mayFoldIntoStore(Op))) {
17947       if (Subtarget.hasFP16())
17948         return Op;
17949 
17950       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
17951                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17952                                      DAG.getBitcast(MVT::v4i32, Vec), Idx));
17953     }
17954 
17955     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec,
17956                                   DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17957     return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
17958   }
17959 
17960   if (Subtarget.hasSSE41())
17961     if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
17962       return Res;
17963 
17964   // Only extract a single element from a v16i8 source - determine the common
17965   // DWORD/WORD that all extractions share, and extract the sub-byte.
17966   // TODO: Add QWORD MOVQ extraction?
17967   if (VT == MVT::i8) {
17968     APInt DemandedElts = getExtractedDemandedElts(Vec.getNode());
17969     assert(DemandedElts.getBitWidth() == 16 && "Vector width mismatch");
17970 
17971     // Extract either the lowest i32 or any i16, and extract the sub-byte.
17972     int DWordIdx = IdxVal / 4;
17973     if (DWordIdx == 0 && DemandedElts == (DemandedElts & 15)) {
17974       SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17975                                 DAG.getBitcast(MVT::v4i32, Vec),
17976                                 DAG.getIntPtrConstant(DWordIdx, dl));
17977       int ShiftVal = (IdxVal % 4) * 8;
17978       if (ShiftVal != 0)
17979         Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res,
17980                           DAG.getConstant(ShiftVal, dl, MVT::i8));
17981       return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
17982     }
17983 
17984     int WordIdx = IdxVal / 2;
17985     if (DemandedElts == (DemandedElts & (3 << (WordIdx * 2)))) {
17986       SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
17987                                 DAG.getBitcast(MVT::v8i16, Vec),
17988                                 DAG.getIntPtrConstant(WordIdx, dl));
17989       int ShiftVal = (IdxVal % 2) * 8;
17990       if (ShiftVal != 0)
17991         Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res,
17992                           DAG.getConstant(ShiftVal, dl, MVT::i8));
17993       return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
17994     }
17995   }
17996 
17997   if (VT == MVT::f16 || VT.getSizeInBits() == 32) {
17998     if (IdxVal == 0)
17999       return Op;
18000 
18001     // Shuffle the element to the lowest element, then movss or movsh.
18002     SmallVector<int, 8> Mask(VecVT.getVectorNumElements(), -1);
18003     Mask[0] = static_cast<int>(IdxVal);
18004     Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
18005     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
18006                        DAG.getIntPtrConstant(0, dl));
18007   }
18008 
18009   if (VT.getSizeInBits() == 64) {
18010     // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b
18011     // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught
18012     //        to match extract_elt for f64.
18013     if (IdxVal == 0)
18014       return Op;
18015 
18016     // UNPCKHPD the element to the lowest double word, then movsd.
18017     // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
18018     // to a f64mem, the whole operation is folded into a single MOVHPDmr.
18019     int Mask[2] = { 1, -1 };
18020     Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
18021     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
18022                        DAG.getIntPtrConstant(0, dl));
18023   }
18024 
18025   return SDValue();
18026 }
18027 
18028 /// Insert one bit to mask vector, like v16i1 or v8i1.
18029 /// AVX-512 feature.
18030 static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
18031                                      const X86Subtarget &Subtarget) {
18032   SDLoc dl(Op);
18033   SDValue Vec = Op.getOperand(0);
18034   SDValue Elt = Op.getOperand(1);
18035   SDValue Idx = Op.getOperand(2);
18036   MVT VecVT = Vec.getSimpleValueType();
18037 
18038   if (!isa<ConstantSDNode>(Idx)) {
18039     // Non constant index. Extend source and destination,
18040     // insert element and then truncate the result.
18041     unsigned NumElts = VecVT.getVectorNumElements();
18042     MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
18043     MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
18044     SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
18045       DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
18046       DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
18047     return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
18048   }
18049 
18050   // Copy into a k-register, extract to v1i1 and insert_subvector.
18051   SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);
18052   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VecVT, Vec, EltInVec, Idx);
18053 }
18054 
18055 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
18056                                                   SelectionDAG &DAG) const {
18057   MVT VT = Op.getSimpleValueType();
18058   MVT EltVT = VT.getVectorElementType();
18059   unsigned NumElts = VT.getVectorNumElements();
18060   unsigned EltSizeInBits = EltVT.getScalarSizeInBits();
18061 
18062   if (EltVT == MVT::i1)
18063     return InsertBitToMaskVector(Op, DAG, Subtarget);
18064 
18065   SDLoc dl(Op);
18066   SDValue N0 = Op.getOperand(0);
18067   SDValue N1 = Op.getOperand(1);
18068   SDValue N2 = Op.getOperand(2);
18069   auto *N2C = dyn_cast<ConstantSDNode>(N2);
18070 
18071   if (EltVT == MVT::bf16) {
18072     MVT IVT = VT.changeVectorElementTypeToInteger();
18073     SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT,
18074                               DAG.getBitcast(IVT, N0),
18075                               DAG.getBitcast(MVT::i16, N1), N2);
18076     return DAG.getBitcast(VT, Res);
18077   }
18078 
18079   if (!N2C) {
18080     // Variable insertion indices, usually we're better off spilling to stack,
18081     // but AVX512 can use a variable compare+select by comparing against all
18082     // possible vector indices, and FP insertion has less gpr->simd traffic.
18083     if (!(Subtarget.hasBWI() ||
18084           (Subtarget.hasAVX512() && EltSizeInBits >= 32) ||
18085           (Subtarget.hasSSE41() && (EltVT == MVT::f32 || EltVT == MVT::f64))))
18086       return SDValue();
18087 
18088     MVT IdxSVT = MVT::getIntegerVT(EltSizeInBits);
18089     MVT IdxVT = MVT::getVectorVT(IdxSVT, NumElts);
18090     if (!isTypeLegal(IdxSVT) || !isTypeLegal(IdxVT))
18091       return SDValue();
18092 
18093     SDValue IdxExt = DAG.getZExtOrTrunc(N2, dl, IdxSVT);
18094     SDValue IdxSplat = DAG.getSplatBuildVector(IdxVT, dl, IdxExt);
18095     SDValue EltSplat = DAG.getSplatBuildVector(VT, dl, N1);
18096 
18097     SmallVector<SDValue, 16> RawIndices;
18098     for (unsigned I = 0; I != NumElts; ++I)
18099       RawIndices.push_back(DAG.getConstant(I, dl, IdxSVT));
18100     SDValue Indices = DAG.getBuildVector(IdxVT, dl, RawIndices);
18101 
18102     // inselt N0, N1, N2 --> select (SplatN2 == {0,1,2...}) ? SplatN1 : N0.
18103     return DAG.getSelectCC(dl, IdxSplat, Indices, EltSplat, N0,
18104                            ISD::CondCode::SETEQ);
18105   }
18106 
18107   if (N2C->getAPIntValue().uge(NumElts))
18108     return SDValue();
18109   uint64_t IdxVal = N2C->getZExtValue();
18110 
18111   bool IsZeroElt = X86::isZeroNode(N1);
18112   bool IsAllOnesElt = VT.isInteger() && llvm::isAllOnesConstant(N1);
18113 
18114   if (IsZeroElt || IsAllOnesElt) {
18115     // Lower insertion of v16i8/v32i8/v64i16 -1 elts as an 'OR' blend.
18116     // We don't deal with i8 0 since it appears to be handled elsewhere.
18117     if (IsAllOnesElt &&
18118         ((VT == MVT::v16i8 && !Subtarget.hasSSE41()) ||
18119          ((VT == MVT::v32i8 || VT == MVT::v16i16) && !Subtarget.hasInt256()))) {
18120       SDValue ZeroCst = DAG.getConstant(0, dl, VT.getScalarType());
18121       SDValue OnesCst = DAG.getAllOnesConstant(dl, VT.getScalarType());
18122       SmallVector<SDValue, 8> CstVectorElts(NumElts, ZeroCst);
18123       CstVectorElts[IdxVal] = OnesCst;
18124       SDValue CstVector = DAG.getBuildVector(VT, dl, CstVectorElts);
18125       return DAG.getNode(ISD::OR, dl, VT, N0, CstVector);
18126     }
18127     // See if we can do this more efficiently with a blend shuffle with a
18128     // rematerializable vector.
18129     if (Subtarget.hasSSE41() &&
18130         (EltSizeInBits >= 16 || (IsZeroElt && !VT.is128BitVector()))) {
18131       SmallVector<int, 8> BlendMask;
18132       for (unsigned i = 0; i != NumElts; ++i)
18133         BlendMask.push_back(i == IdxVal ? i + NumElts : i);
18134       SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)
18135                                     : getOnesVector(VT, DAG, dl);
18136       return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask);
18137     }
18138   }
18139 
18140   // If the vector is wider than 128 bits, extract the 128-bit subvector, insert
18141   // into that, and then insert the subvector back into the result.
18142   if (VT.is256BitVector() || VT.is512BitVector()) {
18143     // With a 256-bit vector, we can insert into the zero element efficiently
18144     // using a blend if we have AVX or AVX2 and the right data type.
18145     if (VT.is256BitVector() && IdxVal == 0) {
18146       // TODO: It is worthwhile to cast integer to floating point and back
18147       // and incur a domain crossing penalty if that's what we'll end up
18148       // doing anyway after extracting to a 128-bit vector.
18149       if ((Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||
18150           (Subtarget.hasAVX2() && (EltVT == MVT::i32 || EltVT == MVT::i64))) {
18151         SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18152         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec,
18153                            DAG.getTargetConstant(1, dl, MVT::i8));
18154       }
18155     }
18156 
18157     unsigned NumEltsIn128 = 128 / EltSizeInBits;
18158     assert(isPowerOf2_32(NumEltsIn128) &&
18159            "Vectors will always have power-of-two number of elements.");
18160 
18161     // If we are not inserting into the low 128-bit vector chunk,
18162     // then prefer the broadcast+blend sequence.
18163     // FIXME: relax the profitability check iff all N1 uses are insertions.
18164     if (IdxVal >= NumEltsIn128 &&
18165         ((Subtarget.hasAVX2() && EltSizeInBits != 8) ||
18166          (Subtarget.hasAVX() && (EltSizeInBits >= 32) &&
18167           X86::mayFoldLoad(N1, Subtarget)))) {
18168       SDValue N1SplatVec = DAG.getSplatBuildVector(VT, dl, N1);
18169       SmallVector<int, 8> BlendMask;
18170       for (unsigned i = 0; i != NumElts; ++i)
18171         BlendMask.push_back(i == IdxVal ? i + NumElts : i);
18172       return DAG.getVectorShuffle(VT, dl, N0, N1SplatVec, BlendMask);
18173     }
18174 
18175     // Get the desired 128-bit vector chunk.
18176     SDValue V = extract128BitVector(N0, IdxVal, DAG, dl);
18177 
18178     // Insert the element into the desired chunk.
18179     // Since NumEltsIn128 is a power of 2 we can use mask instead of modulo.
18180     unsigned IdxIn128 = IdxVal & (NumEltsIn128 - 1);
18181 
18182     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1,
18183                     DAG.getIntPtrConstant(IdxIn128, dl));
18184 
18185     // Insert the changed part back into the bigger vector
18186     return insert128BitVector(N0, V, IdxVal, DAG, dl);
18187   }
18188   assert(VT.is128BitVector() && "Only 128-bit vector types should be left!");
18189 
18190   // This will be just movw/movd/movq/movsh/movss/movsd.
18191   if (IdxVal == 0 && ISD::isBuildVectorAllZeros(N0.getNode())) {
18192     if (EltVT == MVT::i32 || EltVT == MVT::f32 || EltVT == MVT::f64 ||
18193         EltVT == MVT::f16 || EltVT == MVT::i64) {
18194       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18195       return getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
18196     }
18197 
18198     // We can't directly insert an i8 or i16 into a vector, so zero extend
18199     // it to i32 first.
18200     if (EltVT == MVT::i16 || EltVT == MVT::i8) {
18201       N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, N1);
18202       MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
18203       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, N1);
18204       N1 = getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
18205       return DAG.getBitcast(VT, N1);
18206     }
18207   }
18208 
18209   // Transform it so it match pinsr{b,w} which expects a GR32 as its second
18210   // argument. SSE41 required for pinsrb.
18211   if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) {
18212     unsigned Opc;
18213     if (VT == MVT::v8i16) {
18214       assert(Subtarget.hasSSE2() && "SSE2 required for PINSRW");
18215       Opc = X86ISD::PINSRW;
18216     } else {
18217       assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector");
18218       assert(Subtarget.hasSSE41() && "SSE41 required for PINSRB");
18219       Opc = X86ISD::PINSRB;
18220     }
18221 
18222     assert(N1.getValueType() != MVT::i32 && "Unexpected VT");
18223     N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
18224     N2 = DAG.getTargetConstant(IdxVal, dl, MVT::i8);
18225     return DAG.getNode(Opc, dl, VT, N0, N1, N2);
18226   }
18227 
18228   if (Subtarget.hasSSE41()) {
18229     if (EltVT == MVT::f32) {
18230       // Bits [7:6] of the constant are the source select. This will always be
18231       //   zero here. The DAG Combiner may combine an extract_elt index into
18232       //   these bits. For example (insert (extract, 3), 2) could be matched by
18233       //   putting the '3' into bits [7:6] of X86ISD::INSERTPS.
18234       // Bits [5:4] of the constant are the destination select. This is the
18235       //   value of the incoming immediate.
18236       // Bits [3:0] of the constant are the zero mask. The DAG Combiner may
18237       //   combine either bitwise AND or insert of float 0.0 to set these bits.
18238 
18239       bool MinSize = DAG.getMachineFunction().getFunction().hasMinSize();
18240       if (IdxVal == 0 && (!MinSize || !X86::mayFoldLoad(N1, Subtarget))) {
18241         // If this is an insertion of 32-bits into the low 32-bits of
18242         // a vector, we prefer to generate a blend with immediate rather
18243         // than an insertps. Blends are simpler operations in hardware and so
18244         // will always have equal or better performance than insertps.
18245         // But if optimizing for size and there's a load folding opportunity,
18246         // generate insertps because blendps does not have a 32-bit memory
18247         // operand form.
18248         N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
18249         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1,
18250                            DAG.getTargetConstant(1, dl, MVT::i8));
18251       }
18252       // Create this as a scalar to vector..
18253       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
18254       return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1,
18255                          DAG.getTargetConstant(IdxVal << 4, dl, MVT::i8));
18256     }
18257 
18258     // PINSR* works with constant index.
18259     if (EltVT == MVT::i32 || EltVT == MVT::i64)
18260       return Op;
18261   }
18262 
18263   return SDValue();
18264 }
18265 
18266 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget,
18267                                      SelectionDAG &DAG) {
18268   SDLoc dl(Op);
18269   MVT OpVT = Op.getSimpleValueType();
18270 
18271   // It's always cheaper to replace a xor+movd with xorps and simplifies further
18272   // combines.
18273   if (X86::isZeroNode(Op.getOperand(0)))
18274     return getZeroVector(OpVT, Subtarget, DAG, dl);
18275 
18276   // If this is a 256-bit vector result, first insert into a 128-bit
18277   // vector and then insert into the 256-bit vector.
18278   if (!OpVT.is128BitVector()) {
18279     // Insert into a 128-bit vector.
18280     unsigned SizeFactor = OpVT.getSizeInBits() / 128;
18281     MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(),
18282                                  OpVT.getVectorNumElements() / SizeFactor);
18283 
18284     Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
18285 
18286     // Insert the 128-bit vector.
18287     return insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
18288   }
18289   assert(OpVT.is128BitVector() && OpVT.isInteger() && OpVT != MVT::v2i64 &&
18290          "Expected an SSE type!");
18291 
18292   // Pass through a v4i32 or V8i16 SCALAR_TO_VECTOR as that's what we use in
18293   // tblgen.
18294   if (OpVT == MVT::v4i32 || (OpVT == MVT::v8i16 && Subtarget.hasFP16()))
18295     return Op;
18296 
18297   SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
18298   return DAG.getBitcast(
18299       OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt));
18300 }
18301 
18302 // Lower a node with an INSERT_SUBVECTOR opcode.  This may result in a
18303 // simple superregister reference or explicit instructions to insert
18304 // the upper bits of a vector.
18305 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18306                                      SelectionDAG &DAG) {
18307   assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1);
18308 
18309   return insert1BitVector(Op, DAG, Subtarget);
18310 }
18311 
18312 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18313                                       SelectionDAG &DAG) {
18314   assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1 &&
18315          "Only vXi1 extract_subvectors need custom lowering");
18316 
18317   SDLoc dl(Op);
18318   SDValue Vec = Op.getOperand(0);
18319   uint64_t IdxVal = Op.getConstantOperandVal(1);
18320 
18321   if (IdxVal == 0) // the operation is legal
18322     return Op;
18323 
18324   // Extend to natively supported kshift.
18325   Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
18326 
18327   // Shift to the LSB.
18328   Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec,
18329                     DAG.getTargetConstant(IdxVal, dl, MVT::i8));
18330 
18331   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, Op.getValueType(), Vec,
18332                      DAG.getIntPtrConstant(0, dl));
18333 }
18334 
18335 // Returns the appropriate wrapper opcode for a global reference.
18336 unsigned X86TargetLowering::getGlobalWrapperKind(
18337     const GlobalValue *GV, const unsigned char OpFlags) const {
18338   // References to absolute symbols are never PC-relative.
18339   if (GV && GV->isAbsoluteSymbolRef())
18340     return X86ISD::Wrapper;
18341 
18342   // The following OpFlags under RIP-rel PIC use RIP.
18343   if (Subtarget.isPICStyleRIPRel() &&
18344       (OpFlags == X86II::MO_NO_FLAG || OpFlags == X86II::MO_COFFSTUB ||
18345        OpFlags == X86II::MO_DLLIMPORT))
18346     return X86ISD::WrapperRIP;
18347 
18348   // GOTPCREL references must always use RIP.
18349   if (OpFlags == X86II::MO_GOTPCREL || OpFlags == X86II::MO_GOTPCREL_NORELAX)
18350     return X86ISD::WrapperRIP;
18351 
18352   return X86ISD::Wrapper;
18353 }
18354 
18355 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
18356 // their target counterpart wrapped in the X86ISD::Wrapper node. Suppose N is
18357 // one of the above mentioned nodes. It has to be wrapped because otherwise
18358 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
18359 // be used to form addressing mode. These wrapped nodes will be selected
18360 // into MOV32ri.
18361 SDValue
18362 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
18363   ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
18364 
18365   // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18366   // global base reg.
18367   unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
18368 
18369   auto PtrVT = getPointerTy(DAG.getDataLayout());
18370   SDValue Result = DAG.getTargetConstantPool(
18371       CP->getConstVal(), PtrVT, CP->getAlign(), CP->getOffset(), OpFlag);
18372   SDLoc DL(CP);
18373   Result =
18374       DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result);
18375   // With PIC, the address is actually $g + Offset.
18376   if (OpFlag) {
18377     Result =
18378         DAG.getNode(ISD::ADD, DL, PtrVT,
18379                     DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18380   }
18381 
18382   return Result;
18383 }
18384 
18385 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
18386   JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
18387 
18388   // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18389   // global base reg.
18390   unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
18391 
18392   auto PtrVT = getPointerTy(DAG.getDataLayout());
18393   SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
18394   SDLoc DL(JT);
18395   Result =
18396       DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result);
18397 
18398   // With PIC, the address is actually $g + Offset.
18399   if (OpFlag)
18400     Result =
18401         DAG.getNode(ISD::ADD, DL, PtrVT,
18402                     DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18403 
18404   return Result;
18405 }
18406 
18407 SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
18408                                                SelectionDAG &DAG) const {
18409   return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18410 }
18411 
18412 SDValue
18413 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
18414   // Create the TargetBlockAddressAddress node.
18415   unsigned char OpFlags =
18416     Subtarget.classifyBlockAddressReference();
18417   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
18418   int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
18419   SDLoc dl(Op);
18420   auto PtrVT = getPointerTy(DAG.getDataLayout());
18421   SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
18422   Result =
18423       DAG.getNode(getGlobalWrapperKind(nullptr, OpFlags), dl, PtrVT, Result);
18424 
18425   // With PIC, the address is actually $g + Offset.
18426   if (isGlobalRelativeToPICBase(OpFlags)) {
18427     Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18428                          DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18429   }
18430 
18431   return Result;
18432 }
18433 
18434 /// Creates target global address or external symbol nodes for calls or
18435 /// other uses.
18436 SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
18437                                                  bool ForCall) const {
18438   // Unpack the global address or external symbol.
18439   const SDLoc &dl = SDLoc(Op);
18440   const GlobalValue *GV = nullptr;
18441   int64_t Offset = 0;
18442   const char *ExternalSym = nullptr;
18443   if (const auto *G = dyn_cast<GlobalAddressSDNode>(Op)) {
18444     GV = G->getGlobal();
18445     Offset = G->getOffset();
18446   } else {
18447     const auto *ES = cast<ExternalSymbolSDNode>(Op);
18448     ExternalSym = ES->getSymbol();
18449   }
18450 
18451   // Calculate some flags for address lowering.
18452   const Module &Mod = *DAG.getMachineFunction().getFunction().getParent();
18453   unsigned char OpFlags;
18454   if (ForCall)
18455     OpFlags = Subtarget.classifyGlobalFunctionReference(GV, Mod);
18456   else
18457     OpFlags = Subtarget.classifyGlobalReference(GV, Mod);
18458   bool HasPICReg = isGlobalRelativeToPICBase(OpFlags);
18459   bool NeedsLoad = isGlobalStubReference(OpFlags);
18460 
18461   CodeModel::Model M = DAG.getTarget().getCodeModel();
18462   auto PtrVT = getPointerTy(DAG.getDataLayout());
18463   SDValue Result;
18464 
18465   if (GV) {
18466     // Create a target global address if this is a global. If possible, fold the
18467     // offset into the global address reference. Otherwise, ADD it on later.
18468     // Suppress the folding if Offset is negative: movl foo-1, %eax is not
18469     // allowed because if the address of foo is 0, the ELF R_X86_64_32
18470     // relocation will compute to a negative value, which is invalid.
18471     int64_t GlobalOffset = 0;
18472     if (OpFlags == X86II::MO_NO_FLAG && Offset >= 0 &&
18473         X86::isOffsetSuitableForCodeModel(Offset, M, true)) {
18474       std::swap(GlobalOffset, Offset);
18475     }
18476     Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, GlobalOffset, OpFlags);
18477   } else {
18478     // If this is not a global address, this must be an external symbol.
18479     Result = DAG.getTargetExternalSymbol(ExternalSym, PtrVT, OpFlags);
18480   }
18481 
18482   // If this is a direct call, avoid the wrapper if we don't need to do any
18483   // loads or adds. This allows SDAG ISel to match direct calls.
18484   if (ForCall && !NeedsLoad && !HasPICReg && Offset == 0)
18485     return Result;
18486 
18487   Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result);
18488 
18489   // With PIC, the address is actually $g + Offset.
18490   if (HasPICReg) {
18491     Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18492                          DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18493   }
18494 
18495   // For globals that require a load from a stub to get the address, emit the
18496   // load.
18497   if (NeedsLoad)
18498     Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
18499                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
18500 
18501   // If there was a non-zero offset that we didn't fold, create an explicit
18502   // addition for it.
18503   if (Offset != 0)
18504     Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result,
18505                          DAG.getConstant(Offset, dl, PtrVT));
18506 
18507   return Result;
18508 }
18509 
18510 SDValue
18511 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
18512   return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18513 }
18514 
18515 static SDValue
18516 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
18517            SDValue *InGlue, const EVT PtrVT, unsigned ReturnReg,
18518            unsigned char OperandFlags, bool LocalDynamic = false) {
18519   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
18520   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18521   SDLoc dl(GA);
18522   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18523                                            GA->getValueType(0),
18524                                            GA->getOffset(),
18525                                            OperandFlags);
18526 
18527   X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
18528                                            : X86ISD::TLSADDR;
18529 
18530   if (InGlue) {
18531     SDValue Ops[] = { Chain,  TGA, *InGlue };
18532     Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
18533   } else {
18534     SDValue Ops[]  = { Chain, TGA };
18535     Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
18536   }
18537 
18538   // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
18539   MFI.setAdjustsStack(true);
18540   MFI.setHasCalls(true);
18541 
18542   SDValue Glue = Chain.getValue(1);
18543   return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
18544 }
18545 
18546 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
18547 static SDValue
18548 LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18549                                 const EVT PtrVT) {
18550   SDValue InGlue;
18551   SDLoc dl(GA);  // ? function entry point might be better
18552   SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18553                                    DAG.getNode(X86ISD::GlobalBaseReg,
18554                                                SDLoc(), PtrVT), InGlue);
18555   InGlue = Chain.getValue(1);
18556 
18557   return GetTLSADDR(DAG, Chain, GA, &InGlue, PtrVT, X86::EAX, X86II::MO_TLSGD);
18558 }
18559 
18560 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit LP64
18561 static SDValue
18562 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18563                                 const EVT PtrVT) {
18564   return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
18565                     X86::RAX, X86II::MO_TLSGD);
18566 }
18567 
18568 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit ILP32
18569 static SDValue
18570 LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18571                                  const EVT PtrVT) {
18572   return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
18573                     X86::EAX, X86II::MO_TLSGD);
18574 }
18575 
18576 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
18577                                            SelectionDAG &DAG, const EVT PtrVT,
18578                                            bool Is64Bit, bool Is64BitLP64) {
18579   SDLoc dl(GA);
18580 
18581   // Get the start address of the TLS block for this module.
18582   X86MachineFunctionInfo *MFI = DAG.getMachineFunction()
18583       .getInfo<X86MachineFunctionInfo>();
18584   MFI->incNumLocalDynamicTLSAccesses();
18585 
18586   SDValue Base;
18587   if (Is64Bit) {
18588     unsigned ReturnReg = Is64BitLP64 ? X86::RAX : X86::EAX;
18589     Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, ReturnReg,
18590                       X86II::MO_TLSLD, /*LocalDynamic=*/true);
18591   } else {
18592     SDValue InGlue;
18593     SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18594         DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InGlue);
18595     InGlue = Chain.getValue(1);
18596     Base = GetTLSADDR(DAG, Chain, GA, &InGlue, PtrVT, X86::EAX,
18597                       X86II::MO_TLSLDM, /*LocalDynamic=*/true);
18598   }
18599 
18600   // Note: the CleanupLocalDynamicTLSPass will remove redundant computations
18601   // of Base.
18602 
18603   // Build x@dtpoff.
18604   unsigned char OperandFlags = X86II::MO_DTPOFF;
18605   unsigned WrapperKind = X86ISD::Wrapper;
18606   SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18607                                            GA->getValueType(0),
18608                                            GA->getOffset(), OperandFlags);
18609   SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18610 
18611   // Add x@dtpoff with the base.
18612   return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base);
18613 }
18614 
18615 // Lower ISD::GlobalTLSAddress using the "initial exec" or "local exec" model.
18616 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18617                                    const EVT PtrVT, TLSModel::Model model,
18618                                    bool is64Bit, bool isPIC) {
18619   SDLoc dl(GA);
18620 
18621   // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
18622   Value *Ptr = Constant::getNullValue(
18623       PointerType::get(*DAG.getContext(), is64Bit ? 257 : 256));
18624 
18625   SDValue ThreadPointer =
18626       DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl),
18627                   MachinePointerInfo(Ptr));
18628 
18629   unsigned char OperandFlags = 0;
18630   // Most TLS accesses are not RIP relative, even on x86-64.  One exception is
18631   // initialexec.
18632   unsigned WrapperKind = X86ISD::Wrapper;
18633   if (model == TLSModel::LocalExec) {
18634     OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
18635   } else if (model == TLSModel::InitialExec) {
18636     if (is64Bit) {
18637       OperandFlags = X86II::MO_GOTTPOFF;
18638       WrapperKind = X86ISD::WrapperRIP;
18639     } else {
18640       OperandFlags = isPIC ? X86II::MO_GOTNTPOFF : X86II::MO_INDNTPOFF;
18641     }
18642   } else {
18643     llvm_unreachable("Unexpected model");
18644   }
18645 
18646   // emit "addl x@ntpoff,%eax" (local exec)
18647   // or "addl x@indntpoff,%eax" (initial exec)
18648   // or "addl x@gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
18649   SDValue TGA =
18650       DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
18651                                  GA->getOffset(), OperandFlags);
18652   SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18653 
18654   if (model == TLSModel::InitialExec) {
18655     if (isPIC && !is64Bit) {
18656       Offset = DAG.getNode(ISD::ADD, dl, PtrVT,
18657                            DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
18658                            Offset);
18659     }
18660 
18661     Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
18662                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
18663   }
18664 
18665   // The address of the thread local variable is the add of the thread
18666   // pointer with the offset of the variable.
18667   return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
18668 }
18669 
18670 SDValue
18671 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
18672 
18673   GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
18674 
18675   if (DAG.getTarget().useEmulatedTLS())
18676     return LowerToTLSEmulatedModel(GA, DAG);
18677 
18678   const GlobalValue *GV = GA->getGlobal();
18679   auto PtrVT = getPointerTy(DAG.getDataLayout());
18680   bool PositionIndependent = isPositionIndependent();
18681 
18682   if (Subtarget.isTargetELF()) {
18683     TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
18684     switch (model) {
18685       case TLSModel::GeneralDynamic:
18686         if (Subtarget.is64Bit()) {
18687           if (Subtarget.isTarget64BitLP64())
18688             return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
18689           return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
18690         }
18691         return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
18692       case TLSModel::LocalDynamic:
18693         return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
18694                                            Subtarget.isTarget64BitLP64());
18695       case TLSModel::InitialExec:
18696       case TLSModel::LocalExec:
18697         return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
18698                                    PositionIndependent);
18699     }
18700     llvm_unreachable("Unknown TLS model.");
18701   }
18702 
18703   if (Subtarget.isTargetDarwin()) {
18704     // Darwin only has one model of TLS.  Lower to that.
18705     unsigned char OpFlag = 0;
18706     unsigned WrapperKind = Subtarget.isPICStyleRIPRel() ?
18707                            X86ISD::WrapperRIP : X86ISD::Wrapper;
18708 
18709     // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18710     // global base reg.
18711     bool PIC32 = PositionIndependent && !Subtarget.is64Bit();
18712     if (PIC32)
18713       OpFlag = X86II::MO_TLVP_PIC_BASE;
18714     else
18715       OpFlag = X86II::MO_TLVP;
18716     SDLoc DL(Op);
18717     SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
18718                                                 GA->getValueType(0),
18719                                                 GA->getOffset(), OpFlag);
18720     SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
18721 
18722     // With PIC32, the address is actually $g + Offset.
18723     if (PIC32)
18724       Offset = DAG.getNode(ISD::ADD, DL, PtrVT,
18725                            DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
18726                            Offset);
18727 
18728     // Lowering the machine isd will make sure everything is in the right
18729     // location.
18730     SDValue Chain = DAG.getEntryNode();
18731     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18732     Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
18733     SDValue Args[] = { Chain, Offset };
18734     Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
18735     Chain = DAG.getCALLSEQ_END(Chain, 0, 0, Chain.getValue(1), DL);
18736 
18737     // TLSCALL will be codegen'ed as call. Inform MFI that function has calls.
18738     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
18739     MFI.setAdjustsStack(true);
18740 
18741     // And our return value (tls address) is in the standard call return value
18742     // location.
18743     unsigned Reg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
18744     return DAG.getCopyFromReg(Chain, DL, Reg, PtrVT, Chain.getValue(1));
18745   }
18746 
18747   if (Subtarget.isOSWindows()) {
18748     // Just use the implicit TLS architecture
18749     // Need to generate something similar to:
18750     //   mov     rdx, qword [gs:abs 58H]; Load pointer to ThreadLocalStorage
18751     //                                  ; from TEB
18752     //   mov     ecx, dword [rel _tls_index]: Load index (from C runtime)
18753     //   mov     rcx, qword [rdx+rcx*8]
18754     //   mov     eax, .tls$:tlsvar
18755     //   [rax+rcx] contains the address
18756     // Windows 64bit: gs:0x58
18757     // Windows 32bit: fs:__tls_array
18758 
18759     SDLoc dl(GA);
18760     SDValue Chain = DAG.getEntryNode();
18761 
18762     // Get the Thread Pointer, which is %fs:__tls_array (32-bit) or
18763     // %gs:0x58 (64-bit). On MinGW, __tls_array is not available, so directly
18764     // use its literal value of 0x2C.
18765     Value *Ptr = Constant::getNullValue(
18766         Subtarget.is64Bit() ? PointerType::get(*DAG.getContext(), 256)
18767                             : PointerType::get(*DAG.getContext(), 257));
18768 
18769     SDValue TlsArray = Subtarget.is64Bit()
18770                            ? DAG.getIntPtrConstant(0x58, dl)
18771                            : (Subtarget.isTargetWindowsGNU()
18772                                   ? DAG.getIntPtrConstant(0x2C, dl)
18773                                   : DAG.getExternalSymbol("_tls_array", PtrVT));
18774 
18775     SDValue ThreadPointer =
18776         DAG.getLoad(PtrVT, dl, Chain, TlsArray, MachinePointerInfo(Ptr));
18777 
18778     SDValue res;
18779     if (GV->getThreadLocalMode() == GlobalVariable::LocalExecTLSModel) {
18780       res = ThreadPointer;
18781     } else {
18782       // Load the _tls_index variable
18783       SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
18784       if (Subtarget.is64Bit())
18785         IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, IDX,
18786                              MachinePointerInfo(), MVT::i32);
18787       else
18788         IDX = DAG.getLoad(PtrVT, dl, Chain, IDX, MachinePointerInfo());
18789 
18790       const DataLayout &DL = DAG.getDataLayout();
18791       SDValue Scale =
18792           DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8);
18793       IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale);
18794 
18795       res = DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, IDX);
18796     }
18797 
18798     res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo());
18799 
18800     // Get the offset of start of .tls section
18801     SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18802                                              GA->getValueType(0),
18803                                              GA->getOffset(), X86II::MO_SECREL);
18804     SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
18805 
18806     // The address of the thread local variable is the add of the thread
18807     // pointer with the offset of the variable.
18808     return DAG.getNode(ISD::ADD, dl, PtrVT, res, Offset);
18809   }
18810 
18811   llvm_unreachable("TLS not implemented for this target.");
18812 }
18813 
18814 /// Lower SRA_PARTS and friends, which return two i32 values
18815 /// and take a 2 x i32 value to shift plus a shift amount.
18816 /// TODO: Can this be moved to general expansion code?
18817 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
18818   SDValue Lo, Hi;
18819   DAG.getTargetLoweringInfo().expandShiftParts(Op.getNode(), Lo, Hi, DAG);
18820   return DAG.getMergeValues({Lo, Hi}, SDLoc(Op));
18821 }
18822 
18823 // Try to use a packed vector operation to handle i64 on 32-bit targets when
18824 // AVX512DQ is enabled.
18825 static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, SelectionDAG &DAG,
18826                                         const X86Subtarget &Subtarget) {
18827   assert((Op.getOpcode() == ISD::SINT_TO_FP ||
18828           Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
18829           Op.getOpcode() == ISD::STRICT_UINT_TO_FP ||
18830           Op.getOpcode() == ISD::UINT_TO_FP) &&
18831          "Unexpected opcode!");
18832   bool IsStrict = Op->isStrictFPOpcode();
18833   unsigned OpNo = IsStrict ? 1 : 0;
18834   SDValue Src = Op.getOperand(OpNo);
18835   MVT SrcVT = Src.getSimpleValueType();
18836   MVT VT = Op.getSimpleValueType();
18837 
18838    if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||
18839        (VT != MVT::f32 && VT != MVT::f64))
18840     return SDValue();
18841 
18842   // Pack the i64 into a vector, do the operation and extract.
18843 
18844   // Using 256-bit to ensure result is 128-bits for f32 case.
18845   unsigned NumElts = Subtarget.hasVLX() ? 4 : 8;
18846   MVT VecInVT = MVT::getVectorVT(MVT::i64, NumElts);
18847   MVT VecVT = MVT::getVectorVT(VT, NumElts);
18848 
18849   SDLoc dl(Op);
18850   SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src);
18851   if (IsStrict) {
18852     SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other},
18853                                  {Op.getOperand(0), InVec});
18854     SDValue Chain = CvtVec.getValue(1);
18855     SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18856                                 DAG.getIntPtrConstant(0, dl));
18857     return DAG.getMergeValues({Value, Chain}, dl);
18858   }
18859 
18860   SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec);
18861 
18862   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18863                      DAG.getIntPtrConstant(0, dl));
18864 }
18865 
18866 // Try to use a packed vector operation to handle i64 on 32-bit targets.
18867 static SDValue LowerI64IntToFP16(SDValue Op, SelectionDAG &DAG,
18868                                  const X86Subtarget &Subtarget) {
18869   assert((Op.getOpcode() == ISD::SINT_TO_FP ||
18870           Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
18871           Op.getOpcode() == ISD::STRICT_UINT_TO_FP ||
18872           Op.getOpcode() == ISD::UINT_TO_FP) &&
18873          "Unexpected opcode!");
18874   bool IsStrict = Op->isStrictFPOpcode();
18875   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
18876   MVT SrcVT = Src.getSimpleValueType();
18877   MVT VT = Op.getSimpleValueType();
18878 
18879   if (SrcVT != MVT::i64 || Subtarget.is64Bit() || VT != MVT::f16)
18880     return SDValue();
18881 
18882   // Pack the i64 into a vector, do the operation and extract.
18883 
18884   assert(Subtarget.hasFP16() && "Expected FP16");
18885 
18886   SDLoc dl(Op);
18887   SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
18888   if (IsStrict) {
18889     SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {MVT::v2f16, MVT::Other},
18890                                  {Op.getOperand(0), InVec});
18891     SDValue Chain = CvtVec.getValue(1);
18892     SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18893                                 DAG.getIntPtrConstant(0, dl));
18894     return DAG.getMergeValues({Value, Chain}, dl);
18895   }
18896 
18897   SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, MVT::v2f16, InVec);
18898 
18899   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18900                      DAG.getIntPtrConstant(0, dl));
18901 }
18902 
18903 static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT,
18904                           const X86Subtarget &Subtarget) {
18905   switch (Opcode) {
18906     case ISD::SINT_TO_FP:
18907       // TODO: Handle wider types with AVX/AVX512.
18908       if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
18909         return false;
18910       // CVTDQ2PS or (V)CVTDQ2PD
18911       return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
18912 
18913     case ISD::UINT_TO_FP:
18914       // TODO: Handle wider types and i64 elements.
18915       if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
18916         return false;
18917       // VCVTUDQ2PS or VCVTUDQ2PD
18918       return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
18919 
18920     default:
18921       return false;
18922   }
18923 }
18924 
18925 /// Given a scalar cast operation that is extracted from a vector, try to
18926 /// vectorize the cast op followed by extraction. This will avoid an expensive
18927 /// round-trip between XMM and GPR.
18928 static SDValue vectorizeExtractedCast(SDValue Cast, SelectionDAG &DAG,
18929                                       const X86Subtarget &Subtarget) {
18930   // TODO: This could be enhanced to handle smaller integer types by peeking
18931   // through an extend.
18932   SDValue Extract = Cast.getOperand(0);
18933   MVT DestVT = Cast.getSimpleValueType();
18934   if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18935       !isa<ConstantSDNode>(Extract.getOperand(1)))
18936     return SDValue();
18937 
18938   // See if we have a 128-bit vector cast op for this type of cast.
18939   SDValue VecOp = Extract.getOperand(0);
18940   MVT FromVT = VecOp.getSimpleValueType();
18941   unsigned NumEltsInXMM = 128 / FromVT.getScalarSizeInBits();
18942   MVT Vec128VT = MVT::getVectorVT(FromVT.getScalarType(), NumEltsInXMM);
18943   MVT ToVT = MVT::getVectorVT(DestVT, NumEltsInXMM);
18944   if (!useVectorCast(Cast.getOpcode(), Vec128VT, ToVT, Subtarget))
18945     return SDValue();
18946 
18947   // If we are extracting from a non-zero element, first shuffle the source
18948   // vector to allow extracting from element zero.
18949   SDLoc DL(Cast);
18950   if (!isNullConstant(Extract.getOperand(1))) {
18951     SmallVector<int, 16> Mask(FromVT.getVectorNumElements(), -1);
18952     Mask[0] = Extract.getConstantOperandVal(1);
18953     VecOp = DAG.getVectorShuffle(FromVT, DL, VecOp, DAG.getUNDEF(FromVT), Mask);
18954   }
18955   // If the source vector is wider than 128-bits, extract the low part. Do not
18956   // create an unnecessarily wide vector cast op.
18957   if (FromVT != Vec128VT)
18958     VecOp = extract128BitVector(VecOp, 0, DAG, DL);
18959 
18960   // cast (extelt V, 0) --> extelt (cast (extract_subv V)), 0
18961   // cast (extelt V, C) --> extelt (cast (extract_subv (shuffle V, [C...]))), 0
18962   SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
18963   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast,
18964                      DAG.getIntPtrConstant(0, DL));
18965 }
18966 
18967 /// Given a scalar cast to FP with a cast to integer operand (almost an ftrunc),
18968 /// try to vectorize the cast ops. This will avoid an expensive round-trip
18969 /// between XMM and GPR.
18970 static SDValue lowerFPToIntToFP(SDValue CastToFP, SelectionDAG &DAG,
18971                                 const X86Subtarget &Subtarget) {
18972   // TODO: Allow FP_TO_UINT.
18973   SDValue CastToInt = CastToFP.getOperand(0);
18974   MVT VT = CastToFP.getSimpleValueType();
18975   if (CastToInt.getOpcode() != ISD::FP_TO_SINT || VT.isVector())
18976     return SDValue();
18977 
18978   MVT IntVT = CastToInt.getSimpleValueType();
18979   SDValue X = CastToInt.getOperand(0);
18980   MVT SrcVT = X.getSimpleValueType();
18981   if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
18982     return SDValue();
18983 
18984   // See if we have 128-bit vector cast instructions for this type of cast.
18985   // We need cvttps2dq/cvttpd2dq and cvtdq2ps/cvtdq2pd.
18986   if (!Subtarget.hasSSE2() || (VT != MVT::f32 && VT != MVT::f64) ||
18987       IntVT != MVT::i32)
18988     return SDValue();
18989 
18990   unsigned SrcSize = SrcVT.getSizeInBits();
18991   unsigned IntSize = IntVT.getSizeInBits();
18992   unsigned VTSize = VT.getSizeInBits();
18993   MVT VecSrcVT = MVT::getVectorVT(SrcVT, 128 / SrcSize);
18994   MVT VecIntVT = MVT::getVectorVT(IntVT, 128 / IntSize);
18995   MVT VecVT = MVT::getVectorVT(VT, 128 / VTSize);
18996 
18997   // We need target-specific opcodes if this is v2f64 -> v4i32 -> v2f64.
18998   unsigned ToIntOpcode =
18999       SrcSize != IntSize ? X86ISD::CVTTP2SI : (unsigned)ISD::FP_TO_SINT;
19000   unsigned ToFPOpcode =
19001       IntSize != VTSize ? X86ISD::CVTSI2P : (unsigned)ISD::SINT_TO_FP;
19002 
19003   // sint_to_fp (fp_to_sint X) --> extelt (sint_to_fp (fp_to_sint (s2v X))), 0
19004   //
19005   // We are not defining the high elements (for example, zero them) because
19006   // that could nullify any performance advantage that we hoped to gain from
19007   // this vector op hack. We do not expect any adverse effects (like denorm
19008   // penalties) with cast ops.
19009   SDLoc DL(CastToFP);
19010   SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
19011   SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X);
19012   SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX);
19013   SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt);
19014   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx);
19015 }
19016 
19017 static SDValue lowerINT_TO_FP_vXi64(SDValue Op, SelectionDAG &DAG,
19018                                     const X86Subtarget &Subtarget) {
19019   SDLoc DL(Op);
19020   bool IsStrict = Op->isStrictFPOpcode();
19021   MVT VT = Op->getSimpleValueType(0);
19022   SDValue Src = Op->getOperand(IsStrict ? 1 : 0);
19023 
19024   if (Subtarget.hasDQI()) {
19025     assert(!Subtarget.hasVLX() && "Unexpected features");
19026 
19027     assert((Src.getSimpleValueType() == MVT::v2i64 ||
19028             Src.getSimpleValueType() == MVT::v4i64) &&
19029            "Unsupported custom type");
19030 
19031     // With AVX512DQ, but not VLX we need to widen to get a 512-bit result type.
19032     assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) &&
19033            "Unexpected VT!");
19034     MVT WideVT = VT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;
19035 
19036     // Need to concat with zero vector for strict fp to avoid spurious
19037     // exceptions.
19038     SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)
19039                            : DAG.getUNDEF(MVT::v8i64);
19040     Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src,
19041                       DAG.getIntPtrConstant(0, DL));
19042     SDValue Res, Chain;
19043     if (IsStrict) {
19044       Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other},
19045                         {Op->getOperand(0), Src});
19046       Chain = Res.getValue(1);
19047     } else {
19048       Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src);
19049     }
19050 
19051     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19052                       DAG.getIntPtrConstant(0, DL));
19053 
19054     if (IsStrict)
19055       return DAG.getMergeValues({Res, Chain}, DL);
19056     return Res;
19057   }
19058 
19059   bool IsSigned = Op->getOpcode() == ISD::SINT_TO_FP ||
19060                   Op->getOpcode() == ISD::STRICT_SINT_TO_FP;
19061   if (VT != MVT::v4f32 || IsSigned)
19062     return SDValue();
19063 
19064   SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
19065   SDValue One  = DAG.getConstant(1, DL, MVT::v4i64);
19066   SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
19067                              DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),
19068                              DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));
19069   SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);
19070   SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);
19071   SmallVector<SDValue, 4> SignCvts(4);
19072   SmallVector<SDValue, 4> Chains(4);
19073   for (int i = 0; i != 4; ++i) {
19074     SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,
19075                               DAG.getIntPtrConstant(i, DL));
19076     if (IsStrict) {
19077       SignCvts[i] =
19078           DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other},
19079                       {Op.getOperand(0), Elt});
19080       Chains[i] = SignCvts[i].getValue(1);
19081     } else {
19082       SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Elt);
19083     }
19084   }
19085   SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts);
19086 
19087   SDValue Slow, Chain;
19088   if (IsStrict) {
19089     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
19090     Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other},
19091                        {Chain, SignCvt, SignCvt});
19092     Chain = Slow.getValue(1);
19093   } else {
19094     Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt);
19095   }
19096 
19097   IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg);
19098   SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);
19099 
19100   if (IsStrict)
19101     return DAG.getMergeValues({Cvt, Chain}, DL);
19102 
19103   return Cvt;
19104 }
19105 
19106 static SDValue promoteXINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
19107   bool IsStrict = Op->isStrictFPOpcode();
19108   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
19109   SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19110   MVT VT = Op.getSimpleValueType();
19111   MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
19112   SDLoc dl(Op);
19113 
19114   SDValue Rnd = DAG.getIntPtrConstant(0, dl);
19115   if (IsStrict)
19116     return DAG.getNode(
19117         ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other},
19118         {Chain,
19119          DAG.getNode(Op.getOpcode(), dl, {NVT, MVT::Other}, {Chain, Src}),
19120          Rnd});
19121   return DAG.getNode(ISD::FP_ROUND, dl, VT,
19122                      DAG.getNode(Op.getOpcode(), dl, NVT, Src), Rnd);
19123 }
19124 
19125 static bool isLegalConversion(MVT VT, bool IsSigned,
19126                               const X86Subtarget &Subtarget) {
19127   if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned)
19128     return true;
19129   if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned)
19130     return true;
19131   if (Subtarget.hasVLX() && (VT == MVT::v4i32 || VT == MVT::v8i32))
19132     return true;
19133   if (Subtarget.useAVX512Regs()) {
19134     if (VT == MVT::v16i32)
19135       return true;
19136     if (VT == MVT::v8i64 && Subtarget.hasDQI())
19137       return true;
19138   }
19139   if (Subtarget.hasDQI() && Subtarget.hasVLX() &&
19140       (VT == MVT::v2i64 || VT == MVT::v4i64))
19141     return true;
19142   return false;
19143 }
19144 
19145 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
19146                                            SelectionDAG &DAG) const {
19147   bool IsStrict = Op->isStrictFPOpcode();
19148   unsigned OpNo = IsStrict ? 1 : 0;
19149   SDValue Src = Op.getOperand(OpNo);
19150   SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19151   MVT SrcVT = Src.getSimpleValueType();
19152   MVT VT = Op.getSimpleValueType();
19153   SDLoc dl(Op);
19154 
19155   if (isSoftF16(VT, Subtarget))
19156     return promoteXINT_TO_FP(Op, DAG);
19157   else if (isLegalConversion(SrcVT, true, Subtarget))
19158     return Op;
19159 
19160   if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)
19161     return LowerWin64_INT128_TO_FP(Op, DAG);
19162 
19163   if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
19164     return Extract;
19165 
19166   if (SDValue R = lowerFPToIntToFP(Op, DAG, Subtarget))
19167     return R;
19168 
19169   if (SrcVT.isVector()) {
19170     if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) {
19171       // Note: Since v2f64 is a legal type. We don't need to zero extend the
19172       // source for strict FP.
19173       if (IsStrict)
19174         return DAG.getNode(
19175             X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
19176             {Chain, DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
19177                                 DAG.getUNDEF(SrcVT))});
19178       return DAG.getNode(X86ISD::CVTSI2P, dl, VT,
19179                          DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
19180                                      DAG.getUNDEF(SrcVT)));
19181     }
19182     if (SrcVT == MVT::v2i64 || SrcVT == MVT::v4i64)
19183       return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget);
19184 
19185     return SDValue();
19186   }
19187 
19188   assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&
19189          "Unknown SINT_TO_FP to lower!");
19190 
19191   bool UseSSEReg = isScalarFPTypeInSSEReg(VT);
19192 
19193   // These are really Legal; return the operand so the caller accepts it as
19194   // Legal.
19195   if (SrcVT == MVT::i32 && UseSSEReg)
19196     return Op;
19197   if (SrcVT == MVT::i64 && UseSSEReg && Subtarget.is64Bit())
19198     return Op;
19199 
19200   if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
19201     return V;
19202   if (SDValue V = LowerI64IntToFP16(Op, DAG, Subtarget))
19203     return V;
19204 
19205   // SSE doesn't have an i16 conversion so we need to promote.
19206   if (SrcVT == MVT::i16 && (UseSSEReg || VT == MVT::f128)) {
19207     SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src);
19208     if (IsStrict)
19209       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
19210                          {Chain, Ext});
19211 
19212     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext);
19213   }
19214 
19215   if (VT == MVT::f128 || !Subtarget.hasX87())
19216     return SDValue();
19217 
19218   SDValue ValueToStore = Src;
19219   if (SrcVT == MVT::i64 && Subtarget.hasSSE2() && !Subtarget.is64Bit())
19220     // Bitcasting to f64 here allows us to do a single 64-bit store from
19221     // an SSE register, avoiding the store forwarding penalty that would come
19222     // with two 32-bit stores.
19223     ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
19224 
19225   unsigned Size = SrcVT.getStoreSize();
19226   Align Alignment(Size);
19227   MachineFunction &MF = DAG.getMachineFunction();
19228   auto PtrVT = getPointerTy(MF.getDataLayout());
19229   int SSFI = MF.getFrameInfo().CreateStackObject(Size, Alignment, false);
19230   MachinePointerInfo MPI =
19231       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
19232   SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19233   Chain = DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Alignment);
19234   std::pair<SDValue, SDValue> Tmp =
19235       BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG);
19236 
19237   if (IsStrict)
19238     return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
19239 
19240   return Tmp.first;
19241 }
19242 
19243 std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
19244     EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer,
19245     MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const {
19246   // Build the FILD
19247   SDVTList Tys;
19248   bool useSSE = isScalarFPTypeInSSEReg(DstVT);
19249   if (useSSE)
19250     Tys = DAG.getVTList(MVT::f80, MVT::Other);
19251   else
19252     Tys = DAG.getVTList(DstVT, MVT::Other);
19253 
19254   SDValue FILDOps[] = {Chain, Pointer};
19255   SDValue Result =
19256       DAG.getMemIntrinsicNode(X86ISD::FILD, DL, Tys, FILDOps, SrcVT, PtrInfo,
19257                               Alignment, MachineMemOperand::MOLoad);
19258   Chain = Result.getValue(1);
19259 
19260   if (useSSE) {
19261     MachineFunction &MF = DAG.getMachineFunction();
19262     unsigned SSFISize = DstVT.getStoreSize();
19263     int SSFI =
19264         MF.getFrameInfo().CreateStackObject(SSFISize, Align(SSFISize), false);
19265     auto PtrVT = getPointerTy(MF.getDataLayout());
19266     SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19267     Tys = DAG.getVTList(MVT::Other);
19268     SDValue FSTOps[] = {Chain, Result, StackSlot};
19269     MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
19270         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
19271         MachineMemOperand::MOStore, SSFISize, Align(SSFISize));
19272 
19273     Chain =
19274         DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, FSTOps, DstVT, StoreMMO);
19275     Result = DAG.getLoad(
19276         DstVT, DL, Chain, StackSlot,
19277         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI));
19278     Chain = Result.getValue(1);
19279   }
19280 
19281   return { Result, Chain };
19282 }
19283 
19284 /// Horizontal vector math instructions may be slower than normal math with
19285 /// shuffles. Limit horizontal op codegen based on size/speed trade-offs, uarch
19286 /// implementation, and likely shuffle complexity of the alternate sequence.
19287 static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG,
19288                                   const X86Subtarget &Subtarget) {
19289   bool IsOptimizingSize = DAG.shouldOptForSize();
19290   bool HasFastHOps = Subtarget.hasFastHorizontalOps();
19291   return !IsSingleSource || IsOptimizingSize || HasFastHOps;
19292 }
19293 
19294 /// 64-bit unsigned integer to double expansion.
19295 static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG,
19296                                    const X86Subtarget &Subtarget) {
19297   // We can't use this algorithm for strict fp. It produces -0.0 instead of +0.0
19298   // when converting 0 when rounding toward negative infinity. Caller will
19299   // fall back to Expand for when i64 or is legal or use FILD in 32-bit mode.
19300   assert(!Op->isStrictFPOpcode() && "Expected non-strict uint_to_fp!");
19301   // This algorithm is not obvious. Here it is what we're trying to output:
19302   /*
19303      movq       %rax,  %xmm0
19304      punpckldq  (c0),  %xmm0  // c0: (uint4){ 0x43300000U, 0x45300000U, 0U, 0U }
19305      subpd      (c1),  %xmm0  // c1: (double2){ 0x1.0p52, 0x1.0p52 * 0x1.0p32 }
19306      #ifdef __SSE3__
19307        haddpd   %xmm0, %xmm0
19308      #else
19309        pshufd   $0x4e, %xmm0, %xmm1
19310        addpd    %xmm1, %xmm0
19311      #endif
19312   */
19313 
19314   SDLoc dl(Op);
19315   LLVMContext *Context = DAG.getContext();
19316 
19317   // Build some magic constants.
19318   static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
19319   Constant *C0 = ConstantDataVector::get(*Context, CV0);
19320   auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
19321   SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16));
19322 
19323   SmallVector<Constant*,2> CV1;
19324   CV1.push_back(
19325     ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
19326                                       APInt(64, 0x4330000000000000ULL))));
19327   CV1.push_back(
19328     ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
19329                                       APInt(64, 0x4530000000000000ULL))));
19330   Constant *C1 = ConstantVector::get(CV1);
19331   SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16));
19332 
19333   // Load the 64-bit value into an XMM register.
19334   SDValue XR1 =
19335       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(0));
19336   SDValue CLod0 = DAG.getLoad(
19337       MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
19338       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16));
19339   SDValue Unpck1 =
19340       getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0);
19341 
19342   SDValue CLod1 = DAG.getLoad(
19343       MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
19344       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16));
19345   SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
19346   // TODO: Are there any fast-math-flags to propagate here?
19347   SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
19348   SDValue Result;
19349 
19350   if (Subtarget.hasSSE3() &&
19351       shouldUseHorizontalOp(true, DAG, Subtarget)) {
19352     Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
19353   } else {
19354     SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
19355     Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub);
19356   }
19357   Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
19358                        DAG.getIntPtrConstant(0, dl));
19359   return Result;
19360 }
19361 
19362 /// 32-bit unsigned integer to float expansion.
19363 static SDValue LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG,
19364                                    const X86Subtarget &Subtarget) {
19365   unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
19366   SDLoc dl(Op);
19367   // FP constant to bias correct the final result.
19368   SDValue Bias = DAG.getConstantFP(
19369       llvm::bit_cast<double>(0x4330000000000000ULL), dl, MVT::f64);
19370 
19371   // Load the 32-bit value into an XMM register.
19372   SDValue Load =
19373       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Op.getOperand(OpNo));
19374 
19375   // Zero out the upper parts of the register.
19376   Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG);
19377 
19378   // Or the load with the bias.
19379   SDValue Or = DAG.getNode(
19380       ISD::OR, dl, MVT::v2i64,
19381       DAG.getBitcast(MVT::v2i64, Load),
19382       DAG.getBitcast(MVT::v2i64,
19383                      DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));
19384   Or =
19385       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
19386                   DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl));
19387 
19388   if (Op.getNode()->isStrictFPOpcode()) {
19389     // Subtract the bias.
19390     // TODO: Are there any fast-math-flags to propagate here?
19391     SDValue Chain = Op.getOperand(0);
19392     SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
19393                               {Chain, Or, Bias});
19394 
19395     if (Op.getValueType() == Sub.getValueType())
19396       return Sub;
19397 
19398     // Handle final rounding.
19399     std::pair<SDValue, SDValue> ResultPair = DAG.getStrictFPExtendOrRound(
19400         Sub, Sub.getValue(1), dl, Op.getSimpleValueType());
19401 
19402     return DAG.getMergeValues({ResultPair.first, ResultPair.second}, dl);
19403   }
19404 
19405   // Subtract the bias.
19406   // TODO: Are there any fast-math-flags to propagate here?
19407   SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
19408 
19409   // Handle final rounding.
19410   return DAG.getFPExtendOrRound(Sub, dl, Op.getSimpleValueType());
19411 }
19412 
19413 static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, SelectionDAG &DAG,
19414                                      const X86Subtarget &Subtarget,
19415                                      const SDLoc &DL) {
19416   if (Op.getSimpleValueType() != MVT::v2f64)
19417     return SDValue();
19418 
19419   bool IsStrict = Op->isStrictFPOpcode();
19420 
19421   SDValue N0 = Op.getOperand(IsStrict ? 1 : 0);
19422   assert(N0.getSimpleValueType() == MVT::v2i32 && "Unexpected input type");
19423 
19424   if (Subtarget.hasAVX512()) {
19425     if (!Subtarget.hasVLX()) {
19426       // Let generic type legalization widen this.
19427       if (!IsStrict)
19428         return SDValue();
19429       // Otherwise pad the integer input with 0s and widen the operation.
19430       N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19431                        DAG.getConstant(0, DL, MVT::v2i32));
19432       SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},
19433                                 {Op.getOperand(0), N0});
19434       SDValue Chain = Res.getValue(1);
19435       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res,
19436                         DAG.getIntPtrConstant(0, DL));
19437       return DAG.getMergeValues({Res, Chain}, DL);
19438     }
19439 
19440     // Legalize to v4i32 type.
19441     N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19442                      DAG.getUNDEF(MVT::v2i32));
19443     if (IsStrict)
19444       return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other},
19445                          {Op.getOperand(0), N0});
19446     return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0);
19447   }
19448 
19449   // Zero extend to 2i64, OR with the floating point representation of 2^52.
19450   // This gives us the floating point equivalent of 2^52 + the i32 integer
19451   // since double has 52-bits of mantissa. Then subtract 2^52 in floating
19452   // point leaving just our i32 integers in double format.
19453   SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);
19454   SDValue VBias = DAG.getConstantFP(
19455       llvm::bit_cast<double>(0x4330000000000000ULL), DL, MVT::v2f64);
19456   SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,
19457                            DAG.getBitcast(MVT::v2i64, VBias));
19458   Or = DAG.getBitcast(MVT::v2f64, Or);
19459 
19460   if (IsStrict)
19461     return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other},
19462                        {Op.getOperand(0), Or, VBias});
19463   return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias);
19464 }
19465 
19466 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
19467                                      const X86Subtarget &Subtarget) {
19468   SDLoc DL(Op);
19469   bool IsStrict = Op->isStrictFPOpcode();
19470   SDValue V = Op->getOperand(IsStrict ? 1 : 0);
19471   MVT VecIntVT = V.getSimpleValueType();
19472   assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
19473          "Unsupported custom type");
19474 
19475   if (Subtarget.hasAVX512()) {
19476     // With AVX512, but not VLX we need to widen to get a 512-bit result type.
19477     assert(!Subtarget.hasVLX() && "Unexpected features");
19478     MVT VT = Op->getSimpleValueType(0);
19479 
19480     // v8i32->v8f64 is legal with AVX512 so just return it.
19481     if (VT == MVT::v8f64)
19482       return Op;
19483 
19484     assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&
19485            "Unexpected VT!");
19486     MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
19487     MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
19488     // Need to concat with zero vector for strict fp to avoid spurious
19489     // exceptions.
19490     SDValue Tmp =
19491         IsStrict ? DAG.getConstant(0, DL, WideIntVT) : DAG.getUNDEF(WideIntVT);
19492     V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V,
19493                     DAG.getIntPtrConstant(0, DL));
19494     SDValue Res, Chain;
19495     if (IsStrict) {
19496       Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other},
19497                         {Op->getOperand(0), V});
19498       Chain = Res.getValue(1);
19499     } else {
19500       Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V);
19501     }
19502 
19503     Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19504                       DAG.getIntPtrConstant(0, DL));
19505 
19506     if (IsStrict)
19507       return DAG.getMergeValues({Res, Chain}, DL);
19508     return Res;
19509   }
19510 
19511   if (Subtarget.hasAVX() && VecIntVT == MVT::v4i32 &&
19512       Op->getSimpleValueType(0) == MVT::v4f64) {
19513     SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);
19514     Constant *Bias = ConstantFP::get(
19515         *DAG.getContext(),
19516         APFloat(APFloat::IEEEdouble(), APInt(64, 0x4330000000000000ULL)));
19517     auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
19518     SDValue CPIdx = DAG.getConstantPool(Bias, PtrVT, Align(8));
19519     SDVTList Tys = DAG.getVTList(MVT::v4f64, MVT::Other);
19520     SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
19521     SDValue VBias = DAG.getMemIntrinsicNode(
19522         X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,
19523         MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(8),
19524         MachineMemOperand::MOLoad);
19525 
19526     SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
19527                              DAG.getBitcast(MVT::v4i64, VBias));
19528     Or = DAG.getBitcast(MVT::v4f64, Or);
19529 
19530     if (IsStrict)
19531       return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other},
19532                          {Op.getOperand(0), Or, VBias});
19533     return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias);
19534   }
19535 
19536   // The algorithm is the following:
19537   // #ifdef __SSE4_1__
19538   //     uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
19539   //     uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
19540   //                                 (uint4) 0x53000000, 0xaa);
19541   // #else
19542   //     uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
19543   //     uint4 hi = (v >> 16) | (uint4) 0x53000000;
19544   // #endif
19545   //     float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
19546   //     return (float4) lo + fhi;
19547 
19548   bool Is128 = VecIntVT == MVT::v4i32;
19549   MVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32;
19550   // If we convert to something else than the supported type, e.g., to v4f64,
19551   // abort early.
19552   if (VecFloatVT != Op->getSimpleValueType(0))
19553     return SDValue();
19554 
19555   // In the #idef/#else code, we have in common:
19556   // - The vector of constants:
19557   // -- 0x4b000000
19558   // -- 0x53000000
19559   // - A shift:
19560   // -- v >> 16
19561 
19562   // Create the splat vector for 0x4b000000.
19563   SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
19564   // Create the splat vector for 0x53000000.
19565   SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
19566 
19567   // Create the right shift.
19568   SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
19569   SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
19570 
19571   SDValue Low, High;
19572   if (Subtarget.hasSSE41()) {
19573     MVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16;
19574     //     uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
19575     SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
19576     SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
19577     // Low will be bitcasted right away, so do not bother bitcasting back to its
19578     // original type.
19579     Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
19580                       VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19581     //     uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
19582     //                                 (uint4) 0x53000000, 0xaa);
19583     SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
19584     SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
19585     // High will be bitcasted right away, so do not bother bitcasting back to
19586     // its original type.
19587     High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
19588                        VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19589   } else {
19590     SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
19591     //     uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
19592     SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
19593     Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow);
19594 
19595     //     uint4 hi = (v >> 16) | (uint4) 0x53000000;
19596     High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh);
19597   }
19598 
19599   // Create the vector constant for (0x1.0p39f + 0x1.0p23f).
19600   SDValue VecCstFSub = DAG.getConstantFP(
19601       APFloat(APFloat::IEEEsingle(), APInt(32, 0x53000080)), DL, VecFloatVT);
19602 
19603   //     float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
19604   // NOTE: By using fsub of a positive constant instead of fadd of a negative
19605   // constant, we avoid reassociation in MachineCombiner when unsafe-fp-math is
19606   // enabled. See PR24512.
19607   SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
19608   // TODO: Are there any fast-math-flags to propagate here?
19609   //     (float4) lo;
19610   SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
19611   //     return (float4) lo + fhi;
19612   if (IsStrict) {
19613     SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},
19614                                 {Op.getOperand(0), HighBitcast, VecCstFSub});
19615     return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other},
19616                        {FHigh.getValue(1), LowBitcast, FHigh});
19617   }
19618 
19619   SDValue FHigh =
19620       DAG.getNode(ISD::FSUB, DL, VecFloatVT, HighBitcast, VecCstFSub);
19621   return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
19622 }
19623 
19624 static SDValue lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG,
19625                                    const X86Subtarget &Subtarget) {
19626   unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
19627   SDValue N0 = Op.getOperand(OpNo);
19628   MVT SrcVT = N0.getSimpleValueType();
19629   SDLoc dl(Op);
19630 
19631   switch (SrcVT.SimpleTy) {
19632   default:
19633     llvm_unreachable("Custom UINT_TO_FP is not supported!");
19634   case MVT::v2i32:
19635     return lowerUINT_TO_FP_v2i32(Op, DAG, Subtarget, dl);
19636   case MVT::v4i32:
19637   case MVT::v8i32:
19638     return lowerUINT_TO_FP_vXi32(Op, DAG, Subtarget);
19639   case MVT::v2i64:
19640   case MVT::v4i64:
19641     return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget);
19642   }
19643 }
19644 
19645 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
19646                                            SelectionDAG &DAG) const {
19647   bool IsStrict = Op->isStrictFPOpcode();
19648   unsigned OpNo = IsStrict ? 1 : 0;
19649   SDValue Src = Op.getOperand(OpNo);
19650   SDLoc dl(Op);
19651   auto PtrVT = getPointerTy(DAG.getDataLayout());
19652   MVT SrcVT = Src.getSimpleValueType();
19653   MVT DstVT = Op->getSimpleValueType(0);
19654   SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
19655 
19656   // Bail out when we don't have native conversion instructions.
19657   if (DstVT == MVT::f128)
19658     return SDValue();
19659 
19660   if (isSoftF16(DstVT, Subtarget))
19661     return promoteXINT_TO_FP(Op, DAG);
19662   else if (isLegalConversion(SrcVT, false, Subtarget))
19663     return Op;
19664 
19665   if (DstVT.isVector())
19666     return lowerUINT_TO_FP_vec(Op, DAG, Subtarget);
19667 
19668   if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)
19669     return LowerWin64_INT128_TO_FP(Op, DAG);
19670 
19671   if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
19672     return Extract;
19673 
19674   if (Subtarget.hasAVX512() && isScalarFPTypeInSSEReg(DstVT) &&
19675       (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget.is64Bit()))) {
19676     // Conversions from unsigned i32 to f32/f64 are legal,
19677     // using VCVTUSI2SS/SD.  Same for i64 in 64-bit mode.
19678     return Op;
19679   }
19680 
19681   // Promote i32 to i64 and use a signed conversion on 64-bit targets.
19682   if (SrcVT == MVT::i32 && Subtarget.is64Bit()) {
19683     Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Src);
19684     if (IsStrict)
19685       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DstVT, MVT::Other},
19686                          {Chain, Src});
19687     return DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src);
19688   }
19689 
19690   if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
19691     return V;
19692   if (SDValue V = LowerI64IntToFP16(Op, DAG, Subtarget))
19693     return V;
19694 
19695   // The transform for i64->f64 isn't correct for 0 when rounding to negative
19696   // infinity. It produces -0.0, so disable under strictfp.
19697   if (SrcVT == MVT::i64 && DstVT == MVT::f64 && Subtarget.hasSSE2() &&
19698       !IsStrict)
19699     return LowerUINT_TO_FP_i64(Op, DAG, Subtarget);
19700   // The transform for i32->f64/f32 isn't correct for 0 when rounding to
19701   // negative infinity. So disable under strictfp. Using FILD instead.
19702   if (SrcVT == MVT::i32 && Subtarget.hasSSE2() && DstVT != MVT::f80 &&
19703       !IsStrict)
19704     return LowerUINT_TO_FP_i32(Op, DAG, Subtarget);
19705   if (Subtarget.is64Bit() && SrcVT == MVT::i64 &&
19706       (DstVT == MVT::f32 || DstVT == MVT::f64))
19707     return SDValue();
19708 
19709   // Make a 64-bit buffer, and use it to build an FILD.
19710   SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8);
19711   int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex();
19712   Align SlotAlign(8);
19713   MachinePointerInfo MPI =
19714     MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
19715   if (SrcVT == MVT::i32) {
19716     SDValue OffsetSlot =
19717         DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
19718     SDValue Store1 = DAG.getStore(Chain, dl, Src, StackSlot, MPI, SlotAlign);
19719     SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
19720                                   OffsetSlot, MPI.getWithOffset(4), SlotAlign);
19721     std::pair<SDValue, SDValue> Tmp =
19722         BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, SlotAlign, DAG);
19723     if (IsStrict)
19724       return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
19725 
19726     return Tmp.first;
19727   }
19728 
19729   assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP");
19730   SDValue ValueToStore = Src;
19731   if (isScalarFPTypeInSSEReg(Op.getValueType()) && !Subtarget.is64Bit()) {
19732     // Bitcasting to f64 here allows us to do a single 64-bit store from
19733     // an SSE register, avoiding the store forwarding penalty that would come
19734     // with two 32-bit stores.
19735     ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
19736   }
19737   SDValue Store =
19738       DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, SlotAlign);
19739   // For i64 source, we need to add the appropriate power of 2 if the input
19740   // was negative. We must be careful to do the computation in x87 extended
19741   // precision, not in SSE.
19742   SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
19743   SDValue Ops[] = { Store, StackSlot };
19744   SDValue Fild =
19745       DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, MVT::i64, MPI,
19746                               SlotAlign, MachineMemOperand::MOLoad);
19747   Chain = Fild.getValue(1);
19748 
19749 
19750   // Check whether the sign bit is set.
19751   SDValue SignSet = DAG.getSetCC(
19752       dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
19753       Op.getOperand(OpNo), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
19754 
19755   // Build a 64 bit pair (FF, 0) in the constant pool, with FF in the hi bits.
19756   APInt FF(64, 0x5F80000000000000ULL);
19757   SDValue FudgePtr = DAG.getConstantPool(
19758       ConstantInt::get(*DAG.getContext(), FF), PtrVT);
19759   Align CPAlignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlign();
19760 
19761   // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
19762   SDValue Zero = DAG.getIntPtrConstant(0, dl);
19763   SDValue Four = DAG.getIntPtrConstant(4, dl);
19764   SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Four, Zero);
19765   FudgePtr = DAG.getNode(ISD::ADD, dl, PtrVT, FudgePtr, Offset);
19766 
19767   // Load the value out, extending it from f32 to f80.
19768   SDValue Fudge = DAG.getExtLoad(
19769       ISD::EXTLOAD, dl, MVT::f80, Chain, FudgePtr,
19770       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
19771       CPAlignment);
19772   Chain = Fudge.getValue(1);
19773   // Extend everything to 80 bits to force it to be done on x87.
19774   // TODO: Are there any fast-math-flags to propagate here?
19775   if (IsStrict) {
19776     unsigned Opc = ISD::STRICT_FADD;
19777     // Windows needs the precision control changed to 80bits around this add.
19778     if (Subtarget.isOSWindows() && DstVT == MVT::f32)
19779       Opc = X86ISD::STRICT_FP80_ADD;
19780 
19781     SDValue Add =
19782         DAG.getNode(Opc, dl, {MVT::f80, MVT::Other}, {Chain, Fild, Fudge});
19783     // STRICT_FP_ROUND can't handle equal types.
19784     if (DstVT == MVT::f80)
19785       return Add;
19786     return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other},
19787                        {Add.getValue(1), Add, DAG.getIntPtrConstant(0, dl)});
19788   }
19789   unsigned Opc = ISD::FADD;
19790   // Windows needs the precision control changed to 80bits around this add.
19791   if (Subtarget.isOSWindows() && DstVT == MVT::f32)
19792     Opc = X86ISD::FP80_ADD;
19793 
19794   SDValue Add = DAG.getNode(Opc, dl, MVT::f80, Fild, Fudge);
19795   return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add,
19796                      DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
19797 }
19798 
19799 // If the given FP_TO_SINT (IsSigned) or FP_TO_UINT (!IsSigned) operation
19800 // is legal, or has an fp128 or f16 source (which needs to be promoted to f32),
19801 // just return an SDValue().
19802 // Otherwise it is assumed to be a conversion from one of f32, f64 or f80
19803 // to i16, i32 or i64, and we lower it to a legal sequence and return the
19804 // result.
19805 SDValue
19806 X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
19807                                    bool IsSigned, SDValue &Chain) const {
19808   bool IsStrict = Op->isStrictFPOpcode();
19809   SDLoc DL(Op);
19810 
19811   EVT DstTy = Op.getValueType();
19812   SDValue Value = Op.getOperand(IsStrict ? 1 : 0);
19813   EVT TheVT = Value.getValueType();
19814   auto PtrVT = getPointerTy(DAG.getDataLayout());
19815 
19816   if (TheVT != MVT::f32 && TheVT != MVT::f64 && TheVT != MVT::f80) {
19817     // f16 must be promoted before using the lowering in this routine.
19818     // fp128 does not use this lowering.
19819     return SDValue();
19820   }
19821 
19822   // If using FIST to compute an unsigned i64, we'll need some fixup
19823   // to handle values above the maximum signed i64.  A FIST is always
19824   // used for the 32-bit subtarget, but also for f80 on a 64-bit target.
19825   bool UnsignedFixup = !IsSigned && DstTy == MVT::i64;
19826 
19827   // FIXME: This does not generate an invalid exception if the input does not
19828   // fit in i32. PR44019
19829   if (!IsSigned && DstTy != MVT::i64) {
19830     // Replace the fp-to-uint32 operation with an fp-to-sint64 FIST.
19831     // The low 32 bits of the fist result will have the correct uint32 result.
19832     assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
19833     DstTy = MVT::i64;
19834   }
19835 
19836   assert(DstTy.getSimpleVT() <= MVT::i64 &&
19837          DstTy.getSimpleVT() >= MVT::i16 &&
19838          "Unknown FP_TO_INT to lower!");
19839 
19840   // We lower FP->int64 into FISTP64 followed by a load from a temporary
19841   // stack slot.
19842   MachineFunction &MF = DAG.getMachineFunction();
19843   unsigned MemSize = DstTy.getStoreSize();
19844   int SSFI =
19845       MF.getFrameInfo().CreateStackObject(MemSize, Align(MemSize), false);
19846   SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19847 
19848   Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
19849 
19850   SDValue Adjust; // 0x0 or 0x80000000, for result sign bit adjustment.
19851 
19852   if (UnsignedFixup) {
19853     //
19854     // Conversion to unsigned i64 is implemented with a select,
19855     // depending on whether the source value fits in the range
19856     // of a signed i64.  Let Thresh be the FP equivalent of
19857     // 0x8000000000000000ULL.
19858     //
19859     //  Adjust = (Value >= Thresh) ? 0x80000000 : 0;
19860     //  FltOfs = (Value >= Thresh) ? 0x80000000 : 0;
19861     //  FistSrc = (Value - FltOfs);
19862     //  Fist-to-mem64 FistSrc
19863     //  Add 0 or 0x800...0ULL to the 64-bit result, which is equivalent
19864     //  to XOR'ing the high 32 bits with Adjust.
19865     //
19866     // Being a power of 2, Thresh is exactly representable in all FP formats.
19867     // For X87 we'd like to use the smallest FP type for this constant, but
19868     // for DAG type consistency we have to match the FP operand type.
19869 
19870     APFloat Thresh(APFloat::IEEEsingle(), APInt(32, 0x5f000000));
19871     LLVM_ATTRIBUTE_UNUSED APFloat::opStatus Status = APFloat::opOK;
19872     bool LosesInfo = false;
19873     if (TheVT == MVT::f64)
19874       // The rounding mode is irrelevant as the conversion should be exact.
19875       Status = Thresh.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
19876                               &LosesInfo);
19877     else if (TheVT == MVT::f80)
19878       Status = Thresh.convert(APFloat::x87DoubleExtended(),
19879                               APFloat::rmNearestTiesToEven, &LosesInfo);
19880 
19881     assert(Status == APFloat::opOK && !LosesInfo &&
19882            "FP conversion should have been exact");
19883 
19884     SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
19885 
19886     EVT ResVT = getSetCCResultType(DAG.getDataLayout(),
19887                                    *DAG.getContext(), TheVT);
19888     SDValue Cmp;
19889     if (IsStrict) {
19890       Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE, Chain,
19891                          /*IsSignaling*/ true);
19892       Chain = Cmp.getValue(1);
19893     } else {
19894       Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE);
19895     }
19896 
19897     // Our preferred lowering of
19898     //
19899     // (Value >= Thresh) ? 0x8000000000000000ULL : 0
19900     //
19901     // is
19902     //
19903     // (Value >= Thresh) << 63
19904     //
19905     // but since we can get here after LegalOperations, DAGCombine might do the
19906     // wrong thing if we create a select. So, directly create the preferred
19907     // version.
19908     SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp);
19909     SDValue Const63 = DAG.getConstant(63, DL, MVT::i8);
19910     Adjust = DAG.getNode(ISD::SHL, DL, MVT::i64, Zext, Const63);
19911 
19912     SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp, ThreshVal,
19913                                    DAG.getConstantFP(0.0, DL, TheVT));
19914 
19915     if (IsStrict) {
19916       Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},
19917                           { Chain, Value, FltOfs });
19918       Chain = Value.getValue(1);
19919     } else
19920       Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs);
19921   }
19922 
19923   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, SSFI);
19924 
19925   // FIXME This causes a redundant load/store if the SSE-class value is already
19926   // in memory, such as if it is on the callstack.
19927   if (isScalarFPTypeInSSEReg(TheVT)) {
19928     assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
19929     Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI);
19930     SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
19931     SDValue Ops[] = { Chain, StackSlot };
19932 
19933     unsigned FLDSize = TheVT.getStoreSize();
19934     assert(FLDSize <= MemSize && "Stack slot not big enough");
19935     MachineMemOperand *MMO = MF.getMachineMemOperand(
19936         MPI, MachineMemOperand::MOLoad, FLDSize, Align(FLDSize));
19937     Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO);
19938     Chain = Value.getValue(1);
19939   }
19940 
19941   // Build the FP_TO_INT*_IN_MEM
19942   MachineMemOperand *MMO = MF.getMachineMemOperand(
19943       MPI, MachineMemOperand::MOStore, MemSize, Align(MemSize));
19944   SDValue Ops[] = { Chain, Value, StackSlot };
19945   SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
19946                                          DAG.getVTList(MVT::Other),
19947                                          Ops, DstTy, MMO);
19948 
19949   SDValue Res = DAG.getLoad(Op.getValueType(), SDLoc(Op), FIST, StackSlot, MPI);
19950   Chain = Res.getValue(1);
19951 
19952   // If we need an unsigned fixup, XOR the result with adjust.
19953   if (UnsignedFixup)
19954     Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust);
19955 
19956   return Res;
19957 }
19958 
19959 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
19960                               const X86Subtarget &Subtarget) {
19961   MVT VT = Op.getSimpleValueType();
19962   SDValue In = Op.getOperand(0);
19963   MVT InVT = In.getSimpleValueType();
19964   SDLoc dl(Op);
19965   unsigned Opc = Op.getOpcode();
19966 
19967   assert(VT.isVector() && InVT.isVector() && "Expected vector type");
19968   assert((Opc == ISD::ANY_EXTEND || Opc == ISD::ZERO_EXTEND) &&
19969          "Unexpected extension opcode");
19970   assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
19971          "Expected same number of elements");
19972   assert((VT.getVectorElementType() == MVT::i16 ||
19973           VT.getVectorElementType() == MVT::i32 ||
19974           VT.getVectorElementType() == MVT::i64) &&
19975          "Unexpected element type");
19976   assert((InVT.getVectorElementType() == MVT::i8 ||
19977           InVT.getVectorElementType() == MVT::i16 ||
19978           InVT.getVectorElementType() == MVT::i32) &&
19979          "Unexpected element type");
19980 
19981   unsigned ExtendInVecOpc = DAG.getOpcode_EXTEND_VECTOR_INREG(Opc);
19982 
19983   if (VT == MVT::v32i16 && !Subtarget.hasBWI()) {
19984     assert(InVT == MVT::v32i8 && "Unexpected VT!");
19985     return splitVectorIntUnary(Op, DAG);
19986   }
19987 
19988   if (Subtarget.hasInt256())
19989     return Op;
19990 
19991   // Optimize vectors in AVX mode:
19992   //
19993   //   v8i16 -> v8i32
19994   //   Use vpmovzwd for 4 lower elements  v8i16 -> v4i32.
19995   //   Use vpunpckhwd for 4 upper elements  v8i16 -> v4i32.
19996   //   Concat upper and lower parts.
19997   //
19998   //   v4i32 -> v4i64
19999   //   Use vpmovzdq for 4 lower elements  v4i32 -> v2i64.
20000   //   Use vpunpckhdq for 4 upper elements  v4i32 -> v2i64.
20001   //   Concat upper and lower parts.
20002   //
20003   MVT HalfVT = VT.getHalfNumVectorElementsVT();
20004   SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In);
20005 
20006   // Short-circuit if we can determine that each 128-bit half is the same value.
20007   // Otherwise, this is difficult to match and optimize.
20008   if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(In))
20009     if (hasIdenticalHalvesShuffleMask(Shuf->getMask()))
20010       return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo);
20011 
20012   SDValue ZeroVec = DAG.getConstant(0, dl, InVT);
20013   SDValue Undef = DAG.getUNDEF(InVT);
20014   bool NeedZero = Opc == ISD::ZERO_EXTEND;
20015   SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
20016   OpHi = DAG.getBitcast(HalfVT, OpHi);
20017 
20018   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
20019 }
20020 
20021 // Helper to split and extend a v16i1 mask to v16i8 or v16i16.
20022 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,
20023                                    const SDLoc &dl, SelectionDAG &DAG) {
20024   assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT.");
20025   SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20026                            DAG.getIntPtrConstant(0, dl));
20027   SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20028                            DAG.getIntPtrConstant(8, dl));
20029   Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo);
20030   Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi);
20031   SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);
20032   return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20033 }
20034 
20035 static  SDValue LowerZERO_EXTEND_Mask(SDValue Op,
20036                                       const X86Subtarget &Subtarget,
20037                                       SelectionDAG &DAG) {
20038   MVT VT = Op->getSimpleValueType(0);
20039   SDValue In = Op->getOperand(0);
20040   MVT InVT = In.getSimpleValueType();
20041   assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!");
20042   SDLoc DL(Op);
20043   unsigned NumElts = VT.getVectorNumElements();
20044 
20045   // For all vectors, but vXi8 we can just emit a sign_extend and a shift. This
20046   // avoids a constant pool load.
20047   if (VT.getVectorElementType() != MVT::i8) {
20048     SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
20049     return DAG.getNode(ISD::SRL, DL, VT, Extend,
20050                        DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
20051   }
20052 
20053   // Extend VT if BWI is not supported.
20054   MVT ExtVT = VT;
20055   if (!Subtarget.hasBWI()) {
20056     // If v16i32 is to be avoided, we'll need to split and concatenate.
20057     if (NumElts == 16 && !Subtarget.canExtendTo512DQ())
20058       return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG);
20059 
20060     ExtVT = MVT::getVectorVT(MVT::i32, NumElts);
20061   }
20062 
20063   // Widen to 512-bits if VLX is not supported.
20064   MVT WideVT = ExtVT;
20065   if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) {
20066     NumElts *= 512 / ExtVT.getSizeInBits();
20067     InVT = MVT::getVectorVT(MVT::i1, NumElts);
20068     In = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT, DAG.getUNDEF(InVT),
20069                      In, DAG.getIntPtrConstant(0, DL));
20070     WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(),
20071                               NumElts);
20072   }
20073 
20074   SDValue One = DAG.getConstant(1, DL, WideVT);
20075   SDValue Zero = DAG.getConstant(0, DL, WideVT);
20076 
20077   SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
20078 
20079   // Truncate if we had to extend above.
20080   if (VT != ExtVT) {
20081     WideVT = MVT::getVectorVT(MVT::i8, NumElts);
20082     SelectedVal = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SelectedVal);
20083   }
20084 
20085   // Extract back to 128/256-bit if we widened.
20086   if (WideVT != VT)
20087     SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal,
20088                               DAG.getIntPtrConstant(0, DL));
20089 
20090   return SelectedVal;
20091 }
20092 
20093 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
20094                                 SelectionDAG &DAG) {
20095   SDValue In = Op.getOperand(0);
20096   MVT SVT = In.getSimpleValueType();
20097 
20098   if (SVT.getVectorElementType() == MVT::i1)
20099     return LowerZERO_EXTEND_Mask(Op, Subtarget, DAG);
20100 
20101   assert(Subtarget.hasAVX() && "Expected AVX support");
20102   return LowerAVXExtend(Op, DAG, Subtarget);
20103 }
20104 
20105 /// Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
20106 /// It makes use of the fact that vectors with enough leading sign/zero bits
20107 /// prevent the PACKSS/PACKUS from saturating the results.
20108 /// AVX2 (Int256) sub-targets require extra shuffling as the PACK*S operates
20109 /// within each 128-bit lane.
20110 static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
20111                                       const SDLoc &DL, SelectionDAG &DAG,
20112                                       const X86Subtarget &Subtarget) {
20113   assert((Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) &&
20114          "Unexpected PACK opcode");
20115   assert(DstVT.isVector() && "VT not a vector?");
20116 
20117   // Requires SSE2 for PACKSS (SSE41 PACKUSDW is handled below).
20118   if (!Subtarget.hasSSE2())
20119     return SDValue();
20120 
20121   EVT SrcVT = In.getValueType();
20122 
20123   // No truncation required, we might get here due to recursive calls.
20124   if (SrcVT == DstVT)
20125     return In;
20126 
20127   unsigned NumElems = SrcVT.getVectorNumElements();
20128   if (NumElems < 2 || !isPowerOf2_32(NumElems) )
20129     return SDValue();
20130 
20131   unsigned DstSizeInBits = DstVT.getSizeInBits();
20132   unsigned SrcSizeInBits = SrcVT.getSizeInBits();
20133   assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
20134   assert(SrcSizeInBits > DstSizeInBits && "Illegal truncation");
20135 
20136   LLVMContext &Ctx = *DAG.getContext();
20137   EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
20138   EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
20139 
20140   // Pack to the largest type possible:
20141   // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB.
20142   EVT InVT = MVT::i16, OutVT = MVT::i8;
20143   if (SrcVT.getScalarSizeInBits() > 16 &&
20144       (Opcode == X86ISD::PACKSS || Subtarget.hasSSE41())) {
20145     InVT = MVT::i32;
20146     OutVT = MVT::i16;
20147   }
20148 
20149   // Sub-128-bit truncation - widen to 128-bit src and pack in the lower half.
20150   // On pre-AVX512, pack the src in both halves to help value tracking.
20151   if (SrcSizeInBits <= 128) {
20152     InVT = EVT::getVectorVT(Ctx, InVT, 128 / InVT.getSizeInBits());
20153     OutVT = EVT::getVectorVT(Ctx, OutVT, 128 / OutVT.getSizeInBits());
20154     In = widenSubVector(In, false, Subtarget, DAG, DL, 128);
20155     SDValue LHS = DAG.getBitcast(InVT, In);
20156     SDValue RHS = Subtarget.hasAVX512() ? DAG.getUNDEF(InVT) : LHS;
20157     SDValue Res = DAG.getNode(Opcode, DL, OutVT, LHS, RHS);
20158     Res = extractSubVector(Res, 0, DAG, DL, SrcSizeInBits / 2);
20159     Res = DAG.getBitcast(PackedVT, Res);
20160     return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20161   }
20162 
20163   // Split lower/upper subvectors.
20164   SDValue Lo, Hi;
20165   std::tie(Lo, Hi) = splitVector(In, DAG, DL);
20166 
20167   // If Hi is undef, then don't bother packing it and widen the result instead.
20168   if (Hi.isUndef()) {
20169     EVT DstHalfVT = DstVT.getHalfNumVectorElementsVT(Ctx);
20170     if (SDValue Res =
20171             truncateVectorWithPACK(Opcode, DstHalfVT, Lo, DL, DAG, Subtarget))
20172       return widenSubVector(Res, false, Subtarget, DAG, DL, DstSizeInBits);
20173   }
20174 
20175   unsigned SubSizeInBits = SrcSizeInBits / 2;
20176   InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
20177   OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
20178 
20179   // 256bit -> 128bit truncate - PACK lower/upper 128-bit subvectors.
20180   if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
20181     Lo = DAG.getBitcast(InVT, Lo);
20182     Hi = DAG.getBitcast(InVT, Hi);
20183     SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20184     return DAG.getBitcast(DstVT, Res);
20185   }
20186 
20187   // AVX2: 512bit -> 256bit truncate - PACK lower/upper 256-bit subvectors.
20188   // AVX2: 512bit -> 128bit truncate - PACK(PACK, PACK).
20189   if (SrcVT.is512BitVector() && Subtarget.hasInt256()) {
20190     Lo = DAG.getBitcast(InVT, Lo);
20191     Hi = DAG.getBitcast(InVT, Hi);
20192     SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20193 
20194     // 256-bit PACK(ARG0, ARG1) leaves us with ((LO0,LO1),(HI0,HI1)),
20195     // so we need to shuffle to get ((LO0,HI0),(LO1,HI1)).
20196     // Scale shuffle mask to avoid bitcasts and help ComputeNumSignBits.
20197     SmallVector<int, 64> Mask;
20198     int Scale = 64 / OutVT.getScalarSizeInBits();
20199     narrowShuffleMaskElts(Scale, { 0, 2, 1, 3 }, Mask);
20200     Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
20201 
20202     if (DstVT.is256BitVector())
20203       return DAG.getBitcast(DstVT, Res);
20204 
20205     // If 512bit -> 128bit truncate another stage.
20206     Res = DAG.getBitcast(PackedVT, Res);
20207     return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20208   }
20209 
20210   // Recursively pack lower/upper subvectors, concat result and pack again.
20211   assert(SrcSizeInBits >= 256 && "Expected 256-bit vector or greater");
20212 
20213   if (PackedVT.is128BitVector()) {
20214     // Avoid CONCAT_VECTORS on sub-128bit nodes as these can fail after
20215     // type legalization.
20216     SDValue Res =
20217         truncateVectorWithPACK(Opcode, PackedVT, In, DL, DAG, Subtarget);
20218     return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20219   }
20220 
20221   EVT HalfPackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
20222   Lo = truncateVectorWithPACK(Opcode, HalfPackedVT, Lo, DL, DAG, Subtarget);
20223   Hi = truncateVectorWithPACK(Opcode, HalfPackedVT, Hi, DL, DAG, Subtarget);
20224   SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
20225   return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20226 }
20227 
20228 /// Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.
20229 /// e.g. trunc <8 x i32> X to <8 x i16> -->
20230 /// MaskX = X & 0xffff (clear high bits to prevent saturation)
20231 /// packus (extract_subv MaskX, 0), (extract_subv MaskX, 1)
20232 static SDValue truncateVectorWithPACKUS(EVT DstVT, SDValue In, const SDLoc &DL,
20233                                         const X86Subtarget &Subtarget,
20234                                         SelectionDAG &DAG) {
20235   In = DAG.getZeroExtendInReg(In, DL, DstVT);
20236   return truncateVectorWithPACK(X86ISD::PACKUS, DstVT, In, DL, DAG, Subtarget);
20237 }
20238 
20239 /// Truncate using inreg sign extension and X86ISD::PACKSS.
20240 static SDValue truncateVectorWithPACKSS(EVT DstVT, SDValue In, const SDLoc &DL,
20241                                         const X86Subtarget &Subtarget,
20242                                         SelectionDAG &DAG) {
20243   EVT SrcVT = In.getValueType();
20244   In = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SrcVT, In,
20245                    DAG.getValueType(DstVT));
20246   return truncateVectorWithPACK(X86ISD::PACKSS, DstVT, In, DL, DAG, Subtarget);
20247 }
20248 
20249 /// Helper to determine if \p In truncated to \p DstVT has the necessary
20250 /// signbits / leading zero bits to be truncated with PACKSS / PACKUS,
20251 /// possibly by converting a SRL node to SRA for sign extension.
20252 static SDValue matchTruncateWithPACK(unsigned &PackOpcode, EVT DstVT,
20253                                      SDValue In, const SDLoc &DL,
20254                                      SelectionDAG &DAG,
20255                                      const X86Subtarget &Subtarget) {
20256   // Requires SSE2.
20257   if (!Subtarget.hasSSE2())
20258     return SDValue();
20259 
20260   EVT SrcVT = In.getValueType();
20261   EVT DstSVT = DstVT.getVectorElementType();
20262   EVT SrcSVT = SrcVT.getVectorElementType();
20263 
20264   // Check we have a truncation suited for PACKSS/PACKUS.
20265   if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20266         (DstSVT == MVT::i8 || DstSVT == MVT::i16 || DstSVT == MVT::i32)))
20267     return SDValue();
20268 
20269   assert(SrcSVT.getSizeInBits() > DstSVT.getSizeInBits() && "Bad truncation");
20270   unsigned NumStages = Log2_32(SrcSVT.getSizeInBits() / DstSVT.getSizeInBits());
20271 
20272   // Truncation from 128-bit to vXi32 can be better handled with PSHUFD.
20273   // Truncation to sub-64-bit vXi16 can be better handled with PSHUFD/PSHUFLW.
20274   // Truncation from v2i64 to v2i8 can be better handled with PSHUFB.
20275   if ((DstSVT == MVT::i32 && SrcVT.getSizeInBits() <= 128) ||
20276       (DstSVT == MVT::i16 && SrcVT.getSizeInBits() <= (64 * NumStages)) ||
20277       (DstVT == MVT::v2i8 && SrcVT == MVT::v2i64 && Subtarget.hasSSSE3()))
20278     return SDValue();
20279 
20280   // Prefer to lower v4i64 -> v4i32 as a shuffle unless we can cheaply
20281   // split this for packing.
20282   if (SrcVT == MVT::v4i64 && DstVT == MVT::v4i32 &&
20283       !isFreeToSplitVector(In.getNode(), DAG) &&
20284       (!Subtarget.hasAVX() || DAG.ComputeNumSignBits(In) != 64))
20285     return SDValue();
20286 
20287   // Don't truncate AVX512 targets as multiple PACK nodes stages.
20288   if (Subtarget.hasAVX512() && NumStages > 1)
20289     return SDValue();
20290 
20291   unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
20292   unsigned NumPackedSignBits = std::min<unsigned>(DstSVT.getSizeInBits(), 16);
20293   unsigned NumPackedZeroBits = Subtarget.hasSSE41() ? NumPackedSignBits : 8;
20294 
20295   // Truncate with PACKUS if we are truncating a vector with leading zero
20296   // bits that extend all the way to the packed/truncated value.
20297   // e.g. Masks, zext_in_reg, etc.
20298   // Pre-SSE41 we can only use PACKUSWB.
20299   KnownBits Known = DAG.computeKnownBits(In);
20300   if ((NumSrcEltBits - NumPackedZeroBits) <= Known.countMinLeadingZeros()) {
20301     PackOpcode = X86ISD::PACKUS;
20302     return In;
20303   }
20304 
20305   // Truncate with PACKSS if we are truncating a vector with sign-bits
20306   // that extend all the way to the packed/truncated value.
20307   // e.g. Comparison result, sext_in_reg, etc.
20308   unsigned NumSignBits = DAG.ComputeNumSignBits(In);
20309 
20310   // Don't use PACKSS for vXi64 -> vXi32 truncations unless we're dealing with
20311   // a sign splat (or AVX512 VPSRAQ support). ComputeNumSignBits struggles to
20312   // see through BITCASTs later on and combines/simplifications can't then use
20313   // it.
20314   if (DstSVT == MVT::i32 && NumSignBits != SrcSVT.getSizeInBits() &&
20315       !Subtarget.hasAVX512())
20316     return SDValue();
20317 
20318   unsigned MinSignBits = NumSrcEltBits - NumPackedSignBits;
20319   if (MinSignBits < NumSignBits) {
20320     PackOpcode = X86ISD::PACKSS;
20321     return In;
20322   }
20323 
20324   // If we have a srl that only generates signbits that we will discard in
20325   // the truncation then we can use PACKSS by converting the srl to a sra.
20326   // SimplifyDemandedBits often relaxes sra to srl so we need to reverse it.
20327   if (In.getOpcode() == ISD::SRL && In->hasOneUse())
20328     if (const APInt *ShAmt = DAG.getValidShiftAmountConstant(
20329             In, APInt::getAllOnes(SrcVT.getVectorNumElements()))) {
20330       if (*ShAmt == MinSignBits) {
20331         PackOpcode = X86ISD::PACKSS;
20332         return DAG.getNode(ISD::SRA, DL, SrcVT, In->ops());
20333       }
20334     }
20335 
20336   return SDValue();
20337 }
20338 
20339 /// This function lowers a vector truncation of 'extended sign-bits' or
20340 /// 'extended zero-bits' values.
20341 /// vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.
20342 static SDValue LowerTruncateVecPackWithSignBits(MVT DstVT, SDValue In,
20343                                                 const SDLoc &DL,
20344                                                 const X86Subtarget &Subtarget,
20345                                                 SelectionDAG &DAG) {
20346   MVT SrcVT = In.getSimpleValueType();
20347   MVT DstSVT = DstVT.getVectorElementType();
20348   MVT SrcSVT = SrcVT.getVectorElementType();
20349   if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20350         (DstSVT == MVT::i8 || DstSVT == MVT::i16 || DstSVT == MVT::i32)))
20351     return SDValue();
20352 
20353   // If the upper half of the source is undef, then attempt to split and
20354   // only truncate the lower half.
20355   if (DstVT.getSizeInBits() >= 128) {
20356     SmallVector<SDValue> LowerOps;
20357     if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20358       MVT DstHalfVT = DstVT.getHalfNumVectorElementsVT();
20359       if (SDValue Res = LowerTruncateVecPackWithSignBits(DstHalfVT, Lo, DL,
20360                                                          Subtarget, DAG))
20361         return widenSubVector(Res, false, Subtarget, DAG, DL,
20362                               DstVT.getSizeInBits());
20363     }
20364   }
20365 
20366   unsigned PackOpcode;
20367   if (SDValue Src =
20368           matchTruncateWithPACK(PackOpcode, DstVT, In, DL, DAG, Subtarget))
20369     return truncateVectorWithPACK(PackOpcode, DstVT, Src, DL, DAG, Subtarget);
20370 
20371   return SDValue();
20372 }
20373 
20374 /// This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into
20375 /// X86ISD::PACKUS/X86ISD::PACKSS operations.
20376 static SDValue LowerTruncateVecPack(MVT DstVT, SDValue In, const SDLoc &DL,
20377                                     const X86Subtarget &Subtarget,
20378                                     SelectionDAG &DAG) {
20379   MVT SrcVT = In.getSimpleValueType();
20380   MVT DstSVT = DstVT.getVectorElementType();
20381   MVT SrcSVT = SrcVT.getVectorElementType();
20382   unsigned NumElems = DstVT.getVectorNumElements();
20383   if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20384         (DstSVT == MVT::i8 || DstSVT == MVT::i16) && isPowerOf2_32(NumElems) &&
20385         NumElems >= 8))
20386     return SDValue();
20387 
20388   // SSSE3's pshufb results in less instructions in the cases below.
20389   if (Subtarget.hasSSSE3() && NumElems == 8) {
20390     if (SrcSVT == MVT::i16)
20391       return SDValue();
20392     if (SrcSVT == MVT::i32 && (DstSVT == MVT::i8 || !Subtarget.hasSSE41()))
20393       return SDValue();
20394   }
20395 
20396   // If the upper half of the source is undef, then attempt to split and
20397   // only truncate the lower half.
20398   if (DstVT.getSizeInBits() >= 128) {
20399     SmallVector<SDValue> LowerOps;
20400     if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20401       MVT DstHalfVT = DstVT.getHalfNumVectorElementsVT();
20402       if (SDValue Res = LowerTruncateVecPack(DstHalfVT, Lo, DL, Subtarget, DAG))
20403         return widenSubVector(Res, false, Subtarget, DAG, DL,
20404                               DstVT.getSizeInBits());
20405     }
20406   }
20407 
20408   // SSE2 provides PACKUS for only 2 x v8i16 -> v16i8 and SSE4.1 provides PACKUS
20409   // for 2 x v4i32 -> v8i16. For SSSE3 and below, we need to use PACKSS to
20410   // truncate 2 x v4i32 to v8i16.
20411   if (Subtarget.hasSSE41() || DstSVT == MVT::i8)
20412     return truncateVectorWithPACKUS(DstVT, In, DL, Subtarget, DAG);
20413 
20414   if (SrcSVT == MVT::i16 || SrcSVT == MVT::i32)
20415     return truncateVectorWithPACKSS(DstVT, In, DL, Subtarget, DAG);
20416 
20417   // Special case vXi64 -> vXi16, shuffle to vXi32 and then use PACKSS.
20418   if (DstSVT == MVT::i16 && SrcSVT == MVT::i64) {
20419     MVT TruncVT = MVT::getVectorVT(MVT::i32, NumElems);
20420     SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, In);
20421     return truncateVectorWithPACKSS(DstVT, Trunc, DL, Subtarget, DAG);
20422   }
20423 
20424   return SDValue();
20425 }
20426 
20427 static SDValue LowerTruncateVecI1(SDValue Op, SelectionDAG &DAG,
20428                                   const X86Subtarget &Subtarget) {
20429 
20430   SDLoc DL(Op);
20431   MVT VT = Op.getSimpleValueType();
20432   SDValue In = Op.getOperand(0);
20433   MVT InVT = In.getSimpleValueType();
20434 
20435   assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type.");
20436 
20437   // Shift LSB to MSB and use VPMOVB/W2M or TESTD/Q.
20438   unsigned ShiftInx = InVT.getScalarSizeInBits() - 1;
20439   if (InVT.getScalarSizeInBits() <= 16) {
20440     if (Subtarget.hasBWI()) {
20441       // legal, will go to VPMOVB2M, VPMOVW2M
20442       if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
20443         // We need to shift to get the lsb into sign position.
20444         // Shift packed bytes not supported natively, bitcast to word
20445         MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits()/16);
20446         In = DAG.getNode(ISD::SHL, DL, ExtVT,
20447                          DAG.getBitcast(ExtVT, In),
20448                          DAG.getConstant(ShiftInx, DL, ExtVT));
20449         In = DAG.getBitcast(InVT, In);
20450       }
20451       return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),
20452                           In, ISD::SETGT);
20453     }
20454     // Use TESTD/Q, extended vector to packed dword/qword.
20455     assert((InVT.is256BitVector() || InVT.is128BitVector()) &&
20456            "Unexpected vector type.");
20457     unsigned NumElts = InVT.getVectorNumElements();
20458     assert((NumElts == 8 || NumElts == 16) && "Unexpected number of elements");
20459     // We need to change to a wider element type that we have support for.
20460     // For 8 element vectors this is easy, we either extend to v8i32 or v8i64.
20461     // For 16 element vectors we extend to v16i32 unless we are explicitly
20462     // trying to avoid 512-bit vectors. If we are avoiding 512-bit vectors
20463     // we need to split into two 8 element vectors which we can extend to v8i32,
20464     // truncate and concat the results. There's an additional complication if
20465     // the original type is v16i8. In that case we can't split the v16i8
20466     // directly, so we need to shuffle high elements to low and use
20467     // sign_extend_vector_inreg.
20468     if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) {
20469       SDValue Lo, Hi;
20470       if (InVT == MVT::v16i8) {
20471         Lo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, In);
20472         Hi = DAG.getVectorShuffle(
20473             InVT, DL, In, In,
20474             {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
20475         Hi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, Hi);
20476       } else {
20477         assert(InVT == MVT::v16i16 && "Unexpected VT!");
20478         Lo = extract128BitVector(In, 0, DAG, DL);
20479         Hi = extract128BitVector(In, 8, DAG, DL);
20480       }
20481       // We're split now, just emit two truncates and a concat. The two
20482       // truncates will trigger legalization to come back to this function.
20483       Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo);
20484       Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi);
20485       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
20486     }
20487     // We either have 8 elements or we're allowed to use 512-bit vectors.
20488     // If we have VLX, we want to use the narrowest vector that can get the
20489     // job done so we use vXi32.
20490     MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts);
20491     MVT ExtVT = MVT::getVectorVT(EltVT, NumElts);
20492     In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
20493     InVT = ExtVT;
20494     ShiftInx = InVT.getScalarSizeInBits() - 1;
20495   }
20496 
20497   if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
20498     // We need to shift to get the lsb into sign position.
20499     In = DAG.getNode(ISD::SHL, DL, InVT, In,
20500                      DAG.getConstant(ShiftInx, DL, InVT));
20501   }
20502   // If we have DQI, emit a pattern that will be iseled as vpmovq2m/vpmovd2m.
20503   if (Subtarget.hasDQI())
20504     return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);
20505   return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE);
20506 }
20507 
20508 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
20509   SDLoc DL(Op);
20510   MVT VT = Op.getSimpleValueType();
20511   SDValue In = Op.getOperand(0);
20512   MVT InVT = In.getSimpleValueType();
20513   assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
20514          "Invalid TRUNCATE operation");
20515 
20516   // If we're called by the type legalizer, handle a few cases.
20517   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20518   if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(InVT)) {
20519     if ((InVT == MVT::v8i64 || InVT == MVT::v16i32 || InVT == MVT::v16i64) &&
20520         VT.is128BitVector() && Subtarget.hasAVX512()) {
20521       assert((InVT == MVT::v16i64 || Subtarget.hasVLX()) &&
20522              "Unexpected subtarget!");
20523       // The default behavior is to truncate one step, concatenate, and then
20524       // truncate the remainder. We'd rather produce two 64-bit results and
20525       // concatenate those.
20526       SDValue Lo, Hi;
20527       std::tie(Lo, Hi) = DAG.SplitVector(In, DL);
20528 
20529       EVT LoVT, HiVT;
20530       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
20531 
20532       Lo = DAG.getNode(ISD::TRUNCATE, DL, LoVT, Lo);
20533       Hi = DAG.getNode(ISD::TRUNCATE, DL, HiVT, Hi);
20534       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
20535     }
20536 
20537     // Pre-AVX512 (or prefer-256bit) see if we can make use of PACKSS/PACKUS.
20538     if (!Subtarget.hasAVX512() ||
20539         (InVT.is512BitVector() && VT.is256BitVector()))
20540       if (SDValue SignPack =
20541               LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG))
20542         return SignPack;
20543 
20544     // Pre-AVX512 see if we can make use of PACKSS/PACKUS.
20545     if (!Subtarget.hasAVX512())
20546       return LowerTruncateVecPack(VT, In, DL, Subtarget, DAG);
20547 
20548     // Otherwise let default legalization handle it.
20549     return SDValue();
20550   }
20551 
20552   if (VT.getVectorElementType() == MVT::i1)
20553     return LowerTruncateVecI1(Op, DAG, Subtarget);
20554 
20555   // Attempt to truncate with PACKUS/PACKSS even on AVX512 if we'd have to
20556   // concat from subvectors to use VPTRUNC etc.
20557   if (!Subtarget.hasAVX512() || isFreeToSplitVector(In.getNode(), DAG))
20558     if (SDValue SignPack =
20559             LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG))
20560       return SignPack;
20561 
20562   // vpmovqb/w/d, vpmovdb/w, vpmovwb
20563   if (Subtarget.hasAVX512()) {
20564     if (InVT == MVT::v32i16 && !Subtarget.hasBWI()) {
20565       assert(VT == MVT::v32i8 && "Unexpected VT!");
20566       return splitVectorIntUnary(Op, DAG);
20567     }
20568 
20569     // word to byte only under BWI. Otherwise we have to promoted to v16i32
20570     // and then truncate that. But we should only do that if we haven't been
20571     // asked to avoid 512-bit vectors. The actual promotion to v16i32 will be
20572     // handled by isel patterns.
20573     if (InVT != MVT::v16i16 || Subtarget.hasBWI() ||
20574         Subtarget.canExtendTo512DQ())
20575       return Op;
20576   }
20577 
20578   // Handle truncation of V256 to V128 using shuffles.
20579   assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!");
20580 
20581   if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {
20582     // On AVX2, v4i64 -> v4i32 becomes VPERMD.
20583     if (Subtarget.hasInt256()) {
20584       static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1};
20585       In = DAG.getBitcast(MVT::v8i32, In);
20586       In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask);
20587       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
20588                          DAG.getIntPtrConstant(0, DL));
20589     }
20590 
20591     SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20592                                DAG.getIntPtrConstant(0, DL));
20593     SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20594                                DAG.getIntPtrConstant(2, DL));
20595     static const int ShufMask[] = {0, 2, 4, 6};
20596     return DAG.getVectorShuffle(VT, DL, DAG.getBitcast(MVT::v4i32, OpLo),
20597                                 DAG.getBitcast(MVT::v4i32, OpHi), ShufMask);
20598   }
20599 
20600   if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {
20601     // On AVX2, v8i32 -> v8i16 becomes PSHUFB.
20602     if (Subtarget.hasInt256()) {
20603       // The PSHUFB mask:
20604       static const int ShufMask1[] = { 0,  1,  4,  5,  8,  9, 12, 13,
20605                                       -1, -1, -1, -1, -1, -1, -1, -1,
20606                                       16, 17, 20, 21, 24, 25, 28, 29,
20607                                       -1, -1, -1, -1, -1, -1, -1, -1 };
20608       In = DAG.getBitcast(MVT::v32i8, In);
20609       In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
20610       In = DAG.getBitcast(MVT::v4i64, In);
20611 
20612       static const int ShufMask2[] = {0, 2, -1, -1};
20613       In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2);
20614       In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20615                        DAG.getIntPtrConstant(0, DL));
20616       return DAG.getBitcast(MVT::v8i16, In);
20617     }
20618 
20619     return Subtarget.hasSSE41()
20620                ? truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG)
20621                : truncateVectorWithPACKSS(VT, In, DL, Subtarget, DAG);
20622   }
20623 
20624   if (VT == MVT::v16i8 && InVT == MVT::v16i16)
20625     return truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG);
20626 
20627   llvm_unreachable("All 256->128 cases should have been handled above!");
20628 }
20629 
20630 // We can leverage the specific way the "cvttps2dq/cvttpd2dq" instruction
20631 // behaves on out of range inputs to generate optimized conversions.
20632 static SDValue expandFP_TO_UINT_SSE(MVT VT, SDValue Src, const SDLoc &dl,
20633                                     SelectionDAG &DAG,
20634                                     const X86Subtarget &Subtarget) {
20635   MVT SrcVT = Src.getSimpleValueType();
20636   unsigned DstBits = VT.getScalarSizeInBits();
20637   assert(DstBits == 32 && "expandFP_TO_UINT_SSE - only vXi32 supported");
20638 
20639   // Calculate the converted result for values in the range 0 to
20640   // 2^31-1 ("Small") and from 2^31 to 2^32-1 ("Big").
20641   SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src);
20642   SDValue Big =
20643       DAG.getNode(X86ISD::CVTTP2SI, dl, VT,
20644                   DAG.getNode(ISD::FSUB, dl, SrcVT, Src,
20645                               DAG.getConstantFP(2147483648.0f, dl, SrcVT)));
20646 
20647   // The "CVTTP2SI" instruction conveniently sets the sign bit if
20648   // and only if the value was out of range. So we can use that
20649   // as our indicator that we rather use "Big" instead of "Small".
20650   //
20651   // Use "Small" if "IsOverflown" has all bits cleared
20652   // and "0x80000000 | Big" if all bits in "IsOverflown" are set.
20653 
20654   // AVX1 can't use the signsplat masking for 256-bit vectors - we have to
20655   // use the slightly slower blendv select instead.
20656   if (VT == MVT::v8i32 && !Subtarget.hasAVX2()) {
20657     SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big);
20658     return DAG.getNode(X86ISD::BLENDV, dl, VT, Small, Overflow, Small);
20659   }
20660 
20661   SDValue IsOverflown =
20662       DAG.getNode(X86ISD::VSRAI, dl, VT, Small,
20663                   DAG.getTargetConstant(DstBits - 1, dl, MVT::i8));
20664   return DAG.getNode(ISD::OR, dl, VT, Small,
20665                      DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
20666 }
20667 
20668 SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
20669   bool IsStrict = Op->isStrictFPOpcode();
20670   bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
20671                   Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
20672   MVT VT = Op->getSimpleValueType(0);
20673   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
20674   SDValue Chain = IsStrict ? Op->getOperand(0) : SDValue();
20675   MVT SrcVT = Src.getSimpleValueType();
20676   SDLoc dl(Op);
20677 
20678   SDValue Res;
20679   if (isSoftF16(SrcVT, Subtarget)) {
20680     MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
20681     if (IsStrict)
20682       return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
20683                          {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
20684                                              {NVT, MVT::Other}, {Chain, Src})});
20685     return DAG.getNode(Op.getOpcode(), dl, VT,
20686                        DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
20687   } else if (isTypeLegal(SrcVT) && isLegalConversion(VT, IsSigned, Subtarget)) {
20688     return Op;
20689   }
20690 
20691   if (VT.isVector()) {
20692     if (VT == MVT::v2i1 && SrcVT == MVT::v2f64) {
20693       MVT ResVT = MVT::v4i32;
20694       MVT TruncVT = MVT::v4i1;
20695       unsigned Opc;
20696       if (IsStrict)
20697         Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
20698       else
20699         Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
20700 
20701       if (!IsSigned && !Subtarget.hasVLX()) {
20702         assert(Subtarget.useAVX512Regs() && "Unexpected features!");
20703         // Widen to 512-bits.
20704         ResVT = MVT::v8i32;
20705         TruncVT = MVT::v8i1;
20706         Opc = Op.getOpcode();
20707         // Need to concat with zero vector for strict fp to avoid spurious
20708         // exceptions.
20709         // TODO: Should we just do this for non-strict as well?
20710         SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64)
20711                                : DAG.getUNDEF(MVT::v8f64);
20712         Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, Tmp, Src,
20713                           DAG.getIntPtrConstant(0, dl));
20714       }
20715       if (IsStrict) {
20716         Res = DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {Chain, Src});
20717         Chain = Res.getValue(1);
20718       } else {
20719         Res = DAG.getNode(Opc, dl, ResVT, Src);
20720       }
20721 
20722       Res = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Res);
20723       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res,
20724                         DAG.getIntPtrConstant(0, dl));
20725       if (IsStrict)
20726         return DAG.getMergeValues({Res, Chain}, dl);
20727       return Res;
20728     }
20729 
20730     if (Subtarget.hasFP16() && SrcVT.getVectorElementType() == MVT::f16) {
20731       if (VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16)
20732         return Op;
20733 
20734       MVT ResVT = VT;
20735       MVT EleVT = VT.getVectorElementType();
20736       if (EleVT != MVT::i64)
20737         ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
20738 
20739       if (SrcVT != MVT::v8f16) {
20740         SDValue Tmp =
20741             IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
20742         SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
20743         Ops[0] = Src;
20744         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
20745       }
20746 
20747       if (IsStrict) {
20748         Res = DAG.getNode(IsSigned ? X86ISD::STRICT_CVTTP2SI
20749                                    : X86ISD::STRICT_CVTTP2UI,
20750                           dl, {ResVT, MVT::Other}, {Chain, Src});
20751         Chain = Res.getValue(1);
20752       } else {
20753         Res = DAG.getNode(IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI, dl,
20754                           ResVT, Src);
20755       }
20756 
20757       // TODO: Need to add exception check code for strict FP.
20758       if (EleVT.getSizeInBits() < 16) {
20759         ResVT = MVT::getVectorVT(EleVT, 8);
20760         Res = DAG.getNode(ISD::TRUNCATE, dl, ResVT, Res);
20761       }
20762 
20763       if (ResVT != VT)
20764         Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20765                           DAG.getIntPtrConstant(0, dl));
20766 
20767       if (IsStrict)
20768         return DAG.getMergeValues({Res, Chain}, dl);
20769       return Res;
20770     }
20771 
20772     // v8f32/v16f32/v8f64->v8i16/v16i16 need to widen first.
20773     if (VT.getVectorElementType() == MVT::i16) {
20774       assert((SrcVT.getVectorElementType() == MVT::f32 ||
20775               SrcVT.getVectorElementType() == MVT::f64) &&
20776              "Expected f32/f64 vector!");
20777       MVT NVT = VT.changeVectorElementType(MVT::i32);
20778       if (IsStrict) {
20779         Res = DAG.getNode(IsSigned ? ISD::STRICT_FP_TO_SINT
20780                                    : ISD::STRICT_FP_TO_UINT,
20781                           dl, {NVT, MVT::Other}, {Chain, Src});
20782         Chain = Res.getValue(1);
20783       } else {
20784         Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl,
20785                           NVT, Src);
20786       }
20787 
20788       // TODO: Need to add exception check code for strict FP.
20789       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20790 
20791       if (IsStrict)
20792         return DAG.getMergeValues({Res, Chain}, dl);
20793       return Res;
20794     }
20795 
20796     // v8f64->v8i32 is legal, but we need v8i32 to be custom for v8f32.
20797     if (VT == MVT::v8i32 && SrcVT == MVT::v8f64) {
20798       assert(!IsSigned && "Expected unsigned conversion!");
20799       assert(Subtarget.useAVX512Regs() && "Requires avx512f");
20800       return Op;
20801     }
20802 
20803     // Widen vXi32 fp_to_uint with avx512f to 512-bit source.
20804     if ((VT == MVT::v4i32 || VT == MVT::v8i32) &&
20805         (SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v8f32) &&
20806         Subtarget.useAVX512Regs()) {
20807       assert(!IsSigned && "Expected unsigned conversion!");
20808       assert(!Subtarget.hasVLX() && "Unexpected features!");
20809       MVT WideVT = SrcVT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
20810       MVT ResVT = SrcVT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
20811       // Need to concat with zero vector for strict fp to avoid spurious
20812       // exceptions.
20813       // TODO: Should we just do this for non-strict as well?
20814       SDValue Tmp =
20815           IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
20816       Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
20817                         DAG.getIntPtrConstant(0, dl));
20818 
20819       if (IsStrict) {
20820         Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other},
20821                           {Chain, Src});
20822         Chain = Res.getValue(1);
20823       } else {
20824         Res = DAG.getNode(ISD::FP_TO_UINT, dl, ResVT, Src);
20825       }
20826 
20827       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20828                         DAG.getIntPtrConstant(0, dl));
20829 
20830       if (IsStrict)
20831         return DAG.getMergeValues({Res, Chain}, dl);
20832       return Res;
20833     }
20834 
20835     // Widen vXi64 fp_to_uint/fp_to_sint with avx512dq to 512-bit source.
20836     if ((VT == MVT::v2i64 || VT == MVT::v4i64) &&
20837         (SrcVT == MVT::v2f64 || SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32) &&
20838         Subtarget.useAVX512Regs() && Subtarget.hasDQI()) {
20839       assert(!Subtarget.hasVLX() && "Unexpected features!");
20840       MVT WideVT = SrcVT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;
20841       // Need to concat with zero vector for strict fp to avoid spurious
20842       // exceptions.
20843       // TODO: Should we just do this for non-strict as well?
20844       SDValue Tmp =
20845           IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
20846       Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
20847                         DAG.getIntPtrConstant(0, dl));
20848 
20849       if (IsStrict) {
20850         Res = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
20851                           {Chain, Src});
20852         Chain = Res.getValue(1);
20853       } else {
20854         Res = DAG.getNode(Op.getOpcode(), dl, MVT::v8i64, Src);
20855       }
20856 
20857       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20858                         DAG.getIntPtrConstant(0, dl));
20859 
20860       if (IsStrict)
20861         return DAG.getMergeValues({Res, Chain}, dl);
20862       return Res;
20863     }
20864 
20865     if (VT == MVT::v2i64 && SrcVT == MVT::v2f32) {
20866       if (!Subtarget.hasVLX()) {
20867         // Non-strict nodes without VLX can we widened to v4f32->v4i64 by type
20868         // legalizer and then widened again by vector op legalization.
20869         if (!IsStrict)
20870           return SDValue();
20871 
20872         SDValue Zero = DAG.getConstantFP(0.0, dl, MVT::v2f32);
20873         SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32,
20874                                   {Src, Zero, Zero, Zero});
20875         Tmp = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
20876                           {Chain, Tmp});
20877         SDValue Chain = Tmp.getValue(1);
20878         Tmp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Tmp,
20879                           DAG.getIntPtrConstant(0, dl));
20880         return DAG.getMergeValues({Tmp, Chain}, dl);
20881       }
20882 
20883       assert(Subtarget.hasDQI() && Subtarget.hasVLX() && "Requires AVX512DQVL");
20884       SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
20885                                 DAG.getUNDEF(MVT::v2f32));
20886       if (IsStrict) {
20887         unsigned Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI
20888                                 : X86ISD::STRICT_CVTTP2UI;
20889         return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp});
20890       }
20891       unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
20892       return DAG.getNode(Opc, dl, VT, Tmp);
20893     }
20894 
20895     // Generate optimized instructions for pre AVX512 unsigned conversions from
20896     // vXf32 to vXi32.
20897     if ((VT == MVT::v4i32 && SrcVT == MVT::v4f32) ||
20898         (VT == MVT::v4i32 && SrcVT == MVT::v4f64) ||
20899         (VT == MVT::v8i32 && SrcVT == MVT::v8f32)) {
20900       assert(!IsSigned && "Expected unsigned conversion!");
20901       return expandFP_TO_UINT_SSE(VT, Src, dl, DAG, Subtarget);
20902     }
20903 
20904     return SDValue();
20905   }
20906 
20907   assert(!VT.isVector());
20908 
20909   bool UseSSEReg = isScalarFPTypeInSSEReg(SrcVT);
20910 
20911   if (!IsSigned && UseSSEReg) {
20912     // Conversions from f32/f64 with AVX512 should be legal.
20913     if (Subtarget.hasAVX512())
20914       return Op;
20915 
20916     // We can leverage the specific way the "cvttss2si/cvttsd2si" instruction
20917     // behaves on out of range inputs to generate optimized conversions.
20918     if (!IsStrict && ((VT == MVT::i32 && !Subtarget.is64Bit()) ||
20919                       (VT == MVT::i64 && Subtarget.is64Bit()))) {
20920       unsigned DstBits = VT.getScalarSizeInBits();
20921       APInt UIntLimit = APInt::getSignMask(DstBits);
20922       SDValue FloatOffset = DAG.getNode(ISD::UINT_TO_FP, dl, SrcVT,
20923                                         DAG.getConstant(UIntLimit, dl, VT));
20924       MVT SrcVecVT = MVT::getVectorVT(SrcVT, 128 / SrcVT.getScalarSizeInBits());
20925 
20926       // Calculate the converted result for values in the range:
20927       // (i32) 0 to 2^31-1 ("Small") and from 2^31 to 2^32-1 ("Big").
20928       // (i64) 0 to 2^63-1 ("Small") and from 2^63 to 2^64-1 ("Big").
20929       SDValue Small =
20930           DAG.getNode(X86ISD::CVTTS2SI, dl, VT,
20931                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, Src));
20932       SDValue Big = DAG.getNode(
20933           X86ISD::CVTTS2SI, dl, VT,
20934           DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT,
20935                       DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FloatOffset)));
20936 
20937       // The "CVTTS2SI" instruction conveniently sets the sign bit if
20938       // and only if the value was out of range. So we can use that
20939       // as our indicator that we rather use "Big" instead of "Small".
20940       //
20941       // Use "Small" if "IsOverflown" has all bits cleared
20942       // and "0x80000000 | Big" if all bits in "IsOverflown" are set.
20943       SDValue IsOverflown = DAG.getNode(
20944           ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8));
20945       return DAG.getNode(ISD::OR, dl, VT, Small,
20946                          DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
20947     }
20948 
20949     // Use default expansion for i64.
20950     if (VT == MVT::i64)
20951       return SDValue();
20952 
20953     assert(VT == MVT::i32 && "Unexpected VT!");
20954 
20955     // Promote i32 to i64 and use a signed operation on 64-bit targets.
20956     // FIXME: This does not generate an invalid exception if the input does not
20957     // fit in i32. PR44019
20958     if (Subtarget.is64Bit()) {
20959       if (IsStrict) {
20960         Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i64, MVT::Other},
20961                           {Chain, Src});
20962         Chain = Res.getValue(1);
20963       } else
20964         Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src);
20965 
20966       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20967       if (IsStrict)
20968         return DAG.getMergeValues({Res, Chain}, dl);
20969       return Res;
20970     }
20971 
20972     // Use default expansion for SSE1/2 targets without SSE3. With SSE3 we can
20973     // use fisttp which will be handled later.
20974     if (!Subtarget.hasSSE3())
20975       return SDValue();
20976   }
20977 
20978   // Promote i16 to i32 if we can use a SSE operation or the type is f128.
20979   // FIXME: This does not generate an invalid exception if the input does not
20980   // fit in i16. PR44019
20981   if (VT == MVT::i16 && (UseSSEReg || SrcVT == MVT::f128)) {
20982     assert(IsSigned && "Expected i16 FP_TO_UINT to have been promoted!");
20983     if (IsStrict) {
20984       Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i32, MVT::Other},
20985                         {Chain, Src});
20986       Chain = Res.getValue(1);
20987     } else
20988       Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);
20989 
20990     Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20991     if (IsStrict)
20992       return DAG.getMergeValues({Res, Chain}, dl);
20993     return Res;
20994   }
20995 
20996   // If this is a FP_TO_SINT using SSEReg we're done.
20997   if (UseSSEReg && IsSigned)
20998     return Op;
20999 
21000   // fp128 needs to use a libcall.
21001   if (SrcVT == MVT::f128) {
21002     RTLIB::Libcall LC;
21003     if (IsSigned)
21004       LC = RTLIB::getFPTOSINT(SrcVT, VT);
21005     else
21006       LC = RTLIB::getFPTOUINT(SrcVT, VT);
21007 
21008     MakeLibCallOptions CallOptions;
21009     std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, Src, CallOptions,
21010                                                   SDLoc(Op), Chain);
21011 
21012     if (IsStrict)
21013       return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
21014 
21015     return Tmp.first;
21016   }
21017 
21018   // Fall back to X87.
21019   if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned, Chain)) {
21020     if (IsStrict)
21021       return DAG.getMergeValues({V, Chain}, dl);
21022     return V;
21023   }
21024 
21025   llvm_unreachable("Expected FP_TO_INTHelper to handle all remaining cases.");
21026 }
21027 
21028 SDValue X86TargetLowering::LowerLRINT_LLRINT(SDValue Op,
21029                                              SelectionDAG &DAG) const {
21030   SDValue Src = Op.getOperand(0);
21031   MVT SrcVT = Src.getSimpleValueType();
21032 
21033   if (SrcVT == MVT::f16)
21034     return SDValue();
21035 
21036   // If the source is in an SSE register, the node is Legal.
21037   if (isScalarFPTypeInSSEReg(SrcVT))
21038     return Op;
21039 
21040   return LRINT_LLRINTHelper(Op.getNode(), DAG);
21041 }
21042 
21043 SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
21044                                               SelectionDAG &DAG) const {
21045   EVT DstVT = N->getValueType(0);
21046   SDValue Src = N->getOperand(0);
21047   EVT SrcVT = Src.getValueType();
21048 
21049   if (SrcVT != MVT::f32 && SrcVT != MVT::f64 && SrcVT != MVT::f80) {
21050     // f16 must be promoted before using the lowering in this routine.
21051     // fp128 does not use this lowering.
21052     return SDValue();
21053   }
21054 
21055   SDLoc DL(N);
21056   SDValue Chain = DAG.getEntryNode();
21057 
21058   bool UseSSE = isScalarFPTypeInSSEReg(SrcVT);
21059 
21060   // If we're converting from SSE, the stack slot needs to hold both types.
21061   // Otherwise it only needs to hold the DstVT.
21062   EVT OtherVT = UseSSE ? SrcVT : DstVT;
21063   SDValue StackPtr = DAG.CreateStackTemporary(DstVT, OtherVT);
21064   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
21065   MachinePointerInfo MPI =
21066       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
21067 
21068   if (UseSSE) {
21069     assert(DstVT == MVT::i64 && "Invalid LRINT/LLRINT to lower!");
21070     Chain = DAG.getStore(Chain, DL, Src, StackPtr, MPI);
21071     SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
21072     SDValue Ops[] = { Chain, StackPtr };
21073 
21074     Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI,
21075                                   /*Align*/ std::nullopt,
21076                                   MachineMemOperand::MOLoad);
21077     Chain = Src.getValue(1);
21078   }
21079 
21080   SDValue StoreOps[] = { Chain, Src, StackPtr };
21081   Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),
21082                                   StoreOps, DstVT, MPI, /*Align*/ std::nullopt,
21083                                   MachineMemOperand::MOStore);
21084 
21085   return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI);
21086 }
21087 
21088 SDValue
21089 X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
21090   // This is based on the TargetLowering::expandFP_TO_INT_SAT implementation,
21091   // but making use of X86 specifics to produce better instruction sequences.
21092   SDNode *Node = Op.getNode();
21093   bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
21094   unsigned FpToIntOpcode = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
21095   SDLoc dl(SDValue(Node, 0));
21096   SDValue Src = Node->getOperand(0);
21097 
21098   // There are three types involved here: SrcVT is the source floating point
21099   // type, DstVT is the type of the result, and TmpVT is the result of the
21100   // intermediate FP_TO_*INT operation we'll use (which may be a promotion of
21101   // DstVT).
21102   EVT SrcVT = Src.getValueType();
21103   EVT DstVT = Node->getValueType(0);
21104   EVT TmpVT = DstVT;
21105 
21106   // This code is only for floats and doubles. Fall back to generic code for
21107   // anything else.
21108   if (!isScalarFPTypeInSSEReg(SrcVT) || isSoftF16(SrcVT, Subtarget))
21109     return SDValue();
21110 
21111   EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
21112   unsigned SatWidth = SatVT.getScalarSizeInBits();
21113   unsigned DstWidth = DstVT.getScalarSizeInBits();
21114   unsigned TmpWidth = TmpVT.getScalarSizeInBits();
21115   assert(SatWidth <= DstWidth && SatWidth <= TmpWidth &&
21116          "Expected saturation width smaller than result width");
21117 
21118   // Promote result of FP_TO_*INT to at least 32 bits.
21119   if (TmpWidth < 32) {
21120     TmpVT = MVT::i32;
21121     TmpWidth = 32;
21122   }
21123 
21124   // Promote conversions to unsigned 32-bit to 64-bit, because it will allow
21125   // us to use a native signed conversion instead.
21126   if (SatWidth == 32 && !IsSigned && Subtarget.is64Bit()) {
21127     TmpVT = MVT::i64;
21128     TmpWidth = 64;
21129   }
21130 
21131   // If the saturation width is smaller than the size of the temporary result,
21132   // we can always use signed conversion, which is native.
21133   if (SatWidth < TmpWidth)
21134     FpToIntOpcode = ISD::FP_TO_SINT;
21135 
21136   // Determine minimum and maximum integer values and their corresponding
21137   // floating-point values.
21138   APInt MinInt, MaxInt;
21139   if (IsSigned) {
21140     MinInt = APInt::getSignedMinValue(SatWidth).sext(DstWidth);
21141     MaxInt = APInt::getSignedMaxValue(SatWidth).sext(DstWidth);
21142   } else {
21143     MinInt = APInt::getMinValue(SatWidth).zext(DstWidth);
21144     MaxInt = APInt::getMaxValue(SatWidth).zext(DstWidth);
21145   }
21146 
21147   APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));
21148   APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));
21149 
21150   APFloat::opStatus MinStatus = MinFloat.convertFromAPInt(
21151     MinInt, IsSigned, APFloat::rmTowardZero);
21152   APFloat::opStatus MaxStatus = MaxFloat.convertFromAPInt(
21153     MaxInt, IsSigned, APFloat::rmTowardZero);
21154   bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact)
21155                           && !(MaxStatus & APFloat::opStatus::opInexact);
21156 
21157   SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
21158   SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
21159 
21160   // If the integer bounds are exactly representable as floats, emit a
21161   // min+max+fptoi sequence. Otherwise use comparisons and selects.
21162   if (AreExactFloatBounds) {
21163     if (DstVT != TmpVT) {
21164       // Clamp by MinFloat from below. If Src is NaN, propagate NaN.
21165       SDValue MinClamped = DAG.getNode(
21166         X86ISD::FMAX, dl, SrcVT, MinFloatNode, Src);
21167       // Clamp by MaxFloat from above. If Src is NaN, propagate NaN.
21168       SDValue BothClamped = DAG.getNode(
21169         X86ISD::FMIN, dl, SrcVT, MaxFloatNode, MinClamped);
21170       // Convert clamped value to integer.
21171       SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, BothClamped);
21172 
21173       // NaN will become INDVAL, with the top bit set and the rest zero.
21174       // Truncation will discard the top bit, resulting in zero.
21175       return DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt);
21176     }
21177 
21178     // Clamp by MinFloat from below. If Src is NaN, the result is MinFloat.
21179     SDValue MinClamped = DAG.getNode(
21180       X86ISD::FMAX, dl, SrcVT, Src, MinFloatNode);
21181     // Clamp by MaxFloat from above. NaN cannot occur.
21182     SDValue BothClamped = DAG.getNode(
21183       X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode);
21184     // Convert clamped value to integer.
21185     SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped);
21186 
21187     if (!IsSigned) {
21188       // In the unsigned case we're done, because we mapped NaN to MinFloat,
21189       // which is zero.
21190       return FpToInt;
21191     }
21192 
21193     // Otherwise, select zero if Src is NaN.
21194     SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21195     return DAG.getSelectCC(
21196       dl, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
21197   }
21198 
21199   SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
21200   SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
21201 
21202   // Result of direct conversion, which may be selected away.
21203   SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, Src);
21204 
21205   if (DstVT != TmpVT) {
21206     // NaN will become INDVAL, with the top bit set and the rest zero.
21207     // Truncation will discard the top bit, resulting in zero.
21208     FpToInt = DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt);
21209   }
21210 
21211   SDValue Select = FpToInt;
21212   // For signed conversions where we saturate to the same size as the
21213   // result type of the fptoi instructions, INDVAL coincides with integer
21214   // minimum, so we don't need to explicitly check it.
21215   if (!IsSigned || SatWidth != TmpVT.getScalarSizeInBits()) {
21216     // If Src ULT MinFloat, select MinInt. In particular, this also selects
21217     // MinInt if Src is NaN.
21218     Select = DAG.getSelectCC(
21219       dl, Src, MinFloatNode, MinIntNode, Select, ISD::CondCode::SETULT);
21220   }
21221 
21222   // If Src OGT MaxFloat, select MaxInt.
21223   Select = DAG.getSelectCC(
21224     dl, Src, MaxFloatNode, MaxIntNode, Select, ISD::CondCode::SETOGT);
21225 
21226   // In the unsigned case we are done, because we mapped NaN to MinInt, which
21227   // is already zero. The promoted case was already handled above.
21228   if (!IsSigned || DstVT != TmpVT) {
21229     return Select;
21230   }
21231 
21232   // Otherwise, select 0 if Src is NaN.
21233   SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21234   return DAG.getSelectCC(
21235     dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
21236 }
21237 
21238 SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
21239   bool IsStrict = Op->isStrictFPOpcode();
21240 
21241   SDLoc DL(Op);
21242   MVT VT = Op.getSimpleValueType();
21243   SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21244   SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21245   MVT SVT = In.getSimpleValueType();
21246 
21247   // Let f16->f80 get lowered to a libcall, except for darwin, where we should
21248   // lower it to an fp_extend via f32 (as only f16<>f32 libcalls are available)
21249   if (VT == MVT::f128 || (SVT == MVT::f16 && VT == MVT::f80 &&
21250                           !Subtarget.getTargetTriple().isOSDarwin()))
21251     return SDValue();
21252 
21253   if ((SVT == MVT::v8f16 && Subtarget.hasF16C()) ||
21254       (SVT == MVT::v16f16 && Subtarget.useAVX512Regs()))
21255     return Op;
21256 
21257   if (SVT == MVT::f16) {
21258     if (Subtarget.hasFP16())
21259       return Op;
21260 
21261     if (VT != MVT::f32) {
21262       if (IsStrict)
21263         return DAG.getNode(
21264             ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other},
21265             {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, DL,
21266                                 {MVT::f32, MVT::Other}, {Chain, In})});
21267 
21268       return DAG.getNode(ISD::FP_EXTEND, DL, VT,
21269                          DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, In));
21270     }
21271 
21272     if (!Subtarget.hasF16C()) {
21273       if (!Subtarget.getTargetTriple().isOSDarwin())
21274         return SDValue();
21275 
21276       assert(VT == MVT::f32 && SVT == MVT::f16 && "unexpected extend libcall");
21277 
21278       // Need a libcall, but ABI for f16 is soft-float on MacOS.
21279       TargetLowering::CallLoweringInfo CLI(DAG);
21280       Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
21281 
21282       In = DAG.getBitcast(MVT::i16, In);
21283       TargetLowering::ArgListTy Args;
21284       TargetLowering::ArgListEntry Entry;
21285       Entry.Node = In;
21286       Entry.Ty = EVT(MVT::i16).getTypeForEVT(*DAG.getContext());
21287       Entry.IsSExt = false;
21288       Entry.IsZExt = true;
21289       Args.push_back(Entry);
21290 
21291       SDValue Callee = DAG.getExternalSymbol(
21292           getLibcallName(RTLIB::FPEXT_F16_F32),
21293           getPointerTy(DAG.getDataLayout()));
21294       CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
21295           CallingConv::C, EVT(VT).getTypeForEVT(*DAG.getContext()), Callee,
21296           std::move(Args));
21297 
21298       SDValue Res;
21299       std::tie(Res,Chain) = LowerCallTo(CLI);
21300       if (IsStrict)
21301         Res = DAG.getMergeValues({Res, Chain}, DL);
21302 
21303       return Res;
21304     }
21305 
21306     In = DAG.getBitcast(MVT::i16, In);
21307     In = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16,
21308                      getZeroVector(MVT::v8i16, Subtarget, DAG, DL), In,
21309                      DAG.getIntPtrConstant(0, DL));
21310     SDValue Res;
21311     if (IsStrict) {
21312       Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, DL, {MVT::v4f32, MVT::Other},
21313                         {Chain, In});
21314       Chain = Res.getValue(1);
21315     } else {
21316       Res = DAG.getNode(X86ISD::CVTPH2PS, DL, MVT::v4f32, In,
21317                         DAG.getTargetConstant(4, DL, MVT::i32));
21318     }
21319     Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Res,
21320                       DAG.getIntPtrConstant(0, DL));
21321     if (IsStrict)
21322       return DAG.getMergeValues({Res, Chain}, DL);
21323     return Res;
21324   }
21325 
21326   if (!SVT.isVector())
21327     return Op;
21328 
21329   if (SVT.getVectorElementType() == MVT::bf16) {
21330     // FIXME: Do we need to support strict FP?
21331     assert(!IsStrict && "Strict FP doesn't support BF16");
21332     if (VT.getVectorElementType() == MVT::f64) {
21333       MVT TmpVT = VT.changeVectorElementType(MVT::f32);
21334       return DAG.getNode(ISD::FP_EXTEND, DL, VT,
21335                          DAG.getNode(ISD::FP_EXTEND, DL, TmpVT, In));
21336     }
21337     assert(VT.getVectorElementType() == MVT::f32 && "Unexpected fpext");
21338     MVT NVT = SVT.changeVectorElementType(MVT::i32);
21339     In = DAG.getBitcast(SVT.changeTypeToInteger(), In);
21340     In = DAG.getNode(ISD::ZERO_EXTEND, DL, NVT, In);
21341     In = DAG.getNode(ISD::SHL, DL, NVT, In, DAG.getConstant(16, DL, NVT));
21342     return DAG.getBitcast(VT, In);
21343   }
21344 
21345   if (SVT.getVectorElementType() == MVT::f16) {
21346     if (Subtarget.hasFP16() && isTypeLegal(SVT))
21347       return Op;
21348     assert(Subtarget.hasF16C() && "Unexpected features!");
21349     if (SVT == MVT::v2f16)
21350       In = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f16, In,
21351                        DAG.getUNDEF(MVT::v2f16));
21352     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In,
21353                               DAG.getUNDEF(MVT::v4f16));
21354     if (IsStrict)
21355       return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21356                          {Op->getOperand(0), Res});
21357     return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21358   } else if (VT == MVT::v4f64 || VT == MVT::v8f64) {
21359     return Op;
21360   }
21361 
21362   assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!");
21363 
21364   SDValue Res =
21365       DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT));
21366   if (IsStrict)
21367     return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21368                        {Op->getOperand(0), Res});
21369   return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21370 }
21371 
21372 SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21373   bool IsStrict = Op->isStrictFPOpcode();
21374 
21375   SDLoc DL(Op);
21376   SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21377   SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21378   MVT VT = Op.getSimpleValueType();
21379   MVT SVT = In.getSimpleValueType();
21380 
21381   if (SVT == MVT::f128 || (VT == MVT::f16 && SVT == MVT::f80))
21382     return SDValue();
21383 
21384   if (VT == MVT::f16 && (SVT == MVT::f64 || SVT == MVT::f32) &&
21385       !Subtarget.hasFP16() && (SVT == MVT::f64 || !Subtarget.hasF16C())) {
21386     if (!Subtarget.getTargetTriple().isOSDarwin())
21387       return SDValue();
21388 
21389     // We need a libcall but the ABI for f16 libcalls on MacOS is soft.
21390     TargetLowering::CallLoweringInfo CLI(DAG);
21391     Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
21392 
21393     TargetLowering::ArgListTy Args;
21394     TargetLowering::ArgListEntry Entry;
21395     Entry.Node = In;
21396     Entry.Ty = EVT(SVT).getTypeForEVT(*DAG.getContext());
21397     Entry.IsSExt = false;
21398     Entry.IsZExt = true;
21399     Args.push_back(Entry);
21400 
21401     SDValue Callee = DAG.getExternalSymbol(
21402         getLibcallName(SVT == MVT::f64 ? RTLIB::FPROUND_F64_F16
21403                                        : RTLIB::FPROUND_F32_F16),
21404         getPointerTy(DAG.getDataLayout()));
21405     CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
21406         CallingConv::C, EVT(MVT::i16).getTypeForEVT(*DAG.getContext()), Callee,
21407         std::move(Args));
21408 
21409     SDValue Res;
21410     std::tie(Res, Chain) = LowerCallTo(CLI);
21411 
21412     Res = DAG.getBitcast(MVT::f16, Res);
21413 
21414     if (IsStrict)
21415       Res = DAG.getMergeValues({Res, Chain}, DL);
21416 
21417     return Res;
21418   }
21419 
21420   if (VT.getScalarType() == MVT::bf16) {
21421     if (SVT.getScalarType() == MVT::f32 && isTypeLegal(VT))
21422       return Op;
21423     return SDValue();
21424   }
21425 
21426   if (VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16()) {
21427     if (!Subtarget.hasF16C() || SVT.getScalarType() != MVT::f32)
21428       return SDValue();
21429 
21430     if (VT.isVector())
21431       return Op;
21432 
21433     SDValue Res;
21434     SDValue Rnd = DAG.getTargetConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, DL,
21435                                         MVT::i32);
21436     if (IsStrict) {
21437       Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4f32,
21438                         DAG.getConstantFP(0, DL, MVT::v4f32), In,
21439                         DAG.getIntPtrConstant(0, DL));
21440       Res = DAG.getNode(X86ISD::STRICT_CVTPS2PH, DL, {MVT::v8i16, MVT::Other},
21441                         {Chain, Res, Rnd});
21442       Chain = Res.getValue(1);
21443     } else {
21444       // FIXME: Should we use zeros for upper elements for non-strict?
21445       Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, In);
21446       Res = DAG.getNode(X86ISD::CVTPS2PH, DL, MVT::v8i16, Res, Rnd);
21447     }
21448 
21449     Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res,
21450                       DAG.getIntPtrConstant(0, DL));
21451     Res = DAG.getBitcast(MVT::f16, Res);
21452 
21453     if (IsStrict)
21454       return DAG.getMergeValues({Res, Chain}, DL);
21455 
21456     return Res;
21457   }
21458 
21459   return Op;
21460 }
21461 
21462 static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) {
21463   bool IsStrict = Op->isStrictFPOpcode();
21464   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21465   assert(Src.getValueType() == MVT::i16 && Op.getValueType() == MVT::f32 &&
21466          "Unexpected VT!");
21467 
21468   SDLoc dl(Op);
21469   SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16,
21470                             DAG.getConstant(0, dl, MVT::v8i16), Src,
21471                             DAG.getIntPtrConstant(0, dl));
21472 
21473   SDValue Chain;
21474   if (IsStrict) {
21475     Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {MVT::v4f32, MVT::Other},
21476                       {Op.getOperand(0), Res});
21477     Chain = Res.getValue(1);
21478   } else {
21479     Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
21480   }
21481 
21482   Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
21483                     DAG.getIntPtrConstant(0, dl));
21484 
21485   if (IsStrict)
21486     return DAG.getMergeValues({Res, Chain}, dl);
21487 
21488   return Res;
21489 }
21490 
21491 static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
21492   bool IsStrict = Op->isStrictFPOpcode();
21493   SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21494   assert(Src.getValueType() == MVT::f32 && Op.getValueType() == MVT::i16 &&
21495          "Unexpected VT!");
21496 
21497   SDLoc dl(Op);
21498   SDValue Res, Chain;
21499   if (IsStrict) {
21500     Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4f32,
21501                       DAG.getConstantFP(0, dl, MVT::v4f32), Src,
21502                       DAG.getIntPtrConstant(0, dl));
21503     Res = DAG.getNode(
21504         X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},
21505         {Op.getOperand(0), Res, DAG.getTargetConstant(4, dl, MVT::i32)});
21506     Chain = Res.getValue(1);
21507   } else {
21508     // FIXME: Should we use zeros for upper elements for non-strict?
21509     Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, Src);
21510     Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
21511                       DAG.getTargetConstant(4, dl, MVT::i32));
21512   }
21513 
21514   Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res,
21515                     DAG.getIntPtrConstant(0, dl));
21516 
21517   if (IsStrict)
21518     return DAG.getMergeValues({Res, Chain}, dl);
21519 
21520   return Res;
21521 }
21522 
21523 SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op,
21524                                            SelectionDAG &DAG) const {
21525   SDLoc DL(Op);
21526 
21527   MVT SVT = Op.getOperand(0).getSimpleValueType();
21528   if (SVT == MVT::f32 && (Subtarget.hasBF16() || Subtarget.hasAVXNECONVERT())) {
21529     SDValue Res;
21530     Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0));
21531     Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res);
21532     Res = DAG.getBitcast(MVT::v8i16, Res);
21533     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res,
21534                        DAG.getIntPtrConstant(0, DL));
21535   }
21536 
21537   MakeLibCallOptions CallOptions;
21538   RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, MVT::bf16);
21539   SDValue Res =
21540       makeLibCall(DAG, LC, MVT::f16, Op.getOperand(0), CallOptions, DL).first;
21541   return DAG.getBitcast(MVT::i16, Res);
21542 }
21543 
21544 /// Depending on uarch and/or optimizing for size, we might prefer to use a
21545 /// vector operation in place of the typical scalar operation.
21546 static SDValue lowerAddSubToHorizontalOp(SDValue Op, SelectionDAG &DAG,
21547                                          const X86Subtarget &Subtarget) {
21548   // If both operands have other uses, this is probably not profitable.
21549   SDValue LHS = Op.getOperand(0);
21550   SDValue RHS = Op.getOperand(1);
21551   if (!LHS.hasOneUse() && !RHS.hasOneUse())
21552     return Op;
21553 
21554   // FP horizontal add/sub were added with SSE3. Integer with SSSE3.
21555   bool IsFP = Op.getSimpleValueType().isFloatingPoint();
21556   if (IsFP && !Subtarget.hasSSE3())
21557     return Op;
21558   if (!IsFP && !Subtarget.hasSSSE3())
21559     return Op;
21560 
21561   // Extract from a common vector.
21562   if (LHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
21563       RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
21564       LHS.getOperand(0) != RHS.getOperand(0) ||
21565       !isa<ConstantSDNode>(LHS.getOperand(1)) ||
21566       !isa<ConstantSDNode>(RHS.getOperand(1)) ||
21567       !shouldUseHorizontalOp(true, DAG, Subtarget))
21568     return Op;
21569 
21570   // Allow commuted 'hadd' ops.
21571   // TODO: Allow commuted (f)sub by negating the result of (F)HSUB?
21572   unsigned HOpcode;
21573   switch (Op.getOpcode()) {
21574     case ISD::ADD: HOpcode = X86ISD::HADD; break;
21575     case ISD::SUB: HOpcode = X86ISD::HSUB; break;
21576     case ISD::FADD: HOpcode = X86ISD::FHADD; break;
21577     case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
21578     default:
21579       llvm_unreachable("Trying to lower unsupported opcode to horizontal op");
21580   }
21581   unsigned LExtIndex = LHS.getConstantOperandVal(1);
21582   unsigned RExtIndex = RHS.getConstantOperandVal(1);
21583   if ((LExtIndex & 1) == 1 && (RExtIndex & 1) == 0 &&
21584       (HOpcode == X86ISD::HADD || HOpcode == X86ISD::FHADD))
21585     std::swap(LExtIndex, RExtIndex);
21586 
21587   if ((LExtIndex & 1) != 0 || RExtIndex != (LExtIndex + 1))
21588     return Op;
21589 
21590   SDValue X = LHS.getOperand(0);
21591   EVT VecVT = X.getValueType();
21592   unsigned BitWidth = VecVT.getSizeInBits();
21593   unsigned NumLanes = BitWidth / 128;
21594   unsigned NumEltsPerLane = VecVT.getVectorNumElements() / NumLanes;
21595   assert((BitWidth == 128 || BitWidth == 256 || BitWidth == 512) &&
21596          "Not expecting illegal vector widths here");
21597 
21598   // Creating a 256-bit horizontal op would be wasteful, and there is no 512-bit
21599   // equivalent, so extract the 256/512-bit source op to 128-bit if we can.
21600   SDLoc DL(Op);
21601   if (BitWidth == 256 || BitWidth == 512) {
21602     unsigned LaneIdx = LExtIndex / NumEltsPerLane;
21603     X = extract128BitVector(X, LaneIdx * NumEltsPerLane, DAG, DL);
21604     LExtIndex %= NumEltsPerLane;
21605   }
21606 
21607   // add (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hadd X, X), 0
21608   // add (extractelt (X, 1), extractelt (X, 0)) --> extractelt (hadd X, X), 0
21609   // add (extractelt (X, 2), extractelt (X, 3)) --> extractelt (hadd X, X), 1
21610   // sub (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hsub X, X), 0
21611   SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
21612   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp,
21613                      DAG.getIntPtrConstant(LExtIndex / 2, DL));
21614 }
21615 
21616 /// Depending on uarch and/or optimizing for size, we might prefer to use a
21617 /// vector operation in place of the typical scalar operation.
21618 SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
21619   assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) &&
21620          "Only expecting float/double");
21621   return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
21622 }
21623 
21624 /// ISD::FROUND is defined to round to nearest with ties rounding away from 0.
21625 /// This mode isn't supported in hardware on X86. But as long as we aren't
21626 /// compiling with trapping math, we can emulate this with
21627 /// trunc(X + copysign(nextafter(0.5, 0.0), X)).
21628 static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
21629   SDValue N0 = Op.getOperand(0);
21630   SDLoc dl(Op);
21631   MVT VT = Op.getSimpleValueType();
21632 
21633   // N0 += copysign(nextafter(0.5, 0.0), N0)
21634   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
21635   bool Ignored;
21636   APFloat Point5Pred = APFloat(0.5f);
21637   Point5Pred.convert(Sem, APFloat::rmNearestTiesToEven, &Ignored);
21638   Point5Pred.next(/*nextDown*/true);
21639 
21640   SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT,
21641                               DAG.getConstantFP(Point5Pred, dl, VT), N0);
21642   N0 = DAG.getNode(ISD::FADD, dl, VT, N0, Adder);
21643 
21644   // Truncate the result to remove fraction.
21645   return DAG.getNode(ISD::FTRUNC, dl, VT, N0);
21646 }
21647 
21648 /// The only differences between FABS and FNEG are the mask and the logic op.
21649 /// FNEG also has a folding opportunity for FNEG(FABS(x)).
21650 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
21651   assert((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) &&
21652          "Wrong opcode for lowering FABS or FNEG.");
21653 
21654   bool IsFABS = (Op.getOpcode() == ISD::FABS);
21655 
21656   // If this is a FABS and it has an FNEG user, bail out to fold the combination
21657   // into an FNABS. We'll lower the FABS after that if it is still in use.
21658   if (IsFABS)
21659     for (SDNode *User : Op->uses())
21660       if (User->getOpcode() == ISD::FNEG)
21661         return Op;
21662 
21663   SDLoc dl(Op);
21664   MVT VT = Op.getSimpleValueType();
21665 
21666   bool IsF128 = (VT == MVT::f128);
21667   assert(VT.isFloatingPoint() && VT != MVT::f80 &&
21668          DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
21669          "Unexpected type in LowerFABSorFNEG");
21670 
21671   // FIXME: Use function attribute "OptimizeForSize" and/or CodeGenOptLevel to
21672   // decide if we should generate a 16-byte constant mask when we only need 4 or
21673   // 8 bytes for the scalar case.
21674 
21675   // There are no scalar bitwise logical SSE/AVX instructions, so we
21676   // generate a 16-byte vector constant and logic op even for the scalar case.
21677   // Using a 16-byte mask allows folding the load of the mask with
21678   // the logic op, so it can save (~4 bytes) on code size.
21679   bool IsFakeVector = !VT.isVector() && !IsF128;
21680   MVT LogicVT = VT;
21681   if (IsFakeVector)
21682     LogicVT = (VT == MVT::f64)   ? MVT::v2f64
21683               : (VT == MVT::f32) ? MVT::v4f32
21684                                  : MVT::v8f16;
21685 
21686   unsigned EltBits = VT.getScalarSizeInBits();
21687   // For FABS, mask is 0x7f...; for FNEG, mask is 0x80...
21688   APInt MaskElt = IsFABS ? APInt::getSignedMaxValue(EltBits) :
21689                            APInt::getSignMask(EltBits);
21690   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
21691   SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
21692 
21693   SDValue Op0 = Op.getOperand(0);
21694   bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
21695   unsigned LogicOp = IsFABS  ? X86ISD::FAND :
21696                      IsFNABS ? X86ISD::FOR  :
21697                                X86ISD::FXOR;
21698   SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
21699 
21700   if (VT.isVector() || IsF128)
21701     return DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
21702 
21703   // For the scalar case extend to a 128-bit vector, perform the logic op,
21704   // and extract the scalar result back out.
21705   Operand = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Operand);
21706   SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
21707   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode,
21708                      DAG.getIntPtrConstant(0, dl));
21709 }
21710 
21711 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
21712   SDValue Mag = Op.getOperand(0);
21713   SDValue Sign = Op.getOperand(1);
21714   SDLoc dl(Op);
21715 
21716   // If the sign operand is smaller, extend it first.
21717   MVT VT = Op.getSimpleValueType();
21718   if (Sign.getSimpleValueType().bitsLT(VT))
21719     Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign);
21720 
21721   // And if it is bigger, shrink it first.
21722   if (Sign.getSimpleValueType().bitsGT(VT))
21723     Sign = DAG.getNode(ISD::FP_ROUND, dl, VT, Sign,
21724                        DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
21725 
21726   // At this point the operands and the result should have the same
21727   // type, and that won't be f80 since that is not custom lowered.
21728   bool IsF128 = (VT == MVT::f128);
21729   assert(VT.isFloatingPoint() && VT != MVT::f80 &&
21730          DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
21731          "Unexpected type in LowerFCOPYSIGN");
21732 
21733   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
21734 
21735   // Perform all scalar logic operations as 16-byte vectors because there are no
21736   // scalar FP logic instructions in SSE.
21737   // TODO: This isn't necessary. If we used scalar types, we might avoid some
21738   // unnecessary splats, but we might miss load folding opportunities. Should
21739   // this decision be based on OptimizeForSize?
21740   bool IsFakeVector = !VT.isVector() && !IsF128;
21741   MVT LogicVT = VT;
21742   if (IsFakeVector)
21743     LogicVT = (VT == MVT::f64)   ? MVT::v2f64
21744               : (VT == MVT::f32) ? MVT::v4f32
21745                                  : MVT::v8f16;
21746 
21747   // The mask constants are automatically splatted for vector types.
21748   unsigned EltSizeInBits = VT.getScalarSizeInBits();
21749   SDValue SignMask = DAG.getConstantFP(
21750       APFloat(Sem, APInt::getSignMask(EltSizeInBits)), dl, LogicVT);
21751   SDValue MagMask = DAG.getConstantFP(
21752       APFloat(Sem, APInt::getSignedMaxValue(EltSizeInBits)), dl, LogicVT);
21753 
21754   // First, clear all bits but the sign bit from the second operand (sign).
21755   if (IsFakeVector)
21756     Sign = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Sign);
21757   SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
21758 
21759   // Next, clear the sign bit from the first operand (magnitude).
21760   // TODO: If we had general constant folding for FP logic ops, this check
21761   // wouldn't be necessary.
21762   SDValue MagBits;
21763   if (ConstantFPSDNode *Op0CN = isConstOrConstSplatFP(Mag)) {
21764     APFloat APF = Op0CN->getValueAPF();
21765     APF.clearSign();
21766     MagBits = DAG.getConstantFP(APF, dl, LogicVT);
21767   } else {
21768     // If the magnitude operand wasn't a constant, we need to AND out the sign.
21769     if (IsFakeVector)
21770       Mag = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Mag);
21771     MagBits = DAG.getNode(X86ISD::FAND, dl, LogicVT, Mag, MagMask);
21772   }
21773 
21774   // OR the magnitude value with the sign bit.
21775   SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
21776   return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or,
21777                                           DAG.getIntPtrConstant(0, dl));
21778 }
21779 
21780 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
21781   SDValue N0 = Op.getOperand(0);
21782   SDLoc dl(Op);
21783   MVT VT = Op.getSimpleValueType();
21784 
21785   MVT OpVT = N0.getSimpleValueType();
21786   assert((OpVT == MVT::f32 || OpVT == MVT::f64) &&
21787          "Unexpected type for FGETSIGN");
21788 
21789   // Lower ISD::FGETSIGN to (AND (X86ISD::MOVMSK ...) 1).
21790   MVT VecVT = (OpVT == MVT::f32 ? MVT::v4f32 : MVT::v2f64);
21791   SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0);
21792   Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res);
21793   Res = DAG.getZExtOrTrunc(Res, dl, VT);
21794   Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT));
21795   return Res;
21796 }
21797 
21798 /// Helper for attempting to create a X86ISD::BT node.
21799 static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) {
21800   // If Src is i8, promote it to i32 with any_extend.  There is no i8 BT
21801   // instruction.  Since the shift amount is in-range-or-undefined, we know
21802   // that doing a bittest on the i32 value is ok.  We extend to i32 because
21803   // the encoding for the i16 version is larger than the i32 version.
21804   // Also promote i16 to i32 for performance / code size reason.
21805   if (Src.getValueType().getScalarSizeInBits() < 32)
21806     Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src);
21807 
21808   // No legal type found, give up.
21809   if (!DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
21810     return SDValue();
21811 
21812   // See if we can use the 32-bit instruction instead of the 64-bit one for a
21813   // shorter encoding. Since the former takes the modulo 32 of BitNo and the
21814   // latter takes the modulo 64, this is only valid if the 5th bit of BitNo is
21815   // known to be zero.
21816   if (Src.getValueType() == MVT::i64 &&
21817       DAG.MaskedValueIsZero(BitNo, APInt(BitNo.getValueSizeInBits(), 32)))
21818     Src = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Src);
21819 
21820   // If the operand types disagree, extend the shift amount to match.  Since
21821   // BT ignores high bits (like shifts) we can use anyextend.
21822   if (Src.getValueType() != BitNo.getValueType()) {
21823     // Peek through a mask/modulo operation.
21824     // TODO: DAGCombine fails to do this as it just checks isTruncateFree, but
21825     // we probably need a better IsDesirableToPromoteOp to handle this as well.
21826     if (BitNo.getOpcode() == ISD::AND && BitNo->hasOneUse())
21827       BitNo = DAG.getNode(ISD::AND, DL, Src.getValueType(),
21828                           DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(),
21829                                       BitNo.getOperand(0)),
21830                           DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(),
21831                                       BitNo.getOperand(1)));
21832     else
21833       BitNo = DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), BitNo);
21834   }
21835 
21836   return DAG.getNode(X86ISD::BT, DL, MVT::i32, Src, BitNo);
21837 }
21838 
21839 /// Helper for creating a X86ISD::SETCC node.
21840 static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
21841                         SelectionDAG &DAG) {
21842   return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
21843                      DAG.getTargetConstant(Cond, dl, MVT::i8), EFLAGS);
21844 }
21845 
21846 /// Recursive helper for combineVectorSizedSetCCEquality() to see if we have a
21847 /// recognizable memcmp expansion.
21848 static bool isOrXorXorTree(SDValue X, bool Root = true) {
21849   if (X.getOpcode() == ISD::OR)
21850     return isOrXorXorTree(X.getOperand(0), false) &&
21851            isOrXorXorTree(X.getOperand(1), false);
21852   if (Root)
21853     return false;
21854   return X.getOpcode() == ISD::XOR;
21855 }
21856 
21857 /// Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp
21858 /// expansion.
21859 template <typename F>
21860 static SDValue emitOrXorXorTree(SDValue X, const SDLoc &DL, SelectionDAG &DAG,
21861                                 EVT VecVT, EVT CmpVT, bool HasPT, F SToV) {
21862   SDValue Op0 = X.getOperand(0);
21863   SDValue Op1 = X.getOperand(1);
21864   if (X.getOpcode() == ISD::OR) {
21865     SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV);
21866     SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV);
21867     if (VecVT != CmpVT)
21868       return DAG.getNode(ISD::OR, DL, CmpVT, A, B);
21869     if (HasPT)
21870       return DAG.getNode(ISD::OR, DL, VecVT, A, B);
21871     return DAG.getNode(ISD::AND, DL, CmpVT, A, B);
21872   }
21873   if (X.getOpcode() == ISD::XOR) {
21874     SDValue A = SToV(Op0);
21875     SDValue B = SToV(Op1);
21876     if (VecVT != CmpVT)
21877       return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETNE);
21878     if (HasPT)
21879       return DAG.getNode(ISD::XOR, DL, VecVT, A, B);
21880     return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETEQ);
21881   }
21882   llvm_unreachable("Impossible");
21883 }
21884 
21885 /// Try to map a 128-bit or larger integer comparison to vector instructions
21886 /// before type legalization splits it up into chunks.
21887 static SDValue combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y,
21888                                                ISD::CondCode CC,
21889                                                const SDLoc &DL,
21890                                                SelectionDAG &DAG,
21891                                                const X86Subtarget &Subtarget) {
21892   assert((CC == ISD::SETNE || CC == ISD::SETEQ) && "Bad comparison predicate");
21893 
21894   // We're looking for an oversized integer equality comparison.
21895   EVT OpVT = X.getValueType();
21896   unsigned OpSize = OpVT.getSizeInBits();
21897   if (!OpVT.isScalarInteger() || OpSize < 128)
21898     return SDValue();
21899 
21900   // Ignore a comparison with zero because that gets special treatment in
21901   // EmitTest(). But make an exception for the special case of a pair of
21902   // logically-combined vector-sized operands compared to zero. This pattern may
21903   // be generated by the memcmp expansion pass with oversized integer compares
21904   // (see PR33325).
21905   bool IsOrXorXorTreeCCZero = isNullConstant(Y) && isOrXorXorTree(X);
21906   if (isNullConstant(Y) && !IsOrXorXorTreeCCZero)
21907     return SDValue();
21908 
21909   // Don't perform this combine if constructing the vector will be expensive.
21910   auto IsVectorBitCastCheap = [](SDValue X) {
21911     X = peekThroughBitcasts(X);
21912     return isa<ConstantSDNode>(X) || X.getValueType().isVector() ||
21913            X.getOpcode() == ISD::LOAD;
21914   };
21915   if ((!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y)) &&
21916       !IsOrXorXorTreeCCZero)
21917     return SDValue();
21918 
21919   // Use XOR (plus OR) and PTEST after SSE4.1 for 128/256-bit operands.
21920   // Use PCMPNEQ (plus OR) and KORTEST for 512-bit operands.
21921   // Otherwise use PCMPEQ (plus AND) and mask testing.
21922   bool NoImplicitFloatOps =
21923       DAG.getMachineFunction().getFunction().hasFnAttribute(
21924           Attribute::NoImplicitFloat);
21925   if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
21926       ((OpSize == 128 && Subtarget.hasSSE2()) ||
21927        (OpSize == 256 && Subtarget.hasAVX()) ||
21928        (OpSize == 512 && Subtarget.useAVX512Regs()))) {
21929     bool HasPT = Subtarget.hasSSE41();
21930 
21931     // PTEST and MOVMSK are slow on Knights Landing and Knights Mill and widened
21932     // vector registers are essentially free. (Technically, widening registers
21933     // prevents load folding, but the tradeoff is worth it.)
21934     bool PreferKOT = Subtarget.preferMaskRegisters();
21935     bool NeedZExt = PreferKOT && !Subtarget.hasVLX() && OpSize != 512;
21936 
21937     EVT VecVT = MVT::v16i8;
21938     EVT CmpVT = PreferKOT ? MVT::v16i1 : VecVT;
21939     if (OpSize == 256) {
21940       VecVT = MVT::v32i8;
21941       CmpVT = PreferKOT ? MVT::v32i1 : VecVT;
21942     }
21943     EVT CastVT = VecVT;
21944     bool NeedsAVX512FCast = false;
21945     if (OpSize == 512 || NeedZExt) {
21946       if (Subtarget.hasBWI()) {
21947         VecVT = MVT::v64i8;
21948         CmpVT = MVT::v64i1;
21949         if (OpSize == 512)
21950           CastVT = VecVT;
21951       } else {
21952         VecVT = MVT::v16i32;
21953         CmpVT = MVT::v16i1;
21954         CastVT = OpSize == 512   ? VecVT
21955                  : OpSize == 256 ? MVT::v8i32
21956                                  : MVT::v4i32;
21957         NeedsAVX512FCast = true;
21958       }
21959     }
21960 
21961     auto ScalarToVector = [&](SDValue X) -> SDValue {
21962       bool TmpZext = false;
21963       EVT TmpCastVT = CastVT;
21964       if (X.getOpcode() == ISD::ZERO_EXTEND) {
21965         SDValue OrigX = X.getOperand(0);
21966         unsigned OrigSize = OrigX.getScalarValueSizeInBits();
21967         if (OrigSize < OpSize) {
21968           if (OrigSize == 128) {
21969             TmpCastVT = NeedsAVX512FCast ? MVT::v4i32 : MVT::v16i8;
21970             X = OrigX;
21971             TmpZext = true;
21972           } else if (OrigSize == 256) {
21973             TmpCastVT = NeedsAVX512FCast ? MVT::v8i32 : MVT::v32i8;
21974             X = OrigX;
21975             TmpZext = true;
21976           }
21977         }
21978       }
21979       X = DAG.getBitcast(TmpCastVT, X);
21980       if (!NeedZExt && !TmpZext)
21981         return X;
21982       return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT,
21983                          DAG.getConstant(0, DL, VecVT), X,
21984                          DAG.getVectorIdxConstant(0, DL));
21985     };
21986 
21987     SDValue Cmp;
21988     if (IsOrXorXorTreeCCZero) {
21989       // This is a bitwise-combined equality comparison of 2 pairs of vectors:
21990       // setcc i128 (or (xor A, B), (xor C, D)), 0, eq|ne
21991       // Use 2 vector equality compares and 'and' the results before doing a
21992       // MOVMSK.
21993       Cmp = emitOrXorXorTree(X, DL, DAG, VecVT, CmpVT, HasPT, ScalarToVector);
21994     } else {
21995       SDValue VecX = ScalarToVector(X);
21996       SDValue VecY = ScalarToVector(Y);
21997       if (VecVT != CmpVT) {
21998         Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
21999       } else if (HasPT) {
22000         Cmp = DAG.getNode(ISD::XOR, DL, VecVT, VecX, VecY);
22001       } else {
22002         Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETEQ);
22003       }
22004     }
22005     // AVX512 should emit a setcc that will lower to kortest.
22006     if (VecVT != CmpVT) {
22007       EVT KRegVT = CmpVT == MVT::v64i1   ? MVT::i64
22008                    : CmpVT == MVT::v32i1 ? MVT::i32
22009                                          : MVT::i16;
22010       return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp),
22011                           DAG.getConstant(0, DL, KRegVT), CC);
22012     }
22013     if (HasPT) {
22014       SDValue BCCmp =
22015           DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64, Cmp);
22016       SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
22017       X86::CondCode X86CC = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
22018       SDValue X86SetCC = getSETCC(X86CC, PT, DL, DAG);
22019       return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0));
22020     }
22021     // If all bytes match (bitmask is 0x(FFFF)FFFF), that's equality.
22022     // setcc i128 X, Y, eq --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, eq
22023     // setcc i128 X, Y, ne --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, ne
22024     assert(Cmp.getValueType() == MVT::v16i8 &&
22025            "Non 128-bit vector on pre-SSE41 target");
22026     SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
22027     SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32);
22028     return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC);
22029   }
22030 
22031   return SDValue();
22032 }
22033 
22034 /// Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...))
22035 /// style scalarized (associative) reduction patterns. Partial reductions
22036 /// are supported when the pointer SrcMask is non-null.
22037 /// TODO - move this to SelectionDAG?
22038 static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
22039                                  SmallVectorImpl<SDValue> &SrcOps,
22040                                  SmallVectorImpl<APInt> *SrcMask = nullptr) {
22041   SmallVector<SDValue, 8> Opnds;
22042   DenseMap<SDValue, APInt> SrcOpMap;
22043   EVT VT = MVT::Other;
22044 
22045   // Recognize a special case where a vector is casted into wide integer to
22046   // test all 0s.
22047   assert(Op.getOpcode() == unsigned(BinOp) &&
22048          "Unexpected bit reduction opcode");
22049   Opnds.push_back(Op.getOperand(0));
22050   Opnds.push_back(Op.getOperand(1));
22051 
22052   for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) {
22053     SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
22054     // BFS traverse all BinOp operands.
22055     if (I->getOpcode() == unsigned(BinOp)) {
22056       Opnds.push_back(I->getOperand(0));
22057       Opnds.push_back(I->getOperand(1));
22058       // Re-evaluate the number of nodes to be traversed.
22059       e += 2; // 2 more nodes (LHS and RHS) are pushed.
22060       continue;
22061     }
22062 
22063     // Quit if a non-EXTRACT_VECTOR_ELT
22064     if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
22065       return false;
22066 
22067     // Quit if without a constant index.
22068     auto *Idx = dyn_cast<ConstantSDNode>(I->getOperand(1));
22069     if (!Idx)
22070       return false;
22071 
22072     SDValue Src = I->getOperand(0);
22073     DenseMap<SDValue, APInt>::iterator M = SrcOpMap.find(Src);
22074     if (M == SrcOpMap.end()) {
22075       VT = Src.getValueType();
22076       // Quit if not the same type.
22077       if (!SrcOpMap.empty() && VT != SrcOpMap.begin()->first.getValueType())
22078         return false;
22079       unsigned NumElts = VT.getVectorNumElements();
22080       APInt EltCount = APInt::getZero(NumElts);
22081       M = SrcOpMap.insert(std::make_pair(Src, EltCount)).first;
22082       SrcOps.push_back(Src);
22083     }
22084 
22085     // Quit if element already used.
22086     unsigned CIdx = Idx->getZExtValue();
22087     if (M->second[CIdx])
22088       return false;
22089     M->second.setBit(CIdx);
22090   }
22091 
22092   if (SrcMask) {
22093     // Collect the source partial masks.
22094     for (SDValue &SrcOp : SrcOps)
22095       SrcMask->push_back(SrcOpMap[SrcOp]);
22096   } else {
22097     // Quit if not all elements are used.
22098     for (const auto &I : SrcOpMap)
22099       if (!I.second.isAllOnes())
22100         return false;
22101   }
22102 
22103   return true;
22104 }
22105 
22106 // Helper function for comparing all bits of two vectors.
22107 static SDValue LowerVectorAllEqual(const SDLoc &DL, SDValue LHS, SDValue RHS,
22108                                    ISD::CondCode CC, const APInt &OriginalMask,
22109                                    const X86Subtarget &Subtarget,
22110                                    SelectionDAG &DAG, X86::CondCode &X86CC) {
22111   EVT VT = LHS.getValueType();
22112   unsigned ScalarSize = VT.getScalarSizeInBits();
22113   if (OriginalMask.getBitWidth() != ScalarSize) {
22114     assert(ScalarSize == 1 && "Element Mask vs Vector bitwidth mismatch");
22115     return SDValue();
22116   }
22117 
22118   // Quit if not convertable to legal scalar or 128/256-bit vector.
22119   if (!llvm::has_single_bit<uint32_t>(VT.getSizeInBits()))
22120     return SDValue();
22121 
22122   // FCMP may use ISD::SETNE when nnan - early out if we manage to get here.
22123   if (VT.isFloatingPoint())
22124     return SDValue();
22125 
22126   assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
22127   X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE);
22128 
22129   APInt Mask = OriginalMask;
22130 
22131   auto MaskBits = [&](SDValue Src) {
22132     if (Mask.isAllOnes())
22133       return Src;
22134     EVT SrcVT = Src.getValueType();
22135     SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
22136     return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue);
22137   };
22138 
22139   // For sub-128-bit vector, cast to (legal) integer and compare with zero.
22140   if (VT.getSizeInBits() < 128) {
22141     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
22142     if (!DAG.getTargetLoweringInfo().isTypeLegal(IntVT)) {
22143       if (IntVT != MVT::i64)
22144         return SDValue();
22145       auto SplitLHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(LHS)), DL,
22146                                       MVT::i32, MVT::i32);
22147       auto SplitRHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(RHS)), DL,
22148                                       MVT::i32, MVT::i32);
22149       SDValue Lo =
22150           DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.first, SplitRHS.first);
22151       SDValue Hi =
22152           DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.second, SplitRHS.second);
22153       return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22154                          DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi),
22155                          DAG.getConstant(0, DL, MVT::i32));
22156     }
22157     return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22158                        DAG.getBitcast(IntVT, MaskBits(LHS)),
22159                        DAG.getBitcast(IntVT, MaskBits(RHS)));
22160   }
22161 
22162   // Without PTEST, a masked v2i64 or-reduction is not faster than
22163   // scalarization.
22164   bool UseKORTEST = Subtarget.useAVX512Regs();
22165   bool UsePTEST = Subtarget.hasSSE41();
22166   if (!UsePTEST && !Mask.isAllOnes() && ScalarSize > 32)
22167     return SDValue();
22168 
22169   // Split down to 128/256/512-bit vector.
22170   unsigned TestSize = UseKORTEST ? 512 : (Subtarget.hasAVX() ? 256 : 128);
22171 
22172   // If the input vector has vector elements wider than the target test size,
22173   // then cast to <X x i64> so it will safely split.
22174   if (ScalarSize > TestSize) {
22175     if (!Mask.isAllOnes())
22176       return SDValue();
22177     VT = EVT::getVectorVT(*DAG.getContext(), MVT::i64, VT.getSizeInBits() / 64);
22178     LHS = DAG.getBitcast(VT, LHS);
22179     RHS = DAG.getBitcast(VT, RHS);
22180     Mask = APInt::getAllOnes(64);
22181   }
22182 
22183   if (VT.getSizeInBits() > TestSize) {
22184     KnownBits KnownRHS = DAG.computeKnownBits(RHS);
22185     if (KnownRHS.isConstant() && KnownRHS.getConstant() == Mask) {
22186       // If ICMP(AND(LHS,MASK),MASK) - reduce using AND splits.
22187       while (VT.getSizeInBits() > TestSize) {
22188         auto Split = DAG.SplitVector(LHS, DL);
22189         VT = Split.first.getValueType();
22190         LHS = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second);
22191       }
22192       RHS = DAG.getAllOnesConstant(DL, VT);
22193     } else if (!UsePTEST && !KnownRHS.isZero()) {
22194       // MOVMSK Special Case:
22195       // ALLOF(CMPEQ(X,Y)) -> AND(CMPEQ(X[0],Y[0]),CMPEQ(X[1],Y[1]),....)
22196       MVT SVT = ScalarSize >= 32 ? MVT::i32 : MVT::i8;
22197       VT = MVT::getVectorVT(SVT, VT.getSizeInBits() / SVT.getSizeInBits());
22198       LHS = DAG.getBitcast(VT, MaskBits(LHS));
22199       RHS = DAG.getBitcast(VT, MaskBits(RHS));
22200       EVT BoolVT = VT.changeVectorElementType(MVT::i1);
22201       SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETEQ);
22202       V = DAG.getSExtOrTrunc(V, DL, VT);
22203       while (VT.getSizeInBits() > TestSize) {
22204         auto Split = DAG.SplitVector(V, DL);
22205         VT = Split.first.getValueType();
22206         V = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second);
22207       }
22208       V = DAG.getNOT(DL, V, VT);
22209       V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
22210       return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
22211                          DAG.getConstant(0, DL, MVT::i32));
22212     } else {
22213       // Convert to a ICMP_EQ(XOR(LHS,RHS),0) pattern.
22214       SDValue V = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
22215       while (VT.getSizeInBits() > TestSize) {
22216         auto Split = DAG.SplitVector(V, DL);
22217         VT = Split.first.getValueType();
22218         V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second);
22219       }
22220       LHS = V;
22221       RHS = DAG.getConstant(0, DL, VT);
22222     }
22223   }
22224 
22225   if (UseKORTEST && VT.is512BitVector()) {
22226     MVT TestVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
22227     MVT BoolVT = TestVT.changeVectorElementType(MVT::i1);
22228     LHS = DAG.getBitcast(TestVT, MaskBits(LHS));
22229     RHS = DAG.getBitcast(TestVT, MaskBits(RHS));
22230     SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETNE);
22231     return DAG.getNode(X86ISD::KORTEST, DL, MVT::i32, V, V);
22232   }
22233 
22234   if (UsePTEST) {
22235     MVT TestVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
22236     LHS = DAG.getBitcast(TestVT, MaskBits(LHS));
22237     RHS = DAG.getBitcast(TestVT, MaskBits(RHS));
22238     SDValue V = DAG.getNode(ISD::XOR, DL, TestVT, LHS, RHS);
22239     return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V);
22240   }
22241 
22242   assert(VT.getSizeInBits() == 128 && "Failure to split to 128-bits");
22243   MVT MaskVT = ScalarSize >= 32 ? MVT::v4i32 : MVT::v16i8;
22244   LHS = DAG.getBitcast(MaskVT, MaskBits(LHS));
22245   RHS = DAG.getBitcast(MaskVT, MaskBits(RHS));
22246   SDValue V = DAG.getNode(X86ISD::PCMPEQ, DL, MaskVT, LHS, RHS);
22247   V = DAG.getNOT(DL, V, MaskVT);
22248   V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
22249   return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
22250                      DAG.getConstant(0, DL, MVT::i32));
22251 }
22252 
22253 // Check whether an AND/OR'd reduction tree is PTEST-able, or if we can fallback
22254 // to CMP(MOVMSK(PCMPEQB(X,Y))).
22255 static SDValue MatchVectorAllEqualTest(SDValue LHS, SDValue RHS,
22256                                        ISD::CondCode CC, const SDLoc &DL,
22257                                        const X86Subtarget &Subtarget,
22258                                        SelectionDAG &DAG,
22259                                        X86::CondCode &X86CC) {
22260   assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
22261 
22262   bool CmpNull = isNullConstant(RHS);
22263   bool CmpAllOnes = isAllOnesConstant(RHS);
22264   if (!CmpNull && !CmpAllOnes)
22265     return SDValue();
22266 
22267   SDValue Op = LHS;
22268   if (!Subtarget.hasSSE2() || !Op->hasOneUse())
22269     return SDValue();
22270 
22271   // Check whether we're masking/truncating an OR-reduction result, in which
22272   // case track the masked bits.
22273   // TODO: Add CmpAllOnes support.
22274   APInt Mask = APInt::getAllOnes(Op.getScalarValueSizeInBits());
22275   if (CmpNull) {
22276     switch (Op.getOpcode()) {
22277     case ISD::TRUNCATE: {
22278       SDValue Src = Op.getOperand(0);
22279       Mask = APInt::getLowBitsSet(Src.getScalarValueSizeInBits(),
22280                                   Op.getScalarValueSizeInBits());
22281       Op = Src;
22282       break;
22283     }
22284     case ISD::AND: {
22285       if (auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
22286         Mask = Cst->getAPIntValue();
22287         Op = Op.getOperand(0);
22288       }
22289       break;
22290     }
22291     }
22292   }
22293 
22294   ISD::NodeType LogicOp = CmpNull ? ISD::OR : ISD::AND;
22295 
22296   // Match icmp(or(extract(X,0),extract(X,1)),0) anyof reduction patterns.
22297   // Match icmp(and(extract(X,0),extract(X,1)),-1) allof reduction patterns.
22298   SmallVector<SDValue, 8> VecIns;
22299   if (Op.getOpcode() == LogicOp && matchScalarReduction(Op, LogicOp, VecIns)) {
22300     EVT VT = VecIns[0].getValueType();
22301     assert(llvm::all_of(VecIns,
22302                         [VT](SDValue V) { return VT == V.getValueType(); }) &&
22303            "Reduction source vector mismatch");
22304 
22305     // Quit if not splittable to scalar/128/256/512-bit vector.
22306     if (!llvm::has_single_bit<uint32_t>(VT.getSizeInBits()))
22307       return SDValue();
22308 
22309     // If more than one full vector is evaluated, AND/OR them first before
22310     // PTEST.
22311     for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1;
22312          Slot += 2, e += 1) {
22313       // Each iteration will AND/OR 2 nodes and append the result until there is
22314       // only 1 node left, i.e. the final value of all vectors.
22315       SDValue LHS = VecIns[Slot];
22316       SDValue RHS = VecIns[Slot + 1];
22317       VecIns.push_back(DAG.getNode(LogicOp, DL, VT, LHS, RHS));
22318     }
22319 
22320     return LowerVectorAllEqual(DL, VecIns.back(),
22321                                CmpNull ? DAG.getConstant(0, DL, VT)
22322                                        : DAG.getAllOnesConstant(DL, VT),
22323                                CC, Mask, Subtarget, DAG, X86CC);
22324   }
22325 
22326   // Match icmp(reduce_or(X),0) anyof reduction patterns.
22327   // Match icmp(reduce_and(X),-1) allof reduction patterns.
22328   if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22329     ISD::NodeType BinOp;
22330     if (SDValue Match =
22331             DAG.matchBinOpReduction(Op.getNode(), BinOp, {LogicOp})) {
22332       EVT MatchVT = Match.getValueType();
22333       return LowerVectorAllEqual(DL, Match,
22334                                  CmpNull ? DAG.getConstant(0, DL, MatchVT)
22335                                          : DAG.getAllOnesConstant(DL, MatchVT),
22336                                  CC, Mask, Subtarget, DAG, X86CC);
22337     }
22338   }
22339 
22340   if (Mask.isAllOnes()) {
22341     assert(!Op.getValueType().isVector() &&
22342            "Illegal vector type for reduction pattern");
22343     SDValue Src = peekThroughBitcasts(Op);
22344     if (Src.getValueType().isFixedLengthVector() &&
22345         Src.getValueType().getScalarType() == MVT::i1) {
22346       // Match icmp(bitcast(icmp_ne(X,Y)),0) reduction patterns.
22347       // Match icmp(bitcast(icmp_eq(X,Y)),-1) reduction patterns.
22348       if (Src.getOpcode() == ISD::SETCC) {
22349         SDValue LHS = Src.getOperand(0);
22350         SDValue RHS = Src.getOperand(1);
22351         EVT LHSVT = LHS.getValueType();
22352         ISD::CondCode SrcCC = cast<CondCodeSDNode>(Src.getOperand(2))->get();
22353         if (SrcCC == (CmpNull ? ISD::SETNE : ISD::SETEQ) &&
22354             llvm::has_single_bit<uint32_t>(LHSVT.getSizeInBits())) {
22355           APInt SrcMask = APInt::getAllOnes(LHSVT.getScalarSizeInBits());
22356           return LowerVectorAllEqual(DL, LHS, RHS, CC, SrcMask, Subtarget, DAG,
22357                                      X86CC);
22358         }
22359       }
22360       // Match icmp(bitcast(vXi1 trunc(Y)),0) reduction patterns.
22361       // Match icmp(bitcast(vXi1 trunc(Y)),-1) reduction patterns.
22362       // Peek through truncation, mask the LSB and compare against zero/LSB.
22363       if (Src.getOpcode() == ISD::TRUNCATE) {
22364         SDValue Inner = Src.getOperand(0);
22365         EVT InnerVT = Inner.getValueType();
22366         if (llvm::has_single_bit<uint32_t>(InnerVT.getSizeInBits())) {
22367           unsigned BW = InnerVT.getScalarSizeInBits();
22368           APInt SrcMask = APInt(BW, 1);
22369           APInt Cmp = CmpNull ? APInt::getZero(BW) : SrcMask;
22370           return LowerVectorAllEqual(DL, Inner,
22371                                      DAG.getConstant(Cmp, DL, InnerVT), CC,
22372                                      SrcMask, Subtarget, DAG, X86CC);
22373         }
22374       }
22375     }
22376   }
22377 
22378   return SDValue();
22379 }
22380 
22381 /// return true if \c Op has a use that doesn't just read flags.
22382 static bool hasNonFlagsUse(SDValue Op) {
22383   for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE;
22384        ++UI) {
22385     SDNode *User = *UI;
22386     unsigned UOpNo = UI.getOperandNo();
22387     if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) {
22388       // Look pass truncate.
22389       UOpNo = User->use_begin().getOperandNo();
22390       User = *User->use_begin();
22391     }
22392 
22393     if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC &&
22394         !(User->getOpcode() == ISD::SELECT && UOpNo == 0))
22395       return true;
22396   }
22397   return false;
22398 }
22399 
22400 // Transform to an x86-specific ALU node with flags if there is a chance of
22401 // using an RMW op or only the flags are used. Otherwise, leave
22402 // the node alone and emit a 'cmp' or 'test' instruction.
22403 static bool isProfitableToUseFlagOp(SDValue Op) {
22404   for (SDNode *U : Op->uses())
22405     if (U->getOpcode() != ISD::CopyToReg &&
22406         U->getOpcode() != ISD::SETCC &&
22407         U->getOpcode() != ISD::STORE)
22408       return false;
22409 
22410   return true;
22411 }
22412 
22413 /// Emit nodes that will be selected as "test Op0,Op0", or something
22414 /// equivalent.
22415 static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
22416                         SelectionDAG &DAG, const X86Subtarget &Subtarget) {
22417   // CF and OF aren't always set the way we want. Determine which
22418   // of these we need.
22419   bool NeedCF = false;
22420   bool NeedOF = false;
22421   switch (X86CC) {
22422   default: break;
22423   case X86::COND_A: case X86::COND_AE:
22424   case X86::COND_B: case X86::COND_BE:
22425     NeedCF = true;
22426     break;
22427   case X86::COND_G: case X86::COND_GE:
22428   case X86::COND_L: case X86::COND_LE:
22429   case X86::COND_O: case X86::COND_NO: {
22430     // Check if we really need to set the
22431     // Overflow flag. If NoSignedWrap is present
22432     // that is not actually needed.
22433     switch (Op->getOpcode()) {
22434     case ISD::ADD:
22435     case ISD::SUB:
22436     case ISD::MUL:
22437     case ISD::SHL:
22438       if (Op.getNode()->getFlags().hasNoSignedWrap())
22439         break;
22440       [[fallthrough]];
22441     default:
22442       NeedOF = true;
22443       break;
22444     }
22445     break;
22446   }
22447   }
22448   // See if we can use the EFLAGS value from the operand instead of
22449   // doing a separate TEST. TEST always sets OF and CF to 0, so unless
22450   // we prove that the arithmetic won't overflow, we can't use OF or CF.
22451   if (Op.getResNo() != 0 || NeedOF || NeedCF) {
22452     // Emit a CMP with 0, which is the TEST pattern.
22453     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
22454                        DAG.getConstant(0, dl, Op.getValueType()));
22455   }
22456   unsigned Opcode = 0;
22457   unsigned NumOperands = 0;
22458 
22459   SDValue ArithOp = Op;
22460 
22461   // NOTICE: In the code below we use ArithOp to hold the arithmetic operation
22462   // which may be the result of a CAST.  We use the variable 'Op', which is the
22463   // non-casted variable when we check for possible users.
22464   switch (ArithOp.getOpcode()) {
22465   case ISD::AND:
22466     // If the primary 'and' result isn't used, don't bother using X86ISD::AND,
22467     // because a TEST instruction will be better.
22468     if (!hasNonFlagsUse(Op))
22469       break;
22470 
22471     [[fallthrough]];
22472   case ISD::ADD:
22473   case ISD::SUB:
22474   case ISD::OR:
22475   case ISD::XOR:
22476     if (!isProfitableToUseFlagOp(Op))
22477       break;
22478 
22479     // Otherwise use a regular EFLAGS-setting instruction.
22480     switch (ArithOp.getOpcode()) {
22481     default: llvm_unreachable("unexpected operator!");
22482     case ISD::ADD: Opcode = X86ISD::ADD; break;
22483     case ISD::SUB: Opcode = X86ISD::SUB; break;
22484     case ISD::XOR: Opcode = X86ISD::XOR; break;
22485     case ISD::AND: Opcode = X86ISD::AND; break;
22486     case ISD::OR:  Opcode = X86ISD::OR;  break;
22487     }
22488 
22489     NumOperands = 2;
22490     break;
22491   case X86ISD::ADD:
22492   case X86ISD::SUB:
22493   case X86ISD::OR:
22494   case X86ISD::XOR:
22495   case X86ISD::AND:
22496     return SDValue(Op.getNode(), 1);
22497   case ISD::SSUBO:
22498   case ISD::USUBO: {
22499     // /USUBO/SSUBO will become a X86ISD::SUB and we can use its Z flag.
22500     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
22501     return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0),
22502                        Op->getOperand(1)).getValue(1);
22503   }
22504   default:
22505     break;
22506   }
22507 
22508   if (Opcode == 0) {
22509     // Emit a CMP with 0, which is the TEST pattern.
22510     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
22511                        DAG.getConstant(0, dl, Op.getValueType()));
22512   }
22513   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
22514   SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
22515 
22516   SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
22517   DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New);
22518   return SDValue(New.getNode(), 1);
22519 }
22520 
22521 /// Emit nodes that will be selected as "cmp Op0,Op1", or something
22522 /// equivalent.
22523 static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
22524                        const SDLoc &dl, SelectionDAG &DAG,
22525                        const X86Subtarget &Subtarget) {
22526   if (isNullConstant(Op1))
22527     return EmitTest(Op0, X86CC, dl, DAG, Subtarget);
22528 
22529   EVT CmpVT = Op0.getValueType();
22530 
22531   assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 ||
22532           CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!");
22533 
22534   // Only promote the compare up to I32 if it is a 16 bit operation
22535   // with an immediate.  16 bit immediates are to be avoided.
22536   if (CmpVT == MVT::i16 && !Subtarget.isAtom() &&
22537       !DAG.getMachineFunction().getFunction().hasMinSize()) {
22538     ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0);
22539     ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1);
22540     // Don't do this if the immediate can fit in 8-bits.
22541     if ((COp0 && !COp0->getAPIntValue().isSignedIntN(8)) ||
22542         (COp1 && !COp1->getAPIntValue().isSignedIntN(8))) {
22543       unsigned ExtendOp =
22544           isX86CCSigned(X86CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
22545       if (X86CC == X86::COND_E || X86CC == X86::COND_NE) {
22546         // For equality comparisons try to use SIGN_EXTEND if the input was
22547         // truncate from something with enough sign bits.
22548         if (Op0.getOpcode() == ISD::TRUNCATE) {
22549           if (DAG.ComputeMaxSignificantBits(Op0.getOperand(0)) <= 16)
22550             ExtendOp = ISD::SIGN_EXTEND;
22551         } else if (Op1.getOpcode() == ISD::TRUNCATE) {
22552           if (DAG.ComputeMaxSignificantBits(Op1.getOperand(0)) <= 16)
22553             ExtendOp = ISD::SIGN_EXTEND;
22554         }
22555       }
22556 
22557       CmpVT = MVT::i32;
22558       Op0 = DAG.getNode(ExtendOp, dl, CmpVT, Op0);
22559       Op1 = DAG.getNode(ExtendOp, dl, CmpVT, Op1);
22560     }
22561   }
22562 
22563   // Try to shrink i64 compares if the input has enough zero bits.
22564   // FIXME: Do this for non-constant compares for constant on LHS?
22565   if (CmpVT == MVT::i64 && isa<ConstantSDNode>(Op1) && !isX86CCSigned(X86CC) &&
22566       Op0.hasOneUse() && // Hacky way to not break CSE opportunities with sub.
22567       Op1->getAsAPIntVal().getActiveBits() <= 32 &&
22568       DAG.MaskedValueIsZero(Op0, APInt::getHighBitsSet(64, 32))) {
22569     CmpVT = MVT::i32;
22570     Op0 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op0);
22571     Op1 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op1);
22572   }
22573 
22574   // 0-x == y --> x+y == 0
22575   // 0-x != y --> x+y != 0
22576   if (Op0.getOpcode() == ISD::SUB && isNullConstant(Op0.getOperand(0)) &&
22577       Op0.hasOneUse() && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
22578     SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22579     SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(1), Op1);
22580     return Add.getValue(1);
22581   }
22582 
22583   // x == 0-y --> x+y == 0
22584   // x != 0-y --> x+y != 0
22585   if (Op1.getOpcode() == ISD::SUB && isNullConstant(Op1.getOperand(0)) &&
22586       Op1.hasOneUse() && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
22587     SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22588     SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0, Op1.getOperand(1));
22589     return Add.getValue(1);
22590   }
22591 
22592   // Use SUB instead of CMP to enable CSE between SUB and CMP.
22593   SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22594   SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
22595   return Sub.getValue(1);
22596 }
22597 
22598 bool X86TargetLowering::isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond,
22599                                                           EVT VT) const {
22600   return !VT.isVector() || Cond != ISD::CondCode::SETEQ;
22601 }
22602 
22603 bool X86TargetLowering::optimizeFMulOrFDivAsShiftAddBitcast(
22604     SDNode *N, SDValue, SDValue IntPow2) const {
22605   if (N->getOpcode() == ISD::FDIV)
22606     return true;
22607 
22608   EVT FPVT = N->getValueType(0);
22609   EVT IntVT = IntPow2.getValueType();
22610 
22611   // This indicates a non-free bitcast.
22612   // TODO: This is probably overly conservative as we will need to scale the
22613   // integer vector anyways for the int->fp cast.
22614   if (FPVT.isVector() &&
22615       FPVT.getScalarSizeInBits() != IntVT.getScalarSizeInBits())
22616     return false;
22617 
22618   return true;
22619 }
22620 
22621 /// Check if replacement of SQRT with RSQRT should be disabled.
22622 bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
22623   EVT VT = Op.getValueType();
22624 
22625   // We don't need to replace SQRT with RSQRT for half type.
22626   if (VT.getScalarType() == MVT::f16)
22627     return true;
22628 
22629   // We never want to use both SQRT and RSQRT instructions for the same input.
22630   if (DAG.doesNodeExist(X86ISD::FRSQRT, DAG.getVTList(VT), Op))
22631     return false;
22632 
22633   if (VT.isVector())
22634     return Subtarget.hasFastVectorFSQRT();
22635   return Subtarget.hasFastScalarFSQRT();
22636 }
22637 
22638 /// The minimum architected relative accuracy is 2^-12. We need one
22639 /// Newton-Raphson step to have a good float result (24 bits of precision).
22640 SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
22641                                            SelectionDAG &DAG, int Enabled,
22642                                            int &RefinementSteps,
22643                                            bool &UseOneConstNR,
22644                                            bool Reciprocal) const {
22645   SDLoc DL(Op);
22646   EVT VT = Op.getValueType();
22647 
22648   // SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps.
22649   // It is likely not profitable to do this for f64 because a double-precision
22650   // rsqrt estimate with refinement on x86 prior to FMA requires at least 16
22651   // instructions: convert to single, rsqrtss, convert back to double, refine
22652   // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
22653   // along with FMA, this could be a throughput win.
22654   // TODO: SQRT requires SSE2 to prevent the introduction of an illegal v4i32
22655   // after legalize types.
22656   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
22657       (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) ||
22658       (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) ||
22659       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
22660       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
22661     if (RefinementSteps == ReciprocalEstimate::Unspecified)
22662       RefinementSteps = 1;
22663 
22664     UseOneConstNR = false;
22665     // There is no FSQRT for 512-bits, but there is RSQRT14.
22666     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;
22667     SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op);
22668     if (RefinementSteps == 0 && !Reciprocal)
22669       Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate);
22670     return Estimate;
22671   }
22672 
22673   if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&
22674       Subtarget.hasFP16()) {
22675     assert(Reciprocal && "Don't replace SQRT with RSQRT for half type");
22676     if (RefinementSteps == ReciprocalEstimate::Unspecified)
22677       RefinementSteps = 0;
22678 
22679     if (VT == MVT::f16) {
22680       SDValue Zero = DAG.getIntPtrConstant(0, DL);
22681       SDValue Undef = DAG.getUNDEF(MVT::v8f16);
22682       Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
22683       Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op);
22684       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
22685     }
22686 
22687     return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op);
22688   }
22689   return SDValue();
22690 }
22691 
22692 /// The minimum architected relative accuracy is 2^-12. We need one
22693 /// Newton-Raphson step to have a good float result (24 bits of precision).
22694 SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
22695                                             int Enabled,
22696                                             int &RefinementSteps) const {
22697   SDLoc DL(Op);
22698   EVT VT = Op.getValueType();
22699 
22700   // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
22701   // It is likely not profitable to do this for f64 because a double-precision
22702   // reciprocal estimate with refinement on x86 prior to FMA requires
22703   // 15 instructions: convert to single, rcpss, convert back to double, refine
22704   // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
22705   // along with FMA, this could be a throughput win.
22706 
22707   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
22708       (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
22709       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
22710       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
22711     // Enable estimate codegen with 1 refinement step for vector division.
22712     // Scalar division estimates are disabled because they break too much
22713     // real-world code. These defaults are intended to match GCC behavior.
22714     if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)
22715       return SDValue();
22716 
22717     if (RefinementSteps == ReciprocalEstimate::Unspecified)
22718       RefinementSteps = 1;
22719 
22720     // There is no FSQRT for 512-bits, but there is RCP14.
22721     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;
22722     return DAG.getNode(Opcode, DL, VT, Op);
22723   }
22724 
22725   if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&
22726       Subtarget.hasFP16()) {
22727     if (RefinementSteps == ReciprocalEstimate::Unspecified)
22728       RefinementSteps = 0;
22729 
22730     if (VT == MVT::f16) {
22731       SDValue Zero = DAG.getIntPtrConstant(0, DL);
22732       SDValue Undef = DAG.getUNDEF(MVT::v8f16);
22733       Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
22734       Op = DAG.getNode(X86ISD::RCP14S, DL, MVT::v8f16, Undef, Op);
22735       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
22736     }
22737 
22738     return DAG.getNode(X86ISD::RCP14, DL, VT, Op);
22739   }
22740   return SDValue();
22741 }
22742 
22743 /// If we have at least two divisions that use the same divisor, convert to
22744 /// multiplication by a reciprocal. This may need to be adjusted for a given
22745 /// CPU if a division's cost is not at least twice the cost of a multiplication.
22746 /// This is because we still need one division to calculate the reciprocal and
22747 /// then we need two multiplies by that reciprocal as replacements for the
22748 /// original divisions.
22749 unsigned X86TargetLowering::combineRepeatedFPDivisors() const {
22750   return 2;
22751 }
22752 
22753 SDValue
22754 X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
22755                                  SelectionDAG &DAG,
22756                                  SmallVectorImpl<SDNode *> &Created) const {
22757   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
22758   if (isIntDivCheap(N->getValueType(0), Attr))
22759     return SDValue(N,0); // Lower SDIV as SDIV
22760 
22761   assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
22762          "Unexpected divisor!");
22763 
22764   // Only perform this transform if CMOV is supported otherwise the select
22765   // below will become a branch.
22766   if (!Subtarget.canUseCMOV())
22767     return SDValue();
22768 
22769   // fold (sdiv X, pow2)
22770   EVT VT = N->getValueType(0);
22771   // FIXME: Support i8.
22772   if (VT != MVT::i16 && VT != MVT::i32 &&
22773       !(Subtarget.is64Bit() && VT == MVT::i64))
22774     return SDValue();
22775 
22776   // If the divisor is 2 or -2, the default expansion is better.
22777   if (Divisor == 2 ||
22778       Divisor == APInt(Divisor.getBitWidth(), -2, /*isSigned*/ true))
22779     return SDValue();
22780 
22781   return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created);
22782 }
22783 
22784 /// Result of 'and' is compared against zero. Change to a BT node if possible.
22785 /// Returns the BT node and the condition code needed to use it.
22786 static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC, const SDLoc &dl,
22787                             SelectionDAG &DAG, X86::CondCode &X86CC) {
22788   assert(And.getOpcode() == ISD::AND && "Expected AND node!");
22789   SDValue Op0 = And.getOperand(0);
22790   SDValue Op1 = And.getOperand(1);
22791   if (Op0.getOpcode() == ISD::TRUNCATE)
22792     Op0 = Op0.getOperand(0);
22793   if (Op1.getOpcode() == ISD::TRUNCATE)
22794     Op1 = Op1.getOperand(0);
22795 
22796   SDValue Src, BitNo;
22797   if (Op1.getOpcode() == ISD::SHL)
22798     std::swap(Op0, Op1);
22799   if (Op0.getOpcode() == ISD::SHL) {
22800     if (isOneConstant(Op0.getOperand(0))) {
22801       // If we looked past a truncate, check that it's only truncating away
22802       // known zeros.
22803       unsigned BitWidth = Op0.getValueSizeInBits();
22804       unsigned AndBitWidth = And.getValueSizeInBits();
22805       if (BitWidth > AndBitWidth) {
22806         KnownBits Known = DAG.computeKnownBits(Op0);
22807         if (Known.countMinLeadingZeros() < BitWidth - AndBitWidth)
22808           return SDValue();
22809       }
22810       Src = Op1;
22811       BitNo = Op0.getOperand(1);
22812     }
22813   } else if (Op1.getOpcode() == ISD::Constant) {
22814     ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1);
22815     uint64_t AndRHSVal = AndRHS->getZExtValue();
22816     SDValue AndLHS = Op0;
22817 
22818     if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) {
22819       Src = AndLHS.getOperand(0);
22820       BitNo = AndLHS.getOperand(1);
22821     } else {
22822       // Use BT if the immediate can't be encoded in a TEST instruction or we
22823       // are optimizing for size and the immedaite won't fit in a byte.
22824       bool OptForSize = DAG.shouldOptForSize();
22825       if ((!isUInt<32>(AndRHSVal) || (OptForSize && !isUInt<8>(AndRHSVal))) &&
22826           isPowerOf2_64(AndRHSVal)) {
22827         Src = AndLHS;
22828         BitNo = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl,
22829                                 Src.getValueType());
22830       }
22831     }
22832   }
22833 
22834   // No patterns found, give up.
22835   if (!Src.getNode())
22836     return SDValue();
22837 
22838   // Remove any bit flip.
22839   if (isBitwiseNot(Src)) {
22840     Src = Src.getOperand(0);
22841     CC = CC == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
22842   }
22843 
22844   // Attempt to create the X86ISD::BT node.
22845   if (SDValue BT = getBT(Src, BitNo, dl, DAG)) {
22846     X86CC = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
22847     return BT;
22848   }
22849 
22850   return SDValue();
22851 }
22852 
22853 // Check if pre-AVX condcode can be performed by a single FCMP op.
22854 static bool cheapX86FSETCC_SSE(ISD::CondCode SetCCOpcode) {
22855   return (SetCCOpcode != ISD::SETONE) && (SetCCOpcode != ISD::SETUEQ);
22856 }
22857 
22858 /// Turns an ISD::CondCode into a value suitable for SSE floating-point mask
22859 /// CMPs.
22860 static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
22861                                    SDValue &Op1, bool &IsAlwaysSignaling) {
22862   unsigned SSECC;
22863   bool Swap = false;
22864 
22865   // SSE Condition code mapping:
22866   //  0 - EQ
22867   //  1 - LT
22868   //  2 - LE
22869   //  3 - UNORD
22870   //  4 - NEQ
22871   //  5 - NLT
22872   //  6 - NLE
22873   //  7 - ORD
22874   switch (SetCCOpcode) {
22875   default: llvm_unreachable("Unexpected SETCC condition");
22876   case ISD::SETOEQ:
22877   case ISD::SETEQ:  SSECC = 0; break;
22878   case ISD::SETOGT:
22879   case ISD::SETGT:  Swap = true; [[fallthrough]];
22880   case ISD::SETLT:
22881   case ISD::SETOLT: SSECC = 1; break;
22882   case ISD::SETOGE:
22883   case ISD::SETGE:  Swap = true; [[fallthrough]];
22884   case ISD::SETLE:
22885   case ISD::SETOLE: SSECC = 2; break;
22886   case ISD::SETUO:  SSECC = 3; break;
22887   case ISD::SETUNE:
22888   case ISD::SETNE:  SSECC = 4; break;
22889   case ISD::SETULE: Swap = true; [[fallthrough]];
22890   case ISD::SETUGE: SSECC = 5; break;
22891   case ISD::SETULT: Swap = true; [[fallthrough]];
22892   case ISD::SETUGT: SSECC = 6; break;
22893   case ISD::SETO:   SSECC = 7; break;
22894   case ISD::SETUEQ: SSECC = 8; break;
22895   case ISD::SETONE: SSECC = 12; break;
22896   }
22897   if (Swap)
22898     std::swap(Op0, Op1);
22899 
22900   switch (SetCCOpcode) {
22901   default:
22902     IsAlwaysSignaling = true;
22903     break;
22904   case ISD::SETEQ:
22905   case ISD::SETOEQ:
22906   case ISD::SETUEQ:
22907   case ISD::SETNE:
22908   case ISD::SETONE:
22909   case ISD::SETUNE:
22910   case ISD::SETO:
22911   case ISD::SETUO:
22912     IsAlwaysSignaling = false;
22913     break;
22914   }
22915 
22916   return SSECC;
22917 }
22918 
22919 /// Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then
22920 /// concatenate the result back.
22921 static SDValue splitIntVSETCC(EVT VT, SDValue LHS, SDValue RHS,
22922                               ISD::CondCode Cond, SelectionDAG &DAG,
22923                               const SDLoc &dl) {
22924   assert(VT.isInteger() && VT == LHS.getValueType() &&
22925          VT == RHS.getValueType() && "Unsupported VTs!");
22926 
22927   SDValue CC = DAG.getCondCode(Cond);
22928 
22929   // Extract the LHS Lo/Hi vectors
22930   SDValue LHS1, LHS2;
22931   std::tie(LHS1, LHS2) = splitVector(LHS, DAG, dl);
22932 
22933   // Extract the RHS Lo/Hi vectors
22934   SDValue RHS1, RHS2;
22935   std::tie(RHS1, RHS2) = splitVector(RHS, DAG, dl);
22936 
22937   // Issue the operation on the smaller types and concatenate the result back
22938   EVT LoVT, HiVT;
22939   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
22940   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
22941                      DAG.getNode(ISD::SETCC, dl, LoVT, LHS1, RHS1, CC),
22942                      DAG.getNode(ISD::SETCC, dl, HiVT, LHS2, RHS2, CC));
22943 }
22944 
22945 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
22946 
22947   SDValue Op0 = Op.getOperand(0);
22948   SDValue Op1 = Op.getOperand(1);
22949   SDValue CC = Op.getOperand(2);
22950   MVT VT = Op.getSimpleValueType();
22951   SDLoc dl(Op);
22952 
22953   assert(VT.getVectorElementType() == MVT::i1 &&
22954          "Cannot set masked compare for this operation");
22955 
22956   ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
22957 
22958   // Prefer SETGT over SETLT.
22959   if (SetCCOpcode == ISD::SETLT) {
22960     SetCCOpcode = ISD::getSetCCSwappedOperands(SetCCOpcode);
22961     std::swap(Op0, Op1);
22962   }
22963 
22964   return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode);
22965 }
22966 
22967 /// Given a buildvector constant, return a new vector constant with each element
22968 /// incremented or decremented. If incrementing or decrementing would result in
22969 /// unsigned overflow or underflow or this is not a simple vector constant,
22970 /// return an empty value.
22971 static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc,
22972                                     bool NSW) {
22973   auto *BV = dyn_cast<BuildVectorSDNode>(V.getNode());
22974   if (!BV || !V.getValueType().isSimple())
22975     return SDValue();
22976 
22977   MVT VT = V.getSimpleValueType();
22978   MVT EltVT = VT.getVectorElementType();
22979   unsigned NumElts = VT.getVectorNumElements();
22980   SmallVector<SDValue, 8> NewVecC;
22981   SDLoc DL(V);
22982   for (unsigned i = 0; i < NumElts; ++i) {
22983     auto *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
22984     if (!Elt || Elt->isOpaque() || Elt->getSimpleValueType(0) != EltVT)
22985       return SDValue();
22986 
22987     // Avoid overflow/underflow.
22988     const APInt &EltC = Elt->getAPIntValue();
22989     if ((IsInc && EltC.isMaxValue()) || (!IsInc && EltC.isZero()))
22990       return SDValue();
22991     if (NSW && ((IsInc && EltC.isMaxSignedValue()) ||
22992                 (!IsInc && EltC.isMinSignedValue())))
22993       return SDValue();
22994 
22995     NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT));
22996   }
22997 
22998   return DAG.getBuildVector(VT, DL, NewVecC);
22999 }
23000 
23001 /// As another special case, use PSUBUS[BW] when it's profitable. E.g. for
23002 /// Op0 u<= Op1:
23003 ///   t = psubus Op0, Op1
23004 ///   pcmpeq t, <0..0>
23005 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
23006                                     ISD::CondCode Cond, const SDLoc &dl,
23007                                     const X86Subtarget &Subtarget,
23008                                     SelectionDAG &DAG) {
23009   if (!Subtarget.hasSSE2())
23010     return SDValue();
23011 
23012   MVT VET = VT.getVectorElementType();
23013   if (VET != MVT::i8 && VET != MVT::i16)
23014     return SDValue();
23015 
23016   switch (Cond) {
23017   default:
23018     return SDValue();
23019   case ISD::SETULT: {
23020     // If the comparison is against a constant we can turn this into a
23021     // setule.  With psubus, setule does not require a swap.  This is
23022     // beneficial because the constant in the register is no longer
23023     // destructed as the destination so it can be hoisted out of a loop.
23024     // Only do this pre-AVX since vpcmp* is no longer destructive.
23025     if (Subtarget.hasAVX())
23026       return SDValue();
23027     SDValue ULEOp1 =
23028         incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false);
23029     if (!ULEOp1)
23030       return SDValue();
23031     Op1 = ULEOp1;
23032     break;
23033   }
23034   case ISD::SETUGT: {
23035     // If the comparison is against a constant, we can turn this into a setuge.
23036     // This is beneficial because materializing a constant 0 for the PCMPEQ is
23037     // probably cheaper than XOR+PCMPGT using 2 different vector constants:
23038     // cmpgt (xor X, SignMaskC) CmpC --> cmpeq (usubsat (CmpC+1), X), 0
23039     SDValue UGEOp1 =
23040         incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false);
23041     if (!UGEOp1)
23042       return SDValue();
23043     Op1 = Op0;
23044     Op0 = UGEOp1;
23045     break;
23046   }
23047   // Psubus is better than flip-sign because it requires no inversion.
23048   case ISD::SETUGE:
23049     std::swap(Op0, Op1);
23050     break;
23051   case ISD::SETULE:
23052     break;
23053   }
23054 
23055   SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);
23056   return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
23057                      DAG.getConstant(0, dl, VT));
23058 }
23059 
23060 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
23061                            SelectionDAG &DAG) {
23062   bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
23063                   Op.getOpcode() == ISD::STRICT_FSETCCS;
23064   SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
23065   SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
23066   SDValue CC = Op.getOperand(IsStrict ? 3 : 2);
23067   MVT VT = Op->getSimpleValueType(0);
23068   ISD::CondCode Cond = cast<CondCodeSDNode>(CC)->get();
23069   bool isFP = Op1.getSimpleValueType().isFloatingPoint();
23070   SDLoc dl(Op);
23071 
23072   if (isFP) {
23073     MVT EltVT = Op0.getSimpleValueType().getVectorElementType();
23074     assert(EltVT == MVT::f16 || EltVT == MVT::f32 || EltVT == MVT::f64);
23075     if (isSoftF16(EltVT, Subtarget))
23076       return SDValue();
23077 
23078     bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
23079     SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
23080 
23081     // If we have a strict compare with a vXi1 result and the input is 128/256
23082     // bits we can't use a masked compare unless we have VLX. If we use a wider
23083     // compare like we do for non-strict, we might trigger spurious exceptions
23084     // from the upper elements. Instead emit a AVX compare and convert to mask.
23085     unsigned Opc;
23086     if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1 &&
23087         (!IsStrict || Subtarget.hasVLX() ||
23088          Op0.getSimpleValueType().is512BitVector())) {
23089 #ifndef NDEBUG
23090       unsigned Num = VT.getVectorNumElements();
23091       assert(Num <= 16 || (Num == 32 && EltVT == MVT::f16));
23092 #endif
23093       Opc = IsStrict ? X86ISD::STRICT_CMPM : X86ISD::CMPM;
23094     } else {
23095       Opc = IsStrict ? X86ISD::STRICT_CMPP : X86ISD::CMPP;
23096       // The SSE/AVX packed FP comparison nodes are defined with a
23097       // floating-point vector result that matches the operand type. This allows
23098       // them to work with an SSE1 target (integer vector types are not legal).
23099       VT = Op0.getSimpleValueType();
23100     }
23101 
23102     SDValue Cmp;
23103     bool IsAlwaysSignaling;
23104     unsigned SSECC = translateX86FSETCC(Cond, Op0, Op1, IsAlwaysSignaling);
23105     if (!Subtarget.hasAVX()) {
23106       // TODO: We could use following steps to handle a quiet compare with
23107       // signaling encodings.
23108       // 1. Get ordered masks from a quiet ISD::SETO
23109       // 2. Use the masks to mask potential unordered elements in operand A, B
23110       // 3. Get the compare results of masked A, B
23111       // 4. Calculating final result using the mask and result from 3
23112       // But currently, we just fall back to scalar operations.
23113       if (IsStrict && IsAlwaysSignaling && !IsSignaling)
23114         return SDValue();
23115 
23116       // Insert an extra signaling instruction to raise exception.
23117       if (IsStrict && !IsAlwaysSignaling && IsSignaling) {
23118         SDValue SignalCmp = DAG.getNode(
23119             Opc, dl, {VT, MVT::Other},
23120             {Chain, Op0, Op1, DAG.getTargetConstant(1, dl, MVT::i8)}); // LT_OS
23121         // FIXME: It seems we need to update the flags of all new strict nodes.
23122         // Otherwise, mayRaiseFPException in MI will return false due to
23123         // NoFPExcept = false by default. However, I didn't find it in other
23124         // patches.
23125         SignalCmp->setFlags(Op->getFlags());
23126         Chain = SignalCmp.getValue(1);
23127       }
23128 
23129       // In the two cases not handled by SSE compare predicates (SETUEQ/SETONE),
23130       // emit two comparisons and a logic op to tie them together.
23131       if (!cheapX86FSETCC_SSE(Cond)) {
23132         // LLVM predicate is SETUEQ or SETONE.
23133         unsigned CC0, CC1;
23134         unsigned CombineOpc;
23135         if (Cond == ISD::SETUEQ) {
23136           CC0 = 3; // UNORD
23137           CC1 = 0; // EQ
23138           CombineOpc = X86ISD::FOR;
23139         } else {
23140           assert(Cond == ISD::SETONE);
23141           CC0 = 7; // ORD
23142           CC1 = 4; // NEQ
23143           CombineOpc = X86ISD::FAND;
23144         }
23145 
23146         SDValue Cmp0, Cmp1;
23147         if (IsStrict) {
23148           Cmp0 = DAG.getNode(
23149               Opc, dl, {VT, MVT::Other},
23150               {Chain, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)});
23151           Cmp1 = DAG.getNode(
23152               Opc, dl, {VT, MVT::Other},
23153               {Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)});
23154           Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1),
23155                               Cmp1.getValue(1));
23156         } else {
23157           Cmp0 = DAG.getNode(
23158               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8));
23159           Cmp1 = DAG.getNode(
23160               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8));
23161         }
23162         Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
23163       } else {
23164         if (IsStrict) {
23165           Cmp = DAG.getNode(
23166               Opc, dl, {VT, MVT::Other},
23167               {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
23168           Chain = Cmp.getValue(1);
23169         } else
23170           Cmp = DAG.getNode(
23171               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
23172       }
23173     } else {
23174       // Handle all other FP comparisons here.
23175       if (IsStrict) {
23176         // Make a flip on already signaling CCs before setting bit 4 of AVX CC.
23177         SSECC |= (IsAlwaysSignaling ^ IsSignaling) << 4;
23178         Cmp = DAG.getNode(
23179             Opc, dl, {VT, MVT::Other},
23180             {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
23181         Chain = Cmp.getValue(1);
23182       } else
23183         Cmp = DAG.getNode(
23184             Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
23185     }
23186 
23187     if (VT.getFixedSizeInBits() >
23188         Op.getSimpleValueType().getFixedSizeInBits()) {
23189       // We emitted a compare with an XMM/YMM result. Finish converting to a
23190       // mask register using a vptestm.
23191       EVT CastVT = EVT(VT).changeVectorElementTypeToInteger();
23192       Cmp = DAG.getBitcast(CastVT, Cmp);
23193       Cmp = DAG.getSetCC(dl, Op.getSimpleValueType(), Cmp,
23194                          DAG.getConstant(0, dl, CastVT), ISD::SETNE);
23195     } else {
23196       // If this is SSE/AVX CMPP, bitcast the result back to integer to match
23197       // the result type of SETCC. The bitcast is expected to be optimized
23198       // away during combining/isel.
23199       Cmp = DAG.getBitcast(Op.getSimpleValueType(), Cmp);
23200     }
23201 
23202     if (IsStrict)
23203       return DAG.getMergeValues({Cmp, Chain}, dl);
23204 
23205     return Cmp;
23206   }
23207 
23208   assert(!IsStrict && "Strict SETCC only handles FP operands.");
23209 
23210   MVT VTOp0 = Op0.getSimpleValueType();
23211   (void)VTOp0;
23212   assert(VTOp0 == Op1.getSimpleValueType() &&
23213          "Expected operands with same type!");
23214   assert(VT.getVectorNumElements() == VTOp0.getVectorNumElements() &&
23215          "Invalid number of packed elements for source and destination!");
23216 
23217   // The non-AVX512 code below works under the assumption that source and
23218   // destination types are the same.
23219   assert((Subtarget.hasAVX512() || (VT == VTOp0)) &&
23220          "Value types for source and destination must be the same!");
23221 
23222   // The result is boolean, but operands are int/float
23223   if (VT.getVectorElementType() == MVT::i1) {
23224     // In AVX-512 architecture setcc returns mask with i1 elements,
23225     // But there is no compare instruction for i8 and i16 elements in KNL.
23226     assert((VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) &&
23227            "Unexpected operand type");
23228     return LowerIntVSETCC_AVX512(Op, DAG);
23229   }
23230 
23231   // Lower using XOP integer comparisons.
23232   if (VT.is128BitVector() && Subtarget.hasXOP()) {
23233     // Translate compare code to XOP PCOM compare mode.
23234     unsigned CmpMode = 0;
23235     switch (Cond) {
23236     default: llvm_unreachable("Unexpected SETCC condition");
23237     case ISD::SETULT:
23238     case ISD::SETLT: CmpMode = 0x00; break;
23239     case ISD::SETULE:
23240     case ISD::SETLE: CmpMode = 0x01; break;
23241     case ISD::SETUGT:
23242     case ISD::SETGT: CmpMode = 0x02; break;
23243     case ISD::SETUGE:
23244     case ISD::SETGE: CmpMode = 0x03; break;
23245     case ISD::SETEQ: CmpMode = 0x04; break;
23246     case ISD::SETNE: CmpMode = 0x05; break;
23247     }
23248 
23249     // Are we comparing unsigned or signed integers?
23250     unsigned Opc =
23251         ISD::isUnsignedIntSetCC(Cond) ? X86ISD::VPCOMU : X86ISD::VPCOM;
23252 
23253     return DAG.getNode(Opc, dl, VT, Op0, Op1,
23254                        DAG.getTargetConstant(CmpMode, dl, MVT::i8));
23255   }
23256 
23257   // (X & Y) != 0 --> (X & Y) == Y iff Y is power-of-2.
23258   // Revert part of the simplifySetCCWithAnd combine, to avoid an invert.
23259   if (Cond == ISD::SETNE && ISD::isBuildVectorAllZeros(Op1.getNode())) {
23260     SDValue BC0 = peekThroughBitcasts(Op0);
23261     if (BC0.getOpcode() == ISD::AND) {
23262       APInt UndefElts;
23263       SmallVector<APInt, 64> EltBits;
23264       if (getTargetConstantBitsFromNode(BC0.getOperand(1),
23265                                         VT.getScalarSizeInBits(), UndefElts,
23266                                         EltBits, false, false)) {
23267         if (llvm::all_of(EltBits, [](APInt &V) { return V.isPowerOf2(); })) {
23268           Cond = ISD::SETEQ;
23269           Op1 = DAG.getBitcast(VT, BC0.getOperand(1));
23270         }
23271       }
23272     }
23273   }
23274 
23275   // ICMP_EQ(AND(X,C),C) -> SRA(SHL(X,LOG2(C)),BW-1) iff C is power-of-2.
23276   if (Cond == ISD::SETEQ && Op0.getOpcode() == ISD::AND &&
23277       Op0.getOperand(1) == Op1 && Op0.hasOneUse()) {
23278     ConstantSDNode *C1 = isConstOrConstSplat(Op1);
23279     if (C1 && C1->getAPIntValue().isPowerOf2()) {
23280       unsigned BitWidth = VT.getScalarSizeInBits();
23281       unsigned ShiftAmt = BitWidth - C1->getAPIntValue().logBase2() - 1;
23282 
23283       SDValue Result = Op0.getOperand(0);
23284       Result = DAG.getNode(ISD::SHL, dl, VT, Result,
23285                            DAG.getConstant(ShiftAmt, dl, VT));
23286       Result = DAG.getNode(ISD::SRA, dl, VT, Result,
23287                            DAG.getConstant(BitWidth - 1, dl, VT));
23288       return Result;
23289     }
23290   }
23291 
23292   // Break 256-bit integer vector compare into smaller ones.
23293   if (VT.is256BitVector() && !Subtarget.hasInt256())
23294     return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23295 
23296   // Break 512-bit integer vector compare into smaller ones.
23297   // TODO: Try harder to use VPCMPx + VPMOV2x?
23298   if (VT.is512BitVector())
23299     return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23300 
23301   // If we have a limit constant, try to form PCMPGT (signed cmp) to avoid
23302   // not-of-PCMPEQ:
23303   // X != INT_MIN --> X >s INT_MIN
23304   // X != INT_MAX --> X <s INT_MAX --> INT_MAX >s X
23305   // +X != 0 --> +X >s 0
23306   APInt ConstValue;
23307   if (Cond == ISD::SETNE &&
23308       ISD::isConstantSplatVector(Op1.getNode(), ConstValue)) {
23309     if (ConstValue.isMinSignedValue())
23310       Cond = ISD::SETGT;
23311     else if (ConstValue.isMaxSignedValue())
23312       Cond = ISD::SETLT;
23313     else if (ConstValue.isZero() && DAG.SignBitIsZero(Op0))
23314       Cond = ISD::SETGT;
23315   }
23316 
23317   // If both operands are known non-negative, then an unsigned compare is the
23318   // same as a signed compare and there's no need to flip signbits.
23319   // TODO: We could check for more general simplifications here since we're
23320   // computing known bits.
23321   bool FlipSigns = ISD::isUnsignedIntSetCC(Cond) &&
23322                    !(DAG.SignBitIsZero(Op0) && DAG.SignBitIsZero(Op1));
23323 
23324   // Special case: Use min/max operations for unsigned compares.
23325   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23326   if (ISD::isUnsignedIntSetCC(Cond) &&
23327       (FlipSigns || ISD::isTrueWhenEqual(Cond)) &&
23328       TLI.isOperationLegal(ISD::UMIN, VT)) {
23329     // If we have a constant operand, increment/decrement it and change the
23330     // condition to avoid an invert.
23331     if (Cond == ISD::SETUGT) {
23332       // X > C --> X >= (C+1) --> X == umax(X, C+1)
23333       if (SDValue UGTOp1 =
23334               incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false)) {
23335         Op1 = UGTOp1;
23336         Cond = ISD::SETUGE;
23337       }
23338     }
23339     if (Cond == ISD::SETULT) {
23340       // X < C --> X <= (C-1) --> X == umin(X, C-1)
23341       if (SDValue ULTOp1 =
23342               incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false)) {
23343         Op1 = ULTOp1;
23344         Cond = ISD::SETULE;
23345       }
23346     }
23347     bool Invert = false;
23348     unsigned Opc;
23349     switch (Cond) {
23350     default: llvm_unreachable("Unexpected condition code");
23351     case ISD::SETUGT: Invert = true; [[fallthrough]];
23352     case ISD::SETULE: Opc = ISD::UMIN; break;
23353     case ISD::SETULT: Invert = true; [[fallthrough]];
23354     case ISD::SETUGE: Opc = ISD::UMAX; break;
23355     }
23356 
23357     SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
23358     Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
23359 
23360     // If the logical-not of the result is required, perform that now.
23361     if (Invert)
23362       Result = DAG.getNOT(dl, Result, VT);
23363 
23364     return Result;
23365   }
23366 
23367   // Try to use SUBUS and PCMPEQ.
23368   if (FlipSigns)
23369     if (SDValue V =
23370             LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG))
23371       return V;
23372 
23373   // We are handling one of the integer comparisons here. Since SSE only has
23374   // GT and EQ comparisons for integer, swapping operands and multiple
23375   // operations may be required for some comparisons.
23376   unsigned Opc = (Cond == ISD::SETEQ || Cond == ISD::SETNE) ? X86ISD::PCMPEQ
23377                                                             : X86ISD::PCMPGT;
23378   bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT ||
23379               Cond == ISD::SETGE || Cond == ISD::SETUGE;
23380   bool Invert = Cond == ISD::SETNE ||
23381                 (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond));
23382 
23383   if (Swap)
23384     std::swap(Op0, Op1);
23385 
23386   // Check that the operation in question is available (most are plain SSE2,
23387   // but PCMPGTQ and PCMPEQQ have different requirements).
23388   if (VT == MVT::v2i64) {
23389     if (Opc == X86ISD::PCMPGT && !Subtarget.hasSSE42()) {
23390       assert(Subtarget.hasSSE2() && "Don't know how to lower!");
23391 
23392       // Special case for sign bit test. We can use a v4i32 PCMPGT and shuffle
23393       // the odd elements over the even elements.
23394       if (!FlipSigns && !Invert && ISD::isBuildVectorAllZeros(Op0.getNode())) {
23395         Op0 = DAG.getConstant(0, dl, MVT::v4i32);
23396         Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23397 
23398         SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23399         static const int MaskHi[] = { 1, 1, 3, 3 };
23400         SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23401 
23402         return DAG.getBitcast(VT, Result);
23403       }
23404 
23405       if (!FlipSigns && !Invert && ISD::isBuildVectorAllOnes(Op1.getNode())) {
23406         Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23407         Op1 = DAG.getConstant(-1, dl, MVT::v4i32);
23408 
23409         SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23410         static const int MaskHi[] = { 1, 1, 3, 3 };
23411         SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23412 
23413         return DAG.getBitcast(VT, Result);
23414       }
23415 
23416       // Since SSE has no unsigned integer comparisons, we need to flip the sign
23417       // bits of the inputs before performing those operations. The lower
23418       // compare is always unsigned.
23419       SDValue SB = DAG.getConstant(FlipSigns ? 0x8000000080000000ULL
23420                                              : 0x0000000080000000ULL,
23421                                    dl, MVT::v2i64);
23422 
23423       Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB);
23424       Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB);
23425 
23426       // Cast everything to the right type.
23427       Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23428       Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23429 
23430       // Emulate PCMPGTQ with (hi1 > hi2) | ((hi1 == hi2) & (lo1 > lo2))
23431       SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23432       SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
23433 
23434       // Create masks for only the low parts/high parts of the 64 bit integers.
23435       static const int MaskHi[] = { 1, 1, 3, 3 };
23436       static const int MaskLo[] = { 0, 0, 2, 2 };
23437       SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
23438       SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
23439       SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23440 
23441       SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
23442       Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);
23443 
23444       if (Invert)
23445         Result = DAG.getNOT(dl, Result, MVT::v4i32);
23446 
23447       return DAG.getBitcast(VT, Result);
23448     }
23449 
23450     if (Opc == X86ISD::PCMPEQ && !Subtarget.hasSSE41()) {
23451       // If pcmpeqq is missing but pcmpeqd is available synthesize pcmpeqq with
23452       // pcmpeqd + pshufd + pand.
23453       assert(Subtarget.hasSSE2() && !FlipSigns && "Don't know how to lower!");
23454 
23455       // First cast everything to the right type.
23456       Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23457       Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23458 
23459       // Do the compare.
23460       SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
23461 
23462       // Make sure the lower and upper halves are both all-ones.
23463       static const int Mask[] = { 1, 0, 3, 2 };
23464       SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
23465       Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
23466 
23467       if (Invert)
23468         Result = DAG.getNOT(dl, Result, MVT::v4i32);
23469 
23470       return DAG.getBitcast(VT, Result);
23471     }
23472   }
23473 
23474   // Since SSE has no unsigned integer comparisons, we need to flip the sign
23475   // bits of the inputs before performing those operations.
23476   if (FlipSigns) {
23477     MVT EltVT = VT.getVectorElementType();
23478     SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
23479                                  VT);
23480     Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM);
23481     Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM);
23482   }
23483 
23484   SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
23485 
23486   // If the logical-not of the result is required, perform that now.
23487   if (Invert)
23488     Result = DAG.getNOT(dl, Result, VT);
23489 
23490   return Result;
23491 }
23492 
23493 // Try to select this as a KORTEST+SETCC or KTEST+SETCC if possible.
23494 static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
23495                               const SDLoc &dl, SelectionDAG &DAG,
23496                               const X86Subtarget &Subtarget,
23497                               SDValue &X86CC) {
23498   assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
23499 
23500   // Must be a bitcast from vXi1.
23501   if (Op0.getOpcode() != ISD::BITCAST)
23502     return SDValue();
23503 
23504   Op0 = Op0.getOperand(0);
23505   MVT VT = Op0.getSimpleValueType();
23506   if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) &&
23507       !(Subtarget.hasDQI() && VT == MVT::v8i1) &&
23508       !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)))
23509     return SDValue();
23510 
23511   X86::CondCode X86Cond;
23512   if (isNullConstant(Op1)) {
23513     X86Cond = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
23514   } else if (isAllOnesConstant(Op1)) {
23515     // C flag is set for all ones.
23516     X86Cond = CC == ISD::SETEQ ? X86::COND_B : X86::COND_AE;
23517   } else
23518     return SDValue();
23519 
23520   // If the input is an AND, we can combine it's operands into the KTEST.
23521   bool KTestable = false;
23522   if (Subtarget.hasDQI() && (VT == MVT::v8i1 || VT == MVT::v16i1))
23523     KTestable = true;
23524   if (Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))
23525     KTestable = true;
23526   if (!isNullConstant(Op1))
23527     KTestable = false;
23528   if (KTestable && Op0.getOpcode() == ISD::AND && Op0.hasOneUse()) {
23529     SDValue LHS = Op0.getOperand(0);
23530     SDValue RHS = Op0.getOperand(1);
23531     X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23532     return DAG.getNode(X86ISD::KTEST, dl, MVT::i32, LHS, RHS);
23533   }
23534 
23535   // If the input is an OR, we can combine it's operands into the KORTEST.
23536   SDValue LHS = Op0;
23537   SDValue RHS = Op0;
23538   if (Op0.getOpcode() == ISD::OR && Op0.hasOneUse()) {
23539     LHS = Op0.getOperand(0);
23540     RHS = Op0.getOperand(1);
23541   }
23542 
23543   X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23544   return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
23545 }
23546 
23547 /// Emit flags for the given setcc condition and operands. Also returns the
23548 /// corresponding X86 condition code constant in X86CC.
23549 SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
23550                                              ISD::CondCode CC, const SDLoc &dl,
23551                                              SelectionDAG &DAG,
23552                                              SDValue &X86CC) const {
23553   // Equality Combines.
23554   if (CC == ISD::SETEQ || CC == ISD::SETNE) {
23555     X86::CondCode X86CondCode;
23556 
23557     // Optimize to BT if possible.
23558     // Lower (X & (1 << N)) == 0 to BT(X, N).
23559     // Lower ((X >>u N) & 1) != 0 to BT(X, N).
23560     // Lower ((X >>s N) & 1) != 0 to BT(X, N).
23561     if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && isNullConstant(Op1)) {
23562       if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CondCode)) {
23563         X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23564         return BT;
23565       }
23566     }
23567 
23568     // Try to use PTEST/PMOVMSKB for a tree AND/ORs equality compared with -1/0.
23569     if (SDValue CmpZ = MatchVectorAllEqualTest(Op0, Op1, CC, dl, Subtarget, DAG,
23570                                                X86CondCode)) {
23571       X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23572       return CmpZ;
23573     }
23574 
23575     // Try to lower using KORTEST or KTEST.
23576     if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
23577       return Test;
23578 
23579     // Look for X == 0, X == 1, X != 0, or X != 1.  We can simplify some forms
23580     // of these.
23581     if (isOneConstant(Op1) || isNullConstant(Op1)) {
23582       // If the input is a setcc, then reuse the input setcc or use a new one
23583       // with the inverted condition.
23584       if (Op0.getOpcode() == X86ISD::SETCC) {
23585         bool Invert = (CC == ISD::SETNE) ^ isNullConstant(Op1);
23586 
23587         X86CC = Op0.getOperand(0);
23588         if (Invert) {
23589           X86CondCode = (X86::CondCode)Op0.getConstantOperandVal(0);
23590           X86CondCode = X86::GetOppositeBranchCondition(X86CondCode);
23591           X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23592         }
23593 
23594         return Op0.getOperand(1);
23595       }
23596     }
23597 
23598     // Try to use the carry flag from the add in place of an separate CMP for:
23599     // (seteq (add X, -1), -1). Similar for setne.
23600     if (isAllOnesConstant(Op1) && Op0.getOpcode() == ISD::ADD &&
23601         Op0.getOperand(1) == Op1) {
23602       if (isProfitableToUseFlagOp(Op0)) {
23603         SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32);
23604 
23605         SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0),
23606                                   Op0.getOperand(1));
23607         DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New);
23608         X86CondCode = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
23609         X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23610         return SDValue(New.getNode(), 1);
23611       }
23612     }
23613   }
23614 
23615   X86::CondCode CondCode =
23616       TranslateX86CC(CC, dl, /*IsFP*/ false, Op0, Op1, DAG);
23617   assert(CondCode != X86::COND_INVALID && "Unexpected condition code!");
23618 
23619   SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget);
23620   X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
23621   return EFLAGS;
23622 }
23623 
23624 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
23625 
23626   bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
23627                   Op.getOpcode() == ISD::STRICT_FSETCCS;
23628   MVT VT = Op->getSimpleValueType(0);
23629 
23630   if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
23631 
23632   assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");
23633   SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
23634   SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
23635   SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
23636   SDLoc dl(Op);
23637   ISD::CondCode CC =
23638       cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get();
23639 
23640   if (isSoftF16(Op0.getValueType(), Subtarget))
23641     return SDValue();
23642 
23643   // Handle f128 first, since one possible outcome is a normal integer
23644   // comparison which gets handled by emitFlagsForSetcc.
23645   if (Op0.getValueType() == MVT::f128) {
23646     softenSetCCOperands(DAG, MVT::f128, Op0, Op1, CC, dl, Op0, Op1, Chain,
23647                         Op.getOpcode() == ISD::STRICT_FSETCCS);
23648 
23649     // If softenSetCCOperands returned a scalar, use it.
23650     if (!Op1.getNode()) {
23651       assert(Op0.getValueType() == Op.getValueType() &&
23652              "Unexpected setcc expansion!");
23653       if (IsStrict)
23654         return DAG.getMergeValues({Op0, Chain}, dl);
23655       return Op0;
23656     }
23657   }
23658 
23659   if (Op0.getSimpleValueType().isInteger()) {
23660     // Attempt to canonicalize SGT/UGT -> SGE/UGE compares with constant which
23661     // reduces the number of EFLAGs bit reads (the GE conditions don't read ZF),
23662     // this may translate to less uops depending on uarch implementation. The
23663     // equivalent for SLE/ULE -> SLT/ULT isn't likely to happen as we already
23664     // canonicalize to that CondCode.
23665     // NOTE: Only do this if incrementing the constant doesn't increase the bit
23666     // encoding size - so it must either already be a i8 or i32 immediate, or it
23667     // shrinks down to that. We don't do this for any i64's to avoid additional
23668     // constant materializations.
23669     // TODO: Can we move this to TranslateX86CC to handle jumps/branches too?
23670     if (auto *Op1C = dyn_cast<ConstantSDNode>(Op1)) {
23671       const APInt &Op1Val = Op1C->getAPIntValue();
23672       if (!Op1Val.isZero()) {
23673         // Ensure the constant+1 doesn't overflow.
23674         if ((CC == ISD::CondCode::SETGT && !Op1Val.isMaxSignedValue()) ||
23675             (CC == ISD::CondCode::SETUGT && !Op1Val.isMaxValue())) {
23676           APInt Op1ValPlusOne = Op1Val + 1;
23677           if (Op1ValPlusOne.isSignedIntN(32) &&
23678               (!Op1Val.isSignedIntN(8) || Op1ValPlusOne.isSignedIntN(8))) {
23679             Op1 = DAG.getConstant(Op1ValPlusOne, dl, Op0.getValueType());
23680             CC = CC == ISD::CondCode::SETGT ? ISD::CondCode::SETGE
23681                                             : ISD::CondCode::SETUGE;
23682           }
23683         }
23684       }
23685     }
23686 
23687     SDValue X86CC;
23688     SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC);
23689     SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
23690     return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
23691   }
23692 
23693   // Handle floating point.
23694   X86::CondCode CondCode = TranslateX86CC(CC, dl, /*IsFP*/ true, Op0, Op1, DAG);
23695   if (CondCode == X86::COND_INVALID)
23696     return SDValue();
23697 
23698   SDValue EFLAGS;
23699   if (IsStrict) {
23700     bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
23701     EFLAGS =
23702         DAG.getNode(IsSignaling ? X86ISD::STRICT_FCMPS : X86ISD::STRICT_FCMP,
23703                     dl, {MVT::i32, MVT::Other}, {Chain, Op0, Op1});
23704     Chain = EFLAGS.getValue(1);
23705   } else {
23706     EFLAGS = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, Op0, Op1);
23707   }
23708 
23709   SDValue X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
23710   SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
23711   return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
23712 }
23713 
23714 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
23715   SDValue LHS = Op.getOperand(0);
23716   SDValue RHS = Op.getOperand(1);
23717   SDValue Carry = Op.getOperand(2);
23718   SDValue Cond = Op.getOperand(3);
23719   SDLoc DL(Op);
23720 
23721   assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
23722   X86::CondCode CC = TranslateIntegerX86CC(cast<CondCodeSDNode>(Cond)->get());
23723 
23724   // Recreate the carry if needed.
23725   EVT CarryVT = Carry.getValueType();
23726   Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
23727                       Carry, DAG.getAllOnesConstant(DL, CarryVT));
23728 
23729   SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
23730   SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
23731   return getSETCC(CC, Cmp.getValue(1), DL, DAG);
23732 }
23733 
23734 // This function returns three things: the arithmetic computation itself
23735 // (Value), an EFLAGS result (Overflow), and a condition code (Cond).  The
23736 // flag and the condition code define the case in which the arithmetic
23737 // computation overflows.
23738 static std::pair<SDValue, SDValue>
23739 getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
23740   assert(Op.getResNo() == 0 && "Unexpected result number!");
23741   SDValue Value, Overflow;
23742   SDValue LHS = Op.getOperand(0);
23743   SDValue RHS = Op.getOperand(1);
23744   unsigned BaseOp = 0;
23745   SDLoc DL(Op);
23746   switch (Op.getOpcode()) {
23747   default: llvm_unreachable("Unknown ovf instruction!");
23748   case ISD::SADDO:
23749     BaseOp = X86ISD::ADD;
23750     Cond = X86::COND_O;
23751     break;
23752   case ISD::UADDO:
23753     BaseOp = X86ISD::ADD;
23754     Cond = isOneConstant(RHS) ? X86::COND_E : X86::COND_B;
23755     break;
23756   case ISD::SSUBO:
23757     BaseOp = X86ISD::SUB;
23758     Cond = X86::COND_O;
23759     break;
23760   case ISD::USUBO:
23761     BaseOp = X86ISD::SUB;
23762     Cond = X86::COND_B;
23763     break;
23764   case ISD::SMULO:
23765     BaseOp = X86ISD::SMUL;
23766     Cond = X86::COND_O;
23767     break;
23768   case ISD::UMULO:
23769     BaseOp = X86ISD::UMUL;
23770     Cond = X86::COND_O;
23771     break;
23772   }
23773 
23774   if (BaseOp) {
23775     // Also sets EFLAGS.
23776     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
23777     Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
23778     Overflow = Value.getValue(1);
23779   }
23780 
23781   return std::make_pair(Value, Overflow);
23782 }
23783 
23784 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
23785   // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
23786   // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
23787   // looks for this combo and may remove the "setcc" instruction if the "setcc"
23788   // has only one use.
23789   SDLoc DL(Op);
23790   X86::CondCode Cond;
23791   SDValue Value, Overflow;
23792   std::tie(Value, Overflow) = getX86XALUOOp(Cond, Op, DAG);
23793 
23794   SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
23795   assert(Op->getValueType(1) == MVT::i8 && "Unexpected VT!");
23796   return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC);
23797 }
23798 
23799 /// Return true if opcode is a X86 logical comparison.
23800 static bool isX86LogicalCmp(SDValue Op) {
23801   unsigned Opc = Op.getOpcode();
23802   if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
23803       Opc == X86ISD::FCMP)
23804     return true;
23805   if (Op.getResNo() == 1 &&
23806       (Opc == X86ISD::ADD || Opc == X86ISD::SUB || Opc == X86ISD::ADC ||
23807        Opc == X86ISD::SBB || Opc == X86ISD::SMUL || Opc == X86ISD::UMUL ||
23808        Opc == X86ISD::OR || Opc == X86ISD::XOR || Opc == X86ISD::AND))
23809     return true;
23810 
23811   return false;
23812 }
23813 
23814 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
23815   if (V.getOpcode() != ISD::TRUNCATE)
23816     return false;
23817 
23818   SDValue VOp0 = V.getOperand(0);
23819   unsigned InBits = VOp0.getValueSizeInBits();
23820   unsigned Bits = V.getValueSizeInBits();
23821   return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
23822 }
23823 
23824 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
23825   bool AddTest = true;
23826   SDValue Cond  = Op.getOperand(0);
23827   SDValue Op1 = Op.getOperand(1);
23828   SDValue Op2 = Op.getOperand(2);
23829   SDLoc DL(Op);
23830   MVT VT = Op1.getSimpleValueType();
23831   SDValue CC;
23832 
23833   if (isSoftF16(VT, Subtarget)) {
23834     MVT NVT = VT.changeTypeToInteger();
23835     return DAG.getBitcast(VT, DAG.getNode(ISD::SELECT, DL, NVT, Cond,
23836                                           DAG.getBitcast(NVT, Op1),
23837                                           DAG.getBitcast(NVT, Op2)));
23838   }
23839 
23840   // Lower FP selects into a CMP/AND/ANDN/OR sequence when the necessary SSE ops
23841   // are available or VBLENDV if AVX is available.
23842   // Otherwise FP cmovs get lowered into a less efficient branch sequence later.
23843   if (Cond.getOpcode() == ISD::SETCC && isScalarFPTypeInSSEReg(VT) &&
23844       VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) {
23845     SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
23846     bool IsAlwaysSignaling;
23847     unsigned SSECC =
23848         translateX86FSETCC(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
23849                            CondOp0, CondOp1, IsAlwaysSignaling);
23850 
23851     if (Subtarget.hasAVX512()) {
23852       SDValue Cmp =
23853           DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1,
23854                       DAG.getTargetConstant(SSECC, DL, MVT::i8));
23855       assert(!VT.isVector() && "Not a scalar type?");
23856       return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
23857     }
23858 
23859     if (SSECC < 8 || Subtarget.hasAVX()) {
23860       SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
23861                                 DAG.getTargetConstant(SSECC, DL, MVT::i8));
23862 
23863       // If we have AVX, we can use a variable vector select (VBLENDV) instead
23864       // of 3 logic instructions for size savings and potentially speed.
23865       // Unfortunately, there is no scalar form of VBLENDV.
23866 
23867       // If either operand is a +0.0 constant, don't try this. We can expect to
23868       // optimize away at least one of the logic instructions later in that
23869       // case, so that sequence would be faster than a variable blend.
23870 
23871       // BLENDV was introduced with SSE 4.1, but the 2 register form implicitly
23872       // uses XMM0 as the selection register. That may need just as many
23873       // instructions as the AND/ANDN/OR sequence due to register moves, so
23874       // don't bother.
23875       if (Subtarget.hasAVX() && !isNullFPConstant(Op1) &&
23876           !isNullFPConstant(Op2)) {
23877         // Convert to vectors, do a VSELECT, and convert back to scalar.
23878         // All of the conversions should be optimized away.
23879         MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64;
23880         SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
23881         SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
23882         SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
23883 
23884         MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64;
23885         VCmp = DAG.getBitcast(VCmpVT, VCmp);
23886 
23887         SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
23888 
23889         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
23890                            VSel, DAG.getIntPtrConstant(0, DL));
23891       }
23892       SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
23893       SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
23894       return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
23895     }
23896   }
23897 
23898   // AVX512 fallback is to lower selects of scalar floats to masked moves.
23899   if (isScalarFPTypeInSSEReg(VT) && Subtarget.hasAVX512()) {
23900     SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
23901     return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
23902   }
23903 
23904   if (Cond.getOpcode() == ISD::SETCC &&
23905       !isSoftF16(Cond.getOperand(0).getSimpleValueType(), Subtarget)) {
23906     if (SDValue NewCond = LowerSETCC(Cond, DAG)) {
23907       Cond = NewCond;
23908       // If the condition was updated, it's possible that the operands of the
23909       // select were also updated (for example, EmitTest has a RAUW). Refresh
23910       // the local references to the select operands in case they got stale.
23911       Op1 = Op.getOperand(1);
23912       Op2 = Op.getOperand(2);
23913     }
23914   }
23915 
23916   // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y
23917   // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y
23918   // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y
23919   // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y
23920   // (select (and (x , 0x1) == 0), y, (z ^ y) ) -> (-(and (x , 0x1)) & z ) ^ y
23921   // (select (and (x , 0x1) == 0), y, (z | y) ) -> (-(and (x , 0x1)) & z ) | y
23922   // (select (x > 0), x, 0) -> (~(x >> (size_in_bits(x)-1))) & x
23923   // (select (x < 0), x, 0) -> ((x >> (size_in_bits(x)-1))) & x
23924   if (Cond.getOpcode() == X86ISD::SETCC &&
23925       Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
23926       isNullConstant(Cond.getOperand(1).getOperand(1))) {
23927     SDValue Cmp = Cond.getOperand(1);
23928     SDValue CmpOp0 = Cmp.getOperand(0);
23929     unsigned CondCode = Cond.getConstantOperandVal(0);
23930 
23931     // Special handling for __builtin_ffs(X) - 1 pattern which looks like
23932     // (select (seteq X, 0), -1, (cttz_zero_undef X)). Disable the special
23933     // handle to keep the CMP with 0. This should be removed by
23934     // optimizeCompareInst by using the flags from the BSR/TZCNT used for the
23935     // cttz_zero_undef.
23936     auto MatchFFSMinus1 = [&](SDValue Op1, SDValue Op2) {
23937       return (Op1.getOpcode() == ISD::CTTZ_ZERO_UNDEF && Op1.hasOneUse() &&
23938               Op1.getOperand(0) == CmpOp0 && isAllOnesConstant(Op2));
23939     };
23940     if (Subtarget.canUseCMOV() && (VT == MVT::i32 || VT == MVT::i64) &&
23941         ((CondCode == X86::COND_NE && MatchFFSMinus1(Op1, Op2)) ||
23942          (CondCode == X86::COND_E && MatchFFSMinus1(Op2, Op1)))) {
23943       // Keep Cmp.
23944     } else if ((isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
23945         (CondCode == X86::COND_E || CondCode == X86::COND_NE)) {
23946       SDValue Y = isAllOnesConstant(Op2) ? Op1 : Op2;
23947       SDVTList CmpVTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32);
23948 
23949       // 'X - 1' sets the carry flag if X == 0.
23950       // '0 - X' sets the carry flag if X != 0.
23951       // Convert the carry flag to a -1/0 mask with sbb:
23952       // select (X != 0), -1, Y --> 0 - X; or (sbb), Y
23953       // select (X == 0), Y, -1 --> 0 - X; or (sbb), Y
23954       // select (X != 0), Y, -1 --> X - 1; or (sbb), Y
23955       // select (X == 0), -1, Y --> X - 1; or (sbb), Y
23956       SDValue Sub;
23957       if (isAllOnesConstant(Op1) == (CondCode == X86::COND_NE)) {
23958         SDValue Zero = DAG.getConstant(0, DL, CmpOp0.getValueType());
23959         Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, Zero, CmpOp0);
23960       } else {
23961         SDValue One = DAG.getConstant(1, DL, CmpOp0.getValueType());
23962         Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, CmpOp0, One);
23963       }
23964       SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
23965                                 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
23966                                 Sub.getValue(1));
23967       return DAG.getNode(ISD::OR, DL, VT, SBB, Y);
23968     } else if (!Subtarget.canUseCMOV() && CondCode == X86::COND_E &&
23969                CmpOp0.getOpcode() == ISD::AND &&
23970                isOneConstant(CmpOp0.getOperand(1))) {
23971       SDValue Src1, Src2;
23972       // true if Op2 is XOR or OR operator and one of its operands
23973       // is equal to Op1
23974       // ( a , a op b) || ( b , a op b)
23975       auto isOrXorPattern = [&]() {
23976         if ((Op2.getOpcode() == ISD::XOR || Op2.getOpcode() == ISD::OR) &&
23977             (Op2.getOperand(0) == Op1 || Op2.getOperand(1) == Op1)) {
23978           Src1 =
23979               Op2.getOperand(0) == Op1 ? Op2.getOperand(1) : Op2.getOperand(0);
23980           Src2 = Op1;
23981           return true;
23982         }
23983         return false;
23984       };
23985 
23986       if (isOrXorPattern()) {
23987         SDValue Neg;
23988         unsigned int CmpSz = CmpOp0.getSimpleValueType().getSizeInBits();
23989         // we need mask of all zeros or ones with same size of the other
23990         // operands.
23991         if (CmpSz > VT.getSizeInBits())
23992           Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0);
23993         else if (CmpSz < VT.getSizeInBits())
23994           Neg = DAG.getNode(ISD::AND, DL, VT,
23995               DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)),
23996               DAG.getConstant(1, DL, VT));
23997         else
23998           Neg = CmpOp0;
23999         SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
24000                                    Neg); // -(and (x, 0x1))
24001         SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z
24002         return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2);  // And Op y
24003       }
24004     } else if ((VT == MVT::i32 || VT == MVT::i64) && isNullConstant(Op2) &&
24005                Cmp.getNode()->hasOneUse() && (CmpOp0 == Op1) &&
24006                ((CondCode == X86::COND_S) ||                    // smin(x, 0)
24007                 (CondCode == X86::COND_G && hasAndNot(Op1)))) { // smax(x, 0)
24008       // (select (x < 0), x, 0) -> ((x >> (size_in_bits(x)-1))) & x
24009       //
24010       // If the comparison is testing for a positive value, we have to invert
24011       // the sign bit mask, so only do that transform if the target has a
24012       // bitwise 'and not' instruction (the invert is free).
24013       // (select (x > 0), x, 0) -> (~(x >> (size_in_bits(x)-1))) & x
24014       unsigned ShCt = VT.getSizeInBits() - 1;
24015       SDValue ShiftAmt = DAG.getConstant(ShCt, DL, VT);
24016       SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt);
24017       if (CondCode == X86::COND_G)
24018         Shift = DAG.getNOT(DL, Shift, VT);
24019       return DAG.getNode(ISD::AND, DL, VT, Shift, Op1);
24020     }
24021   }
24022 
24023   // Look past (and (setcc_carry (cmp ...)), 1).
24024   if (Cond.getOpcode() == ISD::AND &&
24025       Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
24026       isOneConstant(Cond.getOperand(1)))
24027     Cond = Cond.getOperand(0);
24028 
24029   // If condition flag is set by a X86ISD::CMP, then use it as the condition
24030   // setting operand in place of the X86ISD::SETCC.
24031   unsigned CondOpcode = Cond.getOpcode();
24032   if (CondOpcode == X86ISD::SETCC ||
24033       CondOpcode == X86ISD::SETCC_CARRY) {
24034     CC = Cond.getOperand(0);
24035 
24036     SDValue Cmp = Cond.getOperand(1);
24037     bool IllegalFPCMov = false;
24038     if (VT.isFloatingPoint() && !VT.isVector() &&
24039         !isScalarFPTypeInSSEReg(VT) && Subtarget.canUseCMOV())  // FPStack?
24040       IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
24041 
24042     if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
24043         Cmp.getOpcode() == X86ISD::BT) { // FIXME
24044       Cond = Cmp;
24045       AddTest = false;
24046     }
24047   } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
24048              CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
24049              CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
24050     SDValue Value;
24051     X86::CondCode X86Cond;
24052     std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
24053 
24054     CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
24055     AddTest = false;
24056   }
24057 
24058   if (AddTest) {
24059     // Look past the truncate if the high bits are known zero.
24060     if (isTruncWithZeroHighBitsInput(Cond, DAG))
24061       Cond = Cond.getOperand(0);
24062 
24063     // We know the result of AND is compared against zero. Try to match
24064     // it to BT.
24065     if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
24066       X86::CondCode X86CondCode;
24067       if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) {
24068         CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8);
24069         Cond = BT;
24070         AddTest = false;
24071       }
24072     }
24073   }
24074 
24075   if (AddTest) {
24076     CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
24077     Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
24078   }
24079 
24080   // a <  b ? -1 :  0 -> RES = ~setcc_carry
24081   // a <  b ?  0 : -1 -> RES = setcc_carry
24082   // a >= b ? -1 :  0 -> RES = setcc_carry
24083   // a >= b ?  0 : -1 -> RES = ~setcc_carry
24084   if (Cond.getOpcode() == X86ISD::SUB) {
24085     unsigned CondCode = CC->getAsZExtVal();
24086 
24087     if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
24088         (isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
24089         (isNullConstant(Op1) || isNullConstant(Op2))) {
24090       SDValue Res =
24091           DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
24092                       DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);
24093       if (isAllOnesConstant(Op1) != (CondCode == X86::COND_B))
24094         return DAG.getNOT(DL, Res, Res.getValueType());
24095       return Res;
24096     }
24097   }
24098 
24099   // X86 doesn't have an i8 cmov. If both operands are the result of a truncate
24100   // widen the cmov and push the truncate through. This avoids introducing a new
24101   // branch during isel and doesn't add any extensions.
24102   if (Op.getValueType() == MVT::i8 &&
24103       Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
24104     SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
24105     if (T1.getValueType() == T2.getValueType() &&
24106         // Exclude CopyFromReg to avoid partial register stalls.
24107         T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
24108       SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
24109                                  CC, Cond);
24110       return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
24111     }
24112   }
24113 
24114   // Or finally, promote i8 cmovs if we have CMOV,
24115   //                 or i16 cmovs if it won't prevent folding a load.
24116   // FIXME: we should not limit promotion of i8 case to only when the CMOV is
24117   //        legal, but EmitLoweredSelect() can not deal with these extensions
24118   //        being inserted between two CMOV's. (in i16 case too TBN)
24119   //        https://bugs.llvm.org/show_bug.cgi?id=40974
24120   if ((Op.getValueType() == MVT::i8 && Subtarget.canUseCMOV()) ||
24121       (Op.getValueType() == MVT::i16 && !X86::mayFoldLoad(Op1, Subtarget) &&
24122        !X86::mayFoldLoad(Op2, Subtarget))) {
24123     Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
24124     Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
24125     SDValue Ops[] = { Op2, Op1, CC, Cond };
24126     SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
24127     return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
24128   }
24129 
24130   // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
24131   // condition is true.
24132   SDValue Ops[] = { Op2, Op1, CC, Cond };
24133   return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops, Op->getFlags());
24134 }
24135 
24136 static SDValue LowerSIGN_EXTEND_Mask(SDValue Op,
24137                                      const X86Subtarget &Subtarget,
24138                                      SelectionDAG &DAG) {
24139   MVT VT = Op->getSimpleValueType(0);
24140   SDValue In = Op->getOperand(0);
24141   MVT InVT = In.getSimpleValueType();
24142   assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!");
24143   MVT VTElt = VT.getVectorElementType();
24144   SDLoc dl(Op);
24145 
24146   unsigned NumElts = VT.getVectorNumElements();
24147 
24148   // Extend VT if the scalar type is i8/i16 and BWI is not supported.
24149   MVT ExtVT = VT;
24150   if (!Subtarget.hasBWI() && VTElt.getSizeInBits() <= 16) {
24151     // If v16i32 is to be avoided, we'll need to split and concatenate.
24152     if (NumElts == 16 && !Subtarget.canExtendTo512DQ())
24153       return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG);
24154 
24155     ExtVT = MVT::getVectorVT(MVT::i32, NumElts);
24156   }
24157 
24158   // Widen to 512-bits if VLX is not supported.
24159   MVT WideVT = ExtVT;
24160   if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) {
24161     NumElts *= 512 / ExtVT.getSizeInBits();
24162     InVT = MVT::getVectorVT(MVT::i1, NumElts);
24163     In = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, InVT, DAG.getUNDEF(InVT),
24164                      In, DAG.getIntPtrConstant(0, dl));
24165     WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(), NumElts);
24166   }
24167 
24168   SDValue V;
24169   MVT WideEltVT = WideVT.getVectorElementType();
24170   if ((Subtarget.hasDQI() && WideEltVT.getSizeInBits() >= 32) ||
24171       (Subtarget.hasBWI() && WideEltVT.getSizeInBits() <= 16)) {
24172     V = DAG.getNode(Op.getOpcode(), dl, WideVT, In);
24173   } else {
24174     SDValue NegOne = DAG.getConstant(-1, dl, WideVT);
24175     SDValue Zero = DAG.getConstant(0, dl, WideVT);
24176     V = DAG.getSelect(dl, WideVT, In, NegOne, Zero);
24177   }
24178 
24179   // Truncate if we had to extend i16/i8 above.
24180   if (VT != ExtVT) {
24181     WideVT = MVT::getVectorVT(VTElt, NumElts);
24182     V = DAG.getNode(ISD::TRUNCATE, dl, WideVT, V);
24183   }
24184 
24185   // Extract back to 128/256-bit if we widened.
24186   if (WideVT != VT)
24187     V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V,
24188                     DAG.getIntPtrConstant(0, dl));
24189 
24190   return V;
24191 }
24192 
24193 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
24194                                SelectionDAG &DAG) {
24195   SDValue In = Op->getOperand(0);
24196   MVT InVT = In.getSimpleValueType();
24197 
24198   if (InVT.getVectorElementType() == MVT::i1)
24199     return LowerSIGN_EXTEND_Mask(Op, Subtarget, DAG);
24200 
24201   assert(Subtarget.hasAVX() && "Expected AVX support");
24202   return LowerAVXExtend(Op, DAG, Subtarget);
24203 }
24204 
24205 // Lowering for SIGN_EXTEND_VECTOR_INREG and ZERO_EXTEND_VECTOR_INREG.
24206 // For sign extend this needs to handle all vector sizes and SSE4.1 and
24207 // non-SSE4.1 targets. For zero extend this should only handle inputs of
24208 // MVT::v64i8 when BWI is not supported, but AVX512 is.
24209 static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
24210                                         const X86Subtarget &Subtarget,
24211                                         SelectionDAG &DAG) {
24212   SDValue In = Op->getOperand(0);
24213   MVT VT = Op->getSimpleValueType(0);
24214   MVT InVT = In.getSimpleValueType();
24215 
24216   MVT SVT = VT.getVectorElementType();
24217   MVT InSVT = InVT.getVectorElementType();
24218   assert(SVT.getFixedSizeInBits() > InSVT.getFixedSizeInBits());
24219 
24220   if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16)
24221     return SDValue();
24222   if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8)
24223     return SDValue();
24224   if (!(VT.is128BitVector() && Subtarget.hasSSE2()) &&
24225       !(VT.is256BitVector() && Subtarget.hasAVX()) &&
24226       !(VT.is512BitVector() && Subtarget.hasAVX512()))
24227     return SDValue();
24228 
24229   SDLoc dl(Op);
24230   unsigned Opc = Op.getOpcode();
24231   unsigned NumElts = VT.getVectorNumElements();
24232 
24233   // For 256-bit vectors, we only need the lower (128-bit) half of the input.
24234   // For 512-bit vectors, we need 128-bits or 256-bits.
24235   if (InVT.getSizeInBits() > 128) {
24236     // Input needs to be at least the same number of elements as output, and
24237     // at least 128-bits.
24238     int InSize = InSVT.getSizeInBits() * NumElts;
24239     In = extractSubVector(In, 0, DAG, dl, std::max(InSize, 128));
24240     InVT = In.getSimpleValueType();
24241   }
24242 
24243   // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit results,
24244   // so are legal and shouldn't occur here. AVX2/AVX512 pmovsx* instructions still
24245   // need to be handled here for 256/512-bit results.
24246   if (Subtarget.hasInt256()) {
24247     assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension");
24248 
24249     if (InVT.getVectorNumElements() != NumElts)
24250       return DAG.getNode(Op.getOpcode(), dl, VT, In);
24251 
24252     // FIXME: Apparently we create inreg operations that could be regular
24253     // extends.
24254     unsigned ExtOpc =
24255         Opc == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SIGN_EXTEND
24256                                              : ISD::ZERO_EXTEND;
24257     return DAG.getNode(ExtOpc, dl, VT, In);
24258   }
24259 
24260   // pre-AVX2 256-bit extensions need to be split into 128-bit instructions.
24261   if (Subtarget.hasAVX()) {
24262     assert(VT.is256BitVector() && "256-bit vector expected");
24263     MVT HalfVT = VT.getHalfNumVectorElementsVT();
24264     int HalfNumElts = HalfVT.getVectorNumElements();
24265 
24266     unsigned NumSrcElts = InVT.getVectorNumElements();
24267     SmallVector<int, 16> HiMask(NumSrcElts, SM_SentinelUndef);
24268     for (int i = 0; i != HalfNumElts; ++i)
24269       HiMask[i] = HalfNumElts + i;
24270 
24271     SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In);
24272     SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask);
24273     Hi = DAG.getNode(Opc, dl, HalfVT, Hi);
24274     return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
24275   }
24276 
24277   // We should only get here for sign extend.
24278   assert(Opc == ISD::SIGN_EXTEND_VECTOR_INREG && "Unexpected opcode!");
24279   assert(VT.is128BitVector() && InVT.is128BitVector() && "Unexpected VTs");
24280   unsigned InNumElts = InVT.getVectorNumElements();
24281 
24282   // If the source elements are already all-signbits, we don't need to extend,
24283   // just splat the elements.
24284   APInt DemandedElts = APInt::getLowBitsSet(InNumElts, NumElts);
24285   if (DAG.ComputeNumSignBits(In, DemandedElts) == InVT.getScalarSizeInBits()) {
24286     unsigned Scale = InNumElts / NumElts;
24287     SmallVector<int, 16> ShuffleMask;
24288     for (unsigned I = 0; I != NumElts; ++I)
24289       ShuffleMask.append(Scale, I);
24290     return DAG.getBitcast(VT,
24291                           DAG.getVectorShuffle(InVT, dl, In, In, ShuffleMask));
24292   }
24293 
24294   // pre-SSE41 targets unpack lower lanes and then sign-extend using SRAI.
24295   SDValue Curr = In;
24296   SDValue SignExt = Curr;
24297 
24298   // As SRAI is only available on i16/i32 types, we expand only up to i32
24299   // and handle i64 separately.
24300   if (InVT != MVT::v4i32) {
24301     MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT;
24302 
24303     unsigned DestWidth = DestVT.getScalarSizeInBits();
24304     unsigned Scale = DestWidth / InSVT.getSizeInBits();
24305     unsigned DestElts = DestVT.getVectorNumElements();
24306 
24307     // Build a shuffle mask that takes each input element and places it in the
24308     // MSBs of the new element size.
24309     SmallVector<int, 16> Mask(InNumElts, SM_SentinelUndef);
24310     for (unsigned i = 0; i != DestElts; ++i)
24311       Mask[i * Scale + (Scale - 1)] = i;
24312 
24313     Curr = DAG.getVectorShuffle(InVT, dl, In, In, Mask);
24314     Curr = DAG.getBitcast(DestVT, Curr);
24315 
24316     unsigned SignExtShift = DestWidth - InSVT.getSizeInBits();
24317     SignExt = DAG.getNode(X86ISD::VSRAI, dl, DestVT, Curr,
24318                           DAG.getTargetConstant(SignExtShift, dl, MVT::i8));
24319   }
24320 
24321   if (VT == MVT::v2i64) {
24322     assert(Curr.getValueType() == MVT::v4i32 && "Unexpected input VT");
24323     SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
24324     SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);
24325     SignExt = DAG.getVectorShuffle(MVT::v4i32, dl, SignExt, Sign, {0, 4, 1, 5});
24326     SignExt = DAG.getBitcast(VT, SignExt);
24327   }
24328 
24329   return SignExt;
24330 }
24331 
24332 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
24333                                 SelectionDAG &DAG) {
24334   MVT VT = Op->getSimpleValueType(0);
24335   SDValue In = Op->getOperand(0);
24336   MVT InVT = In.getSimpleValueType();
24337   SDLoc dl(Op);
24338 
24339   if (InVT.getVectorElementType() == MVT::i1)
24340     return LowerSIGN_EXTEND_Mask(Op, Subtarget, DAG);
24341 
24342   assert(VT.isVector() && InVT.isVector() && "Expected vector type");
24343   assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
24344          "Expected same number of elements");
24345   assert((VT.getVectorElementType() == MVT::i16 ||
24346           VT.getVectorElementType() == MVT::i32 ||
24347           VT.getVectorElementType() == MVT::i64) &&
24348          "Unexpected element type");
24349   assert((InVT.getVectorElementType() == MVT::i8 ||
24350           InVT.getVectorElementType() == MVT::i16 ||
24351           InVT.getVectorElementType() == MVT::i32) &&
24352          "Unexpected element type");
24353 
24354   if (VT == MVT::v32i16 && !Subtarget.hasBWI()) {
24355     assert(InVT == MVT::v32i8 && "Unexpected VT!");
24356     return splitVectorIntUnary(Op, DAG);
24357   }
24358 
24359   if (Subtarget.hasInt256())
24360     return Op;
24361 
24362   // Optimize vectors in AVX mode
24363   // Sign extend  v8i16 to v8i32 and
24364   //              v4i32 to v4i64
24365   //
24366   // Divide input vector into two parts
24367   // for v4i32 the high shuffle mask will be {2, 3, -1, -1}
24368   // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32
24369   // concat the vectors to original VT
24370   MVT HalfVT = VT.getHalfNumVectorElementsVT();
24371   SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
24372 
24373   unsigned NumElems = InVT.getVectorNumElements();
24374   SmallVector<int,8> ShufMask(NumElems, -1);
24375   for (unsigned i = 0; i != NumElems/2; ++i)
24376     ShufMask[i] = i + NumElems/2;
24377 
24378   SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
24379   OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi);
24380 
24381   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
24382 }
24383 
24384 /// Change a vector store into a pair of half-size vector stores.
24385 static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) {
24386   SDValue StoredVal = Store->getValue();
24387   assert((StoredVal.getValueType().is256BitVector() ||
24388           StoredVal.getValueType().is512BitVector()) &&
24389          "Expecting 256/512-bit op");
24390 
24391   // Splitting volatile memory ops is not allowed unless the operation was not
24392   // legal to begin with. Assume the input store is legal (this transform is
24393   // only used for targets with AVX). Note: It is possible that we have an
24394   // illegal type like v2i128, and so we could allow splitting a volatile store
24395   // in that case if that is important.
24396   if (!Store->isSimple())
24397     return SDValue();
24398 
24399   SDLoc DL(Store);
24400   SDValue Value0, Value1;
24401   std::tie(Value0, Value1) = splitVector(StoredVal, DAG, DL);
24402   unsigned HalfOffset = Value0.getValueType().getStoreSize();
24403   SDValue Ptr0 = Store->getBasePtr();
24404   SDValue Ptr1 =
24405       DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(HalfOffset), DL);
24406   SDValue Ch0 =
24407       DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(),
24408                    Store->getOriginalAlign(),
24409                    Store->getMemOperand()->getFlags());
24410   SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
24411                              Store->getPointerInfo().getWithOffset(HalfOffset),
24412                              Store->getOriginalAlign(),
24413                              Store->getMemOperand()->getFlags());
24414   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1);
24415 }
24416 
24417 /// Scalarize a vector store, bitcasting to TargetVT to determine the scalar
24418 /// type.
24419 static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT,
24420                                     SelectionDAG &DAG) {
24421   SDValue StoredVal = Store->getValue();
24422   assert(StoreVT.is128BitVector() &&
24423          StoredVal.getValueType().is128BitVector() && "Expecting 128-bit op");
24424   StoredVal = DAG.getBitcast(StoreVT, StoredVal);
24425 
24426   // Splitting volatile memory ops is not allowed unless the operation was not
24427   // legal to begin with. We are assuming the input op is legal (this transform
24428   // is only used for targets with AVX).
24429   if (!Store->isSimple())
24430     return SDValue();
24431 
24432   MVT StoreSVT = StoreVT.getScalarType();
24433   unsigned NumElems = StoreVT.getVectorNumElements();
24434   unsigned ScalarSize = StoreSVT.getStoreSize();
24435 
24436   SDLoc DL(Store);
24437   SmallVector<SDValue, 4> Stores;
24438   for (unsigned i = 0; i != NumElems; ++i) {
24439     unsigned Offset = i * ScalarSize;
24440     SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(),
24441                                            TypeSize::getFixed(Offset), DL);
24442     SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
24443                               DAG.getIntPtrConstant(i, DL));
24444     SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
24445                               Store->getPointerInfo().getWithOffset(Offset),
24446                               Store->getOriginalAlign(),
24447                               Store->getMemOperand()->getFlags());
24448     Stores.push_back(Ch);
24449   }
24450   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
24451 }
24452 
24453 static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
24454                           SelectionDAG &DAG) {
24455   StoreSDNode *St = cast<StoreSDNode>(Op.getNode());
24456   SDLoc dl(St);
24457   SDValue StoredVal = St->getValue();
24458 
24459   // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 stores.
24460   if (StoredVal.getValueType().isVector() &&
24461       StoredVal.getValueType().getVectorElementType() == MVT::i1) {
24462     unsigned NumElts = StoredVal.getValueType().getVectorNumElements();
24463     assert(NumElts <= 8 && "Unexpected VT");
24464     assert(!St->isTruncatingStore() && "Expected non-truncating store");
24465     assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
24466            "Expected AVX512F without AVX512DQI");
24467 
24468     // We must pad with zeros to ensure we store zeroes to any unused bits.
24469     StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
24470                             DAG.getUNDEF(MVT::v16i1), StoredVal,
24471                             DAG.getIntPtrConstant(0, dl));
24472     StoredVal = DAG.getBitcast(MVT::i16, StoredVal);
24473     StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal);
24474     // Make sure we store zeros in the extra bits.
24475     if (NumElts < 8)
24476       StoredVal = DAG.getZeroExtendInReg(
24477           StoredVal, dl, EVT::getIntegerVT(*DAG.getContext(), NumElts));
24478 
24479     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
24480                         St->getPointerInfo(), St->getOriginalAlign(),
24481                         St->getMemOperand()->getFlags());
24482   }
24483 
24484   if (St->isTruncatingStore())
24485     return SDValue();
24486 
24487   // If this is a 256-bit store of concatenated ops, we are better off splitting
24488   // that store into two 128-bit stores. This avoids spurious use of 256-bit ops
24489   // and each half can execute independently. Some cores would split the op into
24490   // halves anyway, so the concat (vinsertf128) is purely an extra op.
24491   MVT StoreVT = StoredVal.getSimpleValueType();
24492   if (StoreVT.is256BitVector() ||
24493       ((StoreVT == MVT::v32i16 || StoreVT == MVT::v64i8) &&
24494        !Subtarget.hasBWI())) {
24495     if (StoredVal.hasOneUse() && isFreeToSplitVector(StoredVal.getNode(), DAG))
24496       return splitVectorStore(St, DAG);
24497     return SDValue();
24498   }
24499 
24500   if (StoreVT.is32BitVector())
24501     return SDValue();
24502 
24503   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24504   assert(StoreVT.is64BitVector() && "Unexpected VT");
24505   assert(TLI.getTypeAction(*DAG.getContext(), StoreVT) ==
24506              TargetLowering::TypeWidenVector &&
24507          "Unexpected type action!");
24508 
24509   EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StoreVT);
24510   StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, StoredVal,
24511                           DAG.getUNDEF(StoreVT));
24512 
24513   if (Subtarget.hasSSE2()) {
24514     // Widen the vector, cast to a v2x64 type, extract the single 64-bit element
24515     // and store it.
24516     MVT StVT = Subtarget.is64Bit() && StoreVT.isInteger() ? MVT::i64 : MVT::f64;
24517     MVT CastVT = MVT::getVectorVT(StVT, 2);
24518     StoredVal = DAG.getBitcast(CastVT, StoredVal);
24519     StoredVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, StVT, StoredVal,
24520                             DAG.getIntPtrConstant(0, dl));
24521 
24522     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
24523                         St->getPointerInfo(), St->getOriginalAlign(),
24524                         St->getMemOperand()->getFlags());
24525   }
24526   assert(Subtarget.hasSSE1() && "Expected SSE");
24527   SDVTList Tys = DAG.getVTList(MVT::Other);
24528   SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()};
24529   return DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops, MVT::i64,
24530                                  St->getMemOperand());
24531 }
24532 
24533 // Lower vector extended loads using a shuffle. If SSSE3 is not available we
24534 // may emit an illegal shuffle but the expansion is still better than scalar
24535 // code. We generate sext/sext_invec for SEXTLOADs if it's available, otherwise
24536 // we'll emit a shuffle and a arithmetic shift.
24537 // FIXME: Is the expansion actually better than scalar code? It doesn't seem so.
24538 // TODO: It is possible to support ZExt by zeroing the undef values during
24539 // the shuffle phase or after the shuffle.
24540 static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
24541                                  SelectionDAG &DAG) {
24542   MVT RegVT = Op.getSimpleValueType();
24543   assert(RegVT.isVector() && "We only custom lower vector loads.");
24544   assert(RegVT.isInteger() &&
24545          "We only custom lower integer vector loads.");
24546 
24547   LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
24548   SDLoc dl(Ld);
24549 
24550   // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 loads.
24551   if (RegVT.getVectorElementType() == MVT::i1) {
24552     assert(EVT(RegVT) == Ld->getMemoryVT() && "Expected non-extending load");
24553     assert(RegVT.getVectorNumElements() <= 8 && "Unexpected VT");
24554     assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
24555            "Expected AVX512F without AVX512DQI");
24556 
24557     SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),
24558                                 Ld->getPointerInfo(), Ld->getOriginalAlign(),
24559                                 Ld->getMemOperand()->getFlags());
24560 
24561     // Replace chain users with the new chain.
24562     assert(NewLd->getNumValues() == 2 && "Loads must carry a chain!");
24563 
24564     SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
24565     Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT,
24566                       DAG.getBitcast(MVT::v16i1, Val),
24567                       DAG.getIntPtrConstant(0, dl));
24568     return DAG.getMergeValues({Val, NewLd.getValue(1)}, dl);
24569   }
24570 
24571   return SDValue();
24572 }
24573 
24574 /// Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes
24575 /// each of which has no other use apart from the AND / OR.
24576 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
24577   Opc = Op.getOpcode();
24578   if (Opc != ISD::OR && Opc != ISD::AND)
24579     return false;
24580   return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
24581           Op.getOperand(0).hasOneUse() &&
24582           Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
24583           Op.getOperand(1).hasOneUse());
24584 }
24585 
24586 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
24587   SDValue Chain = Op.getOperand(0);
24588   SDValue Cond  = Op.getOperand(1);
24589   SDValue Dest  = Op.getOperand(2);
24590   SDLoc dl(Op);
24591 
24592   // Bail out when we don't have native compare instructions.
24593   if (Cond.getOpcode() == ISD::SETCC &&
24594       Cond.getOperand(0).getValueType() != MVT::f128 &&
24595       !isSoftF16(Cond.getOperand(0).getValueType(), Subtarget)) {
24596     SDValue LHS = Cond.getOperand(0);
24597     SDValue RHS = Cond.getOperand(1);
24598     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
24599 
24600     // Special case for
24601     // setcc([su]{add,sub,mul}o == 0)
24602     // setcc([su]{add,sub,mul}o != 1)
24603     if (ISD::isOverflowIntrOpRes(LHS) &&
24604         (CC == ISD::SETEQ || CC == ISD::SETNE) &&
24605         (isNullConstant(RHS) || isOneConstant(RHS))) {
24606       SDValue Value, Overflow;
24607       X86::CondCode X86Cond;
24608       std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, LHS.getValue(0), DAG);
24609 
24610       if ((CC == ISD::SETEQ) == isNullConstant(RHS))
24611         X86Cond = X86::GetOppositeBranchCondition(X86Cond);
24612 
24613       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24614       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24615                          Overflow);
24616     }
24617 
24618     if (LHS.getSimpleValueType().isInteger()) {
24619       SDValue CCVal;
24620       SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
24621       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24622                          EFLAGS);
24623     }
24624 
24625     if (CC == ISD::SETOEQ) {
24626       // For FCMP_OEQ, we can emit
24627       // two branches instead of an explicit AND instruction with a
24628       // separate test. However, we only do this if this block doesn't
24629       // have a fall-through edge, because this requires an explicit
24630       // jmp when the condition is false.
24631       if (Op.getNode()->hasOneUse()) {
24632         SDNode *User = *Op.getNode()->use_begin();
24633         // Look for an unconditional branch following this conditional branch.
24634         // We need this because we need to reverse the successors in order
24635         // to implement FCMP_OEQ.
24636         if (User->getOpcode() == ISD::BR) {
24637           SDValue FalseBB = User->getOperand(1);
24638           SDNode *NewBR =
24639             DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
24640           assert(NewBR == User);
24641           (void)NewBR;
24642           Dest = FalseBB;
24643 
24644           SDValue Cmp =
24645               DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24646           SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
24647           Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest,
24648                               CCVal, Cmp);
24649           CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
24650           return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24651                              Cmp);
24652         }
24653       }
24654     } else if (CC == ISD::SETUNE) {
24655       // For FCMP_UNE, we can emit
24656       // two branches instead of an explicit OR instruction with a
24657       // separate test.
24658       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24659       SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
24660       Chain =
24661           DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp);
24662       CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
24663       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24664                          Cmp);
24665     } else {
24666       X86::CondCode X86Cond =
24667           TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG);
24668       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24669       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24670       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24671                          Cmp);
24672     }
24673   }
24674 
24675   if (ISD::isOverflowIntrOpRes(Cond)) {
24676     SDValue Value, Overflow;
24677     X86::CondCode X86Cond;
24678     std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
24679 
24680     SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24681     return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24682                        Overflow);
24683   }
24684 
24685   // Look past the truncate if the high bits are known zero.
24686   if (isTruncWithZeroHighBitsInput(Cond, DAG))
24687     Cond = Cond.getOperand(0);
24688 
24689   EVT CondVT = Cond.getValueType();
24690 
24691   // Add an AND with 1 if we don't already have one.
24692   if (!(Cond.getOpcode() == ISD::AND && isOneConstant(Cond.getOperand(1))))
24693     Cond =
24694         DAG.getNode(ISD::AND, dl, CondVT, Cond, DAG.getConstant(1, dl, CondVT));
24695 
24696   SDValue LHS = Cond;
24697   SDValue RHS = DAG.getConstant(0, dl, CondVT);
24698 
24699   SDValue CCVal;
24700   SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
24701   return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24702                      EFLAGS);
24703 }
24704 
24705 // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
24706 // Calls to _alloca are needed to probe the stack when allocating more than 4k
24707 // bytes in one go. Touching the stack at 4K increments is necessary to ensure
24708 // that the guard pages used by the OS virtual memory manager are allocated in
24709 // correct sequence.
24710 SDValue
24711 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
24712                                            SelectionDAG &DAG) const {
24713   MachineFunction &MF = DAG.getMachineFunction();
24714   bool SplitStack = MF.shouldSplitStack();
24715   bool EmitStackProbeCall = hasStackProbeSymbol(MF);
24716   bool Lower = (Subtarget.isOSWindows() && !Subtarget.isTargetMachO()) ||
24717                SplitStack || EmitStackProbeCall;
24718   SDLoc dl(Op);
24719 
24720   // Get the inputs.
24721   SDNode *Node = Op.getNode();
24722   SDValue Chain = Op.getOperand(0);
24723   SDValue Size  = Op.getOperand(1);
24724   MaybeAlign Alignment(Op.getConstantOperandVal(2));
24725   EVT VT = Node->getValueType(0);
24726 
24727   // Chain the dynamic stack allocation so that it doesn't modify the stack
24728   // pointer when other instructions are using the stack.
24729   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
24730 
24731   bool Is64Bit = Subtarget.is64Bit();
24732   MVT SPTy = getPointerTy(DAG.getDataLayout());
24733 
24734   SDValue Result;
24735   if (!Lower) {
24736     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24737     Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
24738     assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
24739                     " not tell us which reg is the stack pointer!");
24740 
24741     const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
24742     const Align StackAlign = TFI.getStackAlign();
24743     if (hasInlineStackProbe(MF)) {
24744       MachineRegisterInfo &MRI = MF.getRegInfo();
24745 
24746       const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
24747       Register Vreg = MRI.createVirtualRegister(AddrRegClass);
24748       Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
24749       Result = DAG.getNode(X86ISD::PROBED_ALLOCA, dl, SPTy, Chain,
24750                            DAG.getRegister(Vreg, SPTy));
24751     } else {
24752       SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
24753       Chain = SP.getValue(1);
24754       Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
24755     }
24756     if (Alignment && *Alignment > StackAlign)
24757       Result =
24758           DAG.getNode(ISD::AND, dl, VT, Result,
24759                       DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
24760     Chain = DAG.getCopyToReg(Chain, dl, SPReg, Result); // Output chain
24761   } else if (SplitStack) {
24762     MachineRegisterInfo &MRI = MF.getRegInfo();
24763 
24764     if (Is64Bit) {
24765       // The 64 bit implementation of segmented stacks needs to clobber both r10
24766       // r11. This makes it impossible to use it along with nested parameters.
24767       const Function &F = MF.getFunction();
24768       for (const auto &A : F.args()) {
24769         if (A.hasNestAttr())
24770           report_fatal_error("Cannot use segmented stacks with functions that "
24771                              "have nested arguments.");
24772       }
24773     }
24774 
24775     const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
24776     Register Vreg = MRI.createVirtualRegister(AddrRegClass);
24777     Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
24778     Result = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
24779                                 DAG.getRegister(Vreg, SPTy));
24780   } else {
24781     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
24782     Chain = DAG.getNode(X86ISD::DYN_ALLOCA, dl, NodeTys, Chain, Size);
24783     MF.getInfo<X86MachineFunctionInfo>()->setHasDynAlloca(true);
24784 
24785     const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
24786     Register SPReg = RegInfo->getStackRegister();
24787     SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
24788     Chain = SP.getValue(1);
24789 
24790     if (Alignment) {
24791       SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
24792                        DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
24793       Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
24794     }
24795 
24796     Result = SP;
24797   }
24798 
24799   Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
24800 
24801   SDValue Ops[2] = {Result, Chain};
24802   return DAG.getMergeValues(Ops, dl);
24803 }
24804 
24805 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
24806   MachineFunction &MF = DAG.getMachineFunction();
24807   auto PtrVT = getPointerTy(MF.getDataLayout());
24808   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
24809 
24810   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
24811   SDLoc DL(Op);
24812 
24813   if (!Subtarget.is64Bit() ||
24814       Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv())) {
24815     // vastart just stores the address of the VarArgsFrameIndex slot into the
24816     // memory location argument.
24817     SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
24818     return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
24819                         MachinePointerInfo(SV));
24820   }
24821 
24822   // __va_list_tag:
24823   //   gp_offset         (0 - 6 * 8)
24824   //   fp_offset         (48 - 48 + 8 * 16)
24825   //   overflow_arg_area (point to parameters coming in memory).
24826   //   reg_save_area
24827   SmallVector<SDValue, 8> MemOps;
24828   SDValue FIN = Op.getOperand(1);
24829   // Store gp_offset
24830   SDValue Store = DAG.getStore(
24831       Op.getOperand(0), DL,
24832       DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN,
24833       MachinePointerInfo(SV));
24834   MemOps.push_back(Store);
24835 
24836   // Store fp_offset
24837   FIN = DAG.getMemBasePlusOffset(FIN, TypeSize::getFixed(4), DL);
24838   Store = DAG.getStore(
24839       Op.getOperand(0), DL,
24840       DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN,
24841       MachinePointerInfo(SV, 4));
24842   MemOps.push_back(Store);
24843 
24844   // Store ptr to overflow_arg_area
24845   FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL));
24846   SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
24847   Store =
24848       DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, MachinePointerInfo(SV, 8));
24849   MemOps.push_back(Store);
24850 
24851   // Store ptr to reg_save_area.
24852   FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(
24853       Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
24854   SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
24855   Store = DAG.getStore(
24856       Op.getOperand(0), DL, RSFIN, FIN,
24857       MachinePointerInfo(SV, Subtarget.isTarget64BitLP64() ? 16 : 12));
24858   MemOps.push_back(Store);
24859   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
24860 }
24861 
24862 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
24863   assert(Subtarget.is64Bit() &&
24864          "LowerVAARG only handles 64-bit va_arg!");
24865   assert(Op.getNumOperands() == 4);
24866 
24867   MachineFunction &MF = DAG.getMachineFunction();
24868   if (Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()))
24869     // The Win64 ABI uses char* instead of a structure.
24870     return DAG.expandVAArg(Op.getNode());
24871 
24872   SDValue Chain = Op.getOperand(0);
24873   SDValue SrcPtr = Op.getOperand(1);
24874   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
24875   unsigned Align = Op.getConstantOperandVal(3);
24876   SDLoc dl(Op);
24877 
24878   EVT ArgVT = Op.getNode()->getValueType(0);
24879   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
24880   uint32_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
24881   uint8_t ArgMode;
24882 
24883   // Decide which area this value should be read from.
24884   // TODO: Implement the AMD64 ABI in its entirety. This simple
24885   // selection mechanism works only for the basic types.
24886   assert(ArgVT != MVT::f80 && "va_arg for f80 not yet implemented");
24887   if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) {
24888     ArgMode = 2;  // Argument passed in XMM register. Use fp_offset.
24889   } else {
24890     assert(ArgVT.isInteger() && ArgSize <= 32 /*bytes*/ &&
24891            "Unhandled argument type in LowerVAARG");
24892     ArgMode = 1;  // Argument passed in GPR64 register(s). Use gp_offset.
24893   }
24894 
24895   if (ArgMode == 2) {
24896     // Make sure using fp_offset makes sense.
24897     assert(!Subtarget.useSoftFloat() &&
24898            !(MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) &&
24899            Subtarget.hasSSE1());
24900   }
24901 
24902   // Insert VAARG node into the DAG
24903   // VAARG returns two values: Variable Argument Address, Chain
24904   SDValue InstOps[] = {Chain, SrcPtr,
24905                        DAG.getTargetConstant(ArgSize, dl, MVT::i32),
24906                        DAG.getTargetConstant(ArgMode, dl, MVT::i8),
24907                        DAG.getTargetConstant(Align, dl, MVT::i32)};
24908   SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
24909   SDValue VAARG = DAG.getMemIntrinsicNode(
24910       Subtarget.isTarget64BitLP64() ? X86ISD::VAARG_64 : X86ISD::VAARG_X32, dl,
24911       VTs, InstOps, MVT::i64, MachinePointerInfo(SV),
24912       /*Alignment=*/std::nullopt,
24913       MachineMemOperand::MOLoad | MachineMemOperand::MOStore);
24914   Chain = VAARG.getValue(1);
24915 
24916   // Load the next argument and return it
24917   return DAG.getLoad(ArgVT, dl, Chain, VAARG, MachinePointerInfo());
24918 }
24919 
24920 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
24921                            SelectionDAG &DAG) {
24922   // X86-64 va_list is a struct { i32, i32, i8*, i8* }, except on Windows,
24923   // where a va_list is still an i8*.
24924   assert(Subtarget.is64Bit() && "This code only handles 64-bit va_copy!");
24925   if (Subtarget.isCallingConvWin64(
24926         DAG.getMachineFunction().getFunction().getCallingConv()))
24927     // Probably a Win64 va_copy.
24928     return DAG.expandVACopy(Op.getNode());
24929 
24930   SDValue Chain = Op.getOperand(0);
24931   SDValue DstPtr = Op.getOperand(1);
24932   SDValue SrcPtr = Op.getOperand(2);
24933   const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
24934   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
24935   SDLoc DL(Op);
24936 
24937   return DAG.getMemcpy(
24938       Chain, DL, DstPtr, SrcPtr,
24939       DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL),
24940       Align(Subtarget.isTarget64BitLP64() ? 8 : 4), /*isVolatile*/ false, false,
24941       false, MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
24942 }
24943 
24944 // Helper to get immediate/variable SSE shift opcode from other shift opcodes.
24945 static unsigned getTargetVShiftUniformOpcode(unsigned Opc, bool IsVariable) {
24946   switch (Opc) {
24947   case ISD::SHL:
24948   case X86ISD::VSHL:
24949   case X86ISD::VSHLI:
24950     return IsVariable ? X86ISD::VSHL : X86ISD::VSHLI;
24951   case ISD::SRL:
24952   case X86ISD::VSRL:
24953   case X86ISD::VSRLI:
24954     return IsVariable ? X86ISD::VSRL : X86ISD::VSRLI;
24955   case ISD::SRA:
24956   case X86ISD::VSRA:
24957   case X86ISD::VSRAI:
24958     return IsVariable ? X86ISD::VSRA : X86ISD::VSRAI;
24959   }
24960   llvm_unreachable("Unknown target vector shift node");
24961 }
24962 
24963 /// Handle vector element shifts where the shift amount is a constant.
24964 /// Takes immediate version of shift as input.
24965 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
24966                                           SDValue SrcOp, uint64_t ShiftAmt,
24967                                           SelectionDAG &DAG) {
24968   MVT ElementType = VT.getVectorElementType();
24969 
24970   // Bitcast the source vector to the output type, this is mainly necessary for
24971   // vXi8/vXi64 shifts.
24972   if (VT != SrcOp.getSimpleValueType())
24973     SrcOp = DAG.getBitcast(VT, SrcOp);
24974 
24975   // Fold this packed shift into its first operand if ShiftAmt is 0.
24976   if (ShiftAmt == 0)
24977     return SrcOp;
24978 
24979   // Check for ShiftAmt >= element width
24980   if (ShiftAmt >= ElementType.getSizeInBits()) {
24981     if (Opc == X86ISD::VSRAI)
24982       ShiftAmt = ElementType.getSizeInBits() - 1;
24983     else
24984       return DAG.getConstant(0, dl, VT);
24985   }
24986 
24987   assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
24988          && "Unknown target vector shift-by-constant node");
24989 
24990   // Fold this packed vector shift into a build vector if SrcOp is a
24991   // vector of Constants or UNDEFs.
24992   if (ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
24993     unsigned ShiftOpc;
24994     switch (Opc) {
24995     default: llvm_unreachable("Unknown opcode!");
24996     case X86ISD::VSHLI:
24997       ShiftOpc = ISD::SHL;
24998       break;
24999     case X86ISD::VSRLI:
25000       ShiftOpc = ISD::SRL;
25001       break;
25002     case X86ISD::VSRAI:
25003       ShiftOpc = ISD::SRA;
25004       break;
25005     }
25006 
25007     SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT);
25008     if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt}))
25009       return C;
25010   }
25011 
25012   return DAG.getNode(Opc, dl, VT, SrcOp,
25013                      DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
25014 }
25015 
25016 /// Handle vector element shifts by a splat shift amount
25017 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
25018                                    SDValue SrcOp, SDValue ShAmt, int ShAmtIdx,
25019                                    const X86Subtarget &Subtarget,
25020                                    SelectionDAG &DAG) {
25021   MVT AmtVT = ShAmt.getSimpleValueType();
25022   assert(AmtVT.isVector() && "Vector shift type mismatch");
25023   assert(0 <= ShAmtIdx && ShAmtIdx < (int)AmtVT.getVectorNumElements() &&
25024          "Illegal vector splat index");
25025 
25026   // Move the splat element to the bottom element.
25027   if (ShAmtIdx != 0) {
25028     SmallVector<int> Mask(AmtVT.getVectorNumElements(), -1);
25029     Mask[0] = ShAmtIdx;
25030     ShAmt = DAG.getVectorShuffle(AmtVT, dl, ShAmt, DAG.getUNDEF(AmtVT), Mask);
25031   }
25032 
25033   // Peek through any zext node if we can get back to a 128-bit source.
25034   if (AmtVT.getScalarSizeInBits() == 64 &&
25035       (ShAmt.getOpcode() == ISD::ZERO_EXTEND ||
25036        ShAmt.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
25037       ShAmt.getOperand(0).getValueType().isSimple() &&
25038       ShAmt.getOperand(0).getValueType().is128BitVector()) {
25039     ShAmt = ShAmt.getOperand(0);
25040     AmtVT = ShAmt.getSimpleValueType();
25041   }
25042 
25043   // See if we can mask off the upper elements using the existing source node.
25044   // The shift uses the entire lower 64-bits of the amount vector, so no need to
25045   // do this for vXi64 types.
25046   bool IsMasked = false;
25047   if (AmtVT.getScalarSizeInBits() < 64) {
25048     if (ShAmt.getOpcode() == ISD::BUILD_VECTOR ||
25049         ShAmt.getOpcode() == ISD::SCALAR_TO_VECTOR) {
25050       // If the shift amount has come from a scalar, then zero-extend the scalar
25051       // before moving to the vector.
25052       ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32);
25053       ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
25054       ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt);
25055       AmtVT = MVT::v4i32;
25056       IsMasked = true;
25057     } else if (ShAmt.getOpcode() == ISD::AND) {
25058       // See if the shift amount is already masked (e.g. for rotation modulo),
25059       // then we can zero-extend it by setting all the other mask elements to
25060       // zero.
25061       SmallVector<SDValue> MaskElts(
25062           AmtVT.getVectorNumElements(),
25063           DAG.getConstant(0, dl, AmtVT.getScalarType()));
25064       MaskElts[0] = DAG.getAllOnesConstant(dl, AmtVT.getScalarType());
25065       SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts);
25066       if ((Mask = DAG.FoldConstantArithmetic(ISD::AND, dl, AmtVT,
25067                                              {ShAmt.getOperand(1), Mask}))) {
25068         ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask);
25069         IsMasked = true;
25070       }
25071     }
25072   }
25073 
25074   // Extract if the shift amount vector is larger than 128-bits.
25075   if (AmtVT.getSizeInBits() > 128) {
25076     ShAmt = extract128BitVector(ShAmt, 0, DAG, dl);
25077     AmtVT = ShAmt.getSimpleValueType();
25078   }
25079 
25080   // Zero-extend bottom element to v2i64 vector type, either by extension or
25081   // shuffle masking.
25082   if (!IsMasked && AmtVT.getScalarSizeInBits() < 64) {
25083     if (AmtVT == MVT::v4i32 && (ShAmt.getOpcode() == X86ISD::VBROADCAST ||
25084                                 ShAmt.getOpcode() == X86ISD::VBROADCAST_LOAD)) {
25085       ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt);
25086     } else if (Subtarget.hasSSE41()) {
25087       ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
25088                           MVT::v2i64, ShAmt);
25089     } else {
25090       SDValue ByteShift = DAG.getTargetConstant(
25091           (128 - AmtVT.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);
25092       ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
25093       ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
25094                           ByteShift);
25095       ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
25096                           ByteShift);
25097     }
25098   }
25099 
25100   // Change opcode to non-immediate version.
25101   Opc = getTargetVShiftUniformOpcode(Opc, true);
25102 
25103   // The return type has to be a 128-bit type with the same element
25104   // type as the input type.
25105   MVT EltVT = VT.getVectorElementType();
25106   MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits());
25107 
25108   ShAmt = DAG.getBitcast(ShVT, ShAmt);
25109   return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
25110 }
25111 
25112 /// Return Mask with the necessary casting or extending
25113 /// for \p Mask according to \p MaskVT when lowering masking intrinsics
25114 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
25115                            const X86Subtarget &Subtarget, SelectionDAG &DAG,
25116                            const SDLoc &dl) {
25117 
25118   if (isAllOnesConstant(Mask))
25119     return DAG.getConstant(1, dl, MaskVT);
25120   if (X86::isZeroNode(Mask))
25121     return DAG.getConstant(0, dl, MaskVT);
25122 
25123   assert(MaskVT.bitsLE(Mask.getSimpleValueType()) && "Unexpected mask size!");
25124 
25125   if (Mask.getSimpleValueType() == MVT::i64 && Subtarget.is32Bit()) {
25126     assert(MaskVT == MVT::v64i1 && "Expected v64i1 mask!");
25127     assert(Subtarget.hasBWI() && "Expected AVX512BW target!");
25128     // In case 32bit mode, bitcast i64 is illegal, extend/split it.
25129     SDValue Lo, Hi;
25130     std::tie(Lo, Hi) = DAG.SplitScalar(Mask, dl, MVT::i32, MVT::i32);
25131     Lo = DAG.getBitcast(MVT::v32i1, Lo);
25132     Hi = DAG.getBitcast(MVT::v32i1, Hi);
25133     return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
25134   } else {
25135     MVT BitcastVT = MVT::getVectorVT(MVT::i1,
25136                                      Mask.getSimpleValueType().getSizeInBits());
25137     // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
25138     // are extracted by EXTRACT_SUBVECTOR.
25139     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
25140                        DAG.getBitcast(BitcastVT, Mask),
25141                        DAG.getIntPtrConstant(0, dl));
25142   }
25143 }
25144 
25145 /// Return (and \p Op, \p Mask) for compare instructions or
25146 /// (vselect \p Mask, \p Op, \p PreservedSrc) for others along with the
25147 /// necessary casting or extending for \p Mask when lowering masking intrinsics
25148 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
25149                                     SDValue PreservedSrc,
25150                                     const X86Subtarget &Subtarget,
25151                                     SelectionDAG &DAG) {
25152   MVT VT = Op.getSimpleValueType();
25153   MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
25154   unsigned OpcodeSelect = ISD::VSELECT;
25155   SDLoc dl(Op);
25156 
25157   if (isAllOnesConstant(Mask))
25158     return Op;
25159 
25160   SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25161 
25162   if (PreservedSrc.isUndef())
25163     PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
25164   return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);
25165 }
25166 
25167 /// Creates an SDNode for a predicated scalar operation.
25168 /// \returns (X86vselect \p Mask, \p Op, \p PreservedSrc).
25169 /// The mask is coming as MVT::i8 and it should be transformed
25170 /// to MVT::v1i1 while lowering masking intrinsics.
25171 /// The main difference between ScalarMaskingNode and VectorMaskingNode is using
25172 /// "X86select" instead of "vselect". We just can't create the "vselect" node
25173 /// for a scalar instruction.
25174 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
25175                                     SDValue PreservedSrc,
25176                                     const X86Subtarget &Subtarget,
25177                                     SelectionDAG &DAG) {
25178 
25179   if (auto *MaskConst = dyn_cast<ConstantSDNode>(Mask))
25180     if (MaskConst->getZExtValue() & 0x1)
25181       return Op;
25182 
25183   MVT VT = Op.getSimpleValueType();
25184   SDLoc dl(Op);
25185 
25186   assert(Mask.getValueType() == MVT::i8 && "Unexpect type");
25187   SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,
25188                               DAG.getBitcast(MVT::v8i1, Mask),
25189                               DAG.getIntPtrConstant(0, dl));
25190   if (Op.getOpcode() == X86ISD::FSETCCM ||
25191       Op.getOpcode() == X86ISD::FSETCCM_SAE ||
25192       Op.getOpcode() == X86ISD::VFPCLASSS)
25193     return DAG.getNode(ISD::AND, dl, VT, Op, IMask);
25194 
25195   if (PreservedSrc.isUndef())
25196     PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
25197   return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc);
25198 }
25199 
25200 static int getSEHRegistrationNodeSize(const Function *Fn) {
25201   if (!Fn->hasPersonalityFn())
25202     report_fatal_error(
25203         "querying registration node size for function without personality");
25204   // The RegNodeSize is 6 32-bit words for SEH and 4 for C++ EH. See
25205   // WinEHStatePass for the full struct definition.
25206   switch (classifyEHPersonality(Fn->getPersonalityFn())) {
25207   case EHPersonality::MSVC_X86SEH: return 24;
25208   case EHPersonality::MSVC_CXX: return 16;
25209   default: break;
25210   }
25211   report_fatal_error(
25212       "can only recover FP for 32-bit MSVC EH personality functions");
25213 }
25214 
25215 /// When the MSVC runtime transfers control to us, either to an outlined
25216 /// function or when returning to a parent frame after catching an exception, we
25217 /// recover the parent frame pointer by doing arithmetic on the incoming EBP.
25218 /// Here's the math:
25219 ///   RegNodeBase = EntryEBP - RegNodeSize
25220 ///   ParentFP = RegNodeBase - ParentFrameOffset
25221 /// Subtracting RegNodeSize takes us to the offset of the registration node, and
25222 /// subtracting the offset (negative on x86) takes us back to the parent FP.
25223 static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
25224                                    SDValue EntryEBP) {
25225   MachineFunction &MF = DAG.getMachineFunction();
25226   SDLoc dl;
25227 
25228   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25229   MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
25230 
25231   // It's possible that the parent function no longer has a personality function
25232   // if the exceptional code was optimized away, in which case we just return
25233   // the incoming EBP.
25234   if (!Fn->hasPersonalityFn())
25235     return EntryEBP;
25236 
25237   // Get an MCSymbol that will ultimately resolve to the frame offset of the EH
25238   // registration, or the .set_setframe offset.
25239   MCSymbol *OffsetSym =
25240       MF.getMMI().getContext().getOrCreateParentFrameOffsetSymbol(
25241           GlobalValue::dropLLVMManglingEscape(Fn->getName()));
25242   SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
25243   SDValue ParentFrameOffset =
25244       DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
25245 
25246   // Return EntryEBP + ParentFrameOffset for x64. This adjusts from RSP after
25247   // prologue to RBP in the parent function.
25248   const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
25249   if (Subtarget.is64Bit())
25250     return DAG.getNode(ISD::ADD, dl, PtrVT, EntryEBP, ParentFrameOffset);
25251 
25252   int RegNodeSize = getSEHRegistrationNodeSize(Fn);
25253   // RegNodeBase = EntryEBP - RegNodeSize
25254   // ParentFP = RegNodeBase - ParentFrameOffset
25255   SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
25256                                     DAG.getConstant(RegNodeSize, dl, PtrVT));
25257   return DAG.getNode(ISD::SUB, dl, PtrVT, RegNodeBase, ParentFrameOffset);
25258 }
25259 
25260 SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
25261                                                    SelectionDAG &DAG) const {
25262   // Helper to detect if the operand is CUR_DIRECTION rounding mode.
25263   auto isRoundModeCurDirection = [](SDValue Rnd) {
25264     if (auto *C = dyn_cast<ConstantSDNode>(Rnd))
25265       return C->getAPIntValue() == X86::STATIC_ROUNDING::CUR_DIRECTION;
25266 
25267     return false;
25268   };
25269   auto isRoundModeSAE = [](SDValue Rnd) {
25270     if (auto *C = dyn_cast<ConstantSDNode>(Rnd)) {
25271       unsigned RC = C->getZExtValue();
25272       if (RC & X86::STATIC_ROUNDING::NO_EXC) {
25273         // Clear the NO_EXC bit and check remaining bits.
25274         RC ^= X86::STATIC_ROUNDING::NO_EXC;
25275         // As a convenience we allow no other bits or explicitly
25276         // current direction.
25277         return RC == 0 || RC == X86::STATIC_ROUNDING::CUR_DIRECTION;
25278       }
25279     }
25280 
25281     return false;
25282   };
25283   auto isRoundModeSAEToX = [](SDValue Rnd, unsigned &RC) {
25284     if (auto *C = dyn_cast<ConstantSDNode>(Rnd)) {
25285       RC = C->getZExtValue();
25286       if (RC & X86::STATIC_ROUNDING::NO_EXC) {
25287         // Clear the NO_EXC bit and check remaining bits.
25288         RC ^= X86::STATIC_ROUNDING::NO_EXC;
25289         return RC == X86::STATIC_ROUNDING::TO_NEAREST_INT ||
25290                RC == X86::STATIC_ROUNDING::TO_NEG_INF ||
25291                RC == X86::STATIC_ROUNDING::TO_POS_INF ||
25292                RC == X86::STATIC_ROUNDING::TO_ZERO;
25293       }
25294     }
25295 
25296     return false;
25297   };
25298 
25299   SDLoc dl(Op);
25300   unsigned IntNo = Op.getConstantOperandVal(0);
25301   MVT VT = Op.getSimpleValueType();
25302   const IntrinsicData* IntrData = getIntrinsicWithoutChain(IntNo);
25303 
25304   // Propagate flags from original node to transformed node(s).
25305   SelectionDAG::FlagInserter FlagsInserter(DAG, Op->getFlags());
25306 
25307   if (IntrData) {
25308     switch(IntrData->Type) {
25309     case INTR_TYPE_1OP: {
25310       // We specify 2 possible opcodes for intrinsics with rounding modes.
25311       // First, we check if the intrinsic may have non-default rounding mode,
25312       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25313       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25314       if (IntrWithRoundingModeOpcode != 0) {
25315         SDValue Rnd = Op.getOperand(2);
25316         unsigned RC = 0;
25317         if (isRoundModeSAEToX(Rnd, RC))
25318           return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25319                              Op.getOperand(1),
25320                              DAG.getTargetConstant(RC, dl, MVT::i32));
25321         if (!isRoundModeCurDirection(Rnd))
25322           return SDValue();
25323       }
25324       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25325                          Op.getOperand(1));
25326     }
25327     case INTR_TYPE_1OP_SAE: {
25328       SDValue Sae = Op.getOperand(2);
25329 
25330       unsigned Opc;
25331       if (isRoundModeCurDirection(Sae))
25332         Opc = IntrData->Opc0;
25333       else if (isRoundModeSAE(Sae))
25334         Opc = IntrData->Opc1;
25335       else
25336         return SDValue();
25337 
25338       return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1));
25339     }
25340     case INTR_TYPE_2OP: {
25341       SDValue Src2 = Op.getOperand(2);
25342 
25343       // We specify 2 possible opcodes for intrinsics with rounding modes.
25344       // First, we check if the intrinsic may have non-default rounding mode,
25345       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25346       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25347       if (IntrWithRoundingModeOpcode != 0) {
25348         SDValue Rnd = Op.getOperand(3);
25349         unsigned RC = 0;
25350         if (isRoundModeSAEToX(Rnd, RC))
25351           return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25352                              Op.getOperand(1), Src2,
25353                              DAG.getTargetConstant(RC, dl, MVT::i32));
25354         if (!isRoundModeCurDirection(Rnd))
25355           return SDValue();
25356       }
25357 
25358       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25359                          Op.getOperand(1), Src2);
25360     }
25361     case INTR_TYPE_2OP_SAE: {
25362       SDValue Sae = Op.getOperand(3);
25363 
25364       unsigned Opc;
25365       if (isRoundModeCurDirection(Sae))
25366         Opc = IntrData->Opc0;
25367       else if (isRoundModeSAE(Sae))
25368         Opc = IntrData->Opc1;
25369       else
25370         return SDValue();
25371 
25372       return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
25373                          Op.getOperand(2));
25374     }
25375     case INTR_TYPE_3OP:
25376     case INTR_TYPE_3OP_IMM8: {
25377       SDValue Src1 = Op.getOperand(1);
25378       SDValue Src2 = Op.getOperand(2);
25379       SDValue Src3 = Op.getOperand(3);
25380 
25381       if (IntrData->Type == INTR_TYPE_3OP_IMM8 &&
25382           Src3.getValueType() != MVT::i8) {
25383         Src3 = DAG.getTargetConstant(Src3->getAsZExtVal() & 0xff, dl, MVT::i8);
25384       }
25385 
25386       // We specify 2 possible opcodes for intrinsics with rounding modes.
25387       // First, we check if the intrinsic may have non-default rounding mode,
25388       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25389       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25390       if (IntrWithRoundingModeOpcode != 0) {
25391         SDValue Rnd = Op.getOperand(4);
25392         unsigned RC = 0;
25393         if (isRoundModeSAEToX(Rnd, RC))
25394           return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25395                              Src1, Src2, Src3,
25396                              DAG.getTargetConstant(RC, dl, MVT::i32));
25397         if (!isRoundModeCurDirection(Rnd))
25398           return SDValue();
25399       }
25400 
25401       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25402                          {Src1, Src2, Src3});
25403     }
25404     case INTR_TYPE_4OP_IMM8: {
25405       assert(Op.getOperand(4)->getOpcode() == ISD::TargetConstant);
25406       SDValue Src4 = Op.getOperand(4);
25407       if (Src4.getValueType() != MVT::i8) {
25408         Src4 = DAG.getTargetConstant(Src4->getAsZExtVal() & 0xff, dl, MVT::i8);
25409       }
25410 
25411       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25412                          Op.getOperand(1), Op.getOperand(2), Op.getOperand(3),
25413                          Src4);
25414     }
25415     case INTR_TYPE_1OP_MASK: {
25416       SDValue Src = Op.getOperand(1);
25417       SDValue PassThru = Op.getOperand(2);
25418       SDValue Mask = Op.getOperand(3);
25419       // We add rounding mode to the Node when
25420       //   - RC Opcode is specified and
25421       //   - RC is not "current direction".
25422       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25423       if (IntrWithRoundingModeOpcode != 0) {
25424         SDValue Rnd = Op.getOperand(4);
25425         unsigned RC = 0;
25426         if (isRoundModeSAEToX(Rnd, RC))
25427           return getVectorMaskingNode(
25428               DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25429                           Src, DAG.getTargetConstant(RC, dl, MVT::i32)),
25430               Mask, PassThru, Subtarget, DAG);
25431         if (!isRoundModeCurDirection(Rnd))
25432           return SDValue();
25433       }
25434       return getVectorMaskingNode(
25435           DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru,
25436           Subtarget, DAG);
25437     }
25438     case INTR_TYPE_1OP_MASK_SAE: {
25439       SDValue Src = Op.getOperand(1);
25440       SDValue PassThru = Op.getOperand(2);
25441       SDValue Mask = Op.getOperand(3);
25442       SDValue Rnd = Op.getOperand(4);
25443 
25444       unsigned Opc;
25445       if (isRoundModeCurDirection(Rnd))
25446         Opc = IntrData->Opc0;
25447       else if (isRoundModeSAE(Rnd))
25448         Opc = IntrData->Opc1;
25449       else
25450         return SDValue();
25451 
25452       return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru,
25453                                   Subtarget, DAG);
25454     }
25455     case INTR_TYPE_SCALAR_MASK: {
25456       SDValue Src1 = Op.getOperand(1);
25457       SDValue Src2 = Op.getOperand(2);
25458       SDValue passThru = Op.getOperand(3);
25459       SDValue Mask = Op.getOperand(4);
25460       unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25461       // There are 2 kinds of intrinsics in this group:
25462       // (1) With suppress-all-exceptions (sae) or rounding mode- 6 operands
25463       // (2) With rounding mode and sae - 7 operands.
25464       bool HasRounding = IntrWithRoundingModeOpcode != 0;
25465       if (Op.getNumOperands() == (5U + HasRounding)) {
25466         if (HasRounding) {
25467           SDValue Rnd = Op.getOperand(5);
25468           unsigned RC = 0;
25469           if (isRoundModeSAEToX(Rnd, RC))
25470             return getScalarMaskingNode(
25471                 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2,
25472                             DAG.getTargetConstant(RC, dl, MVT::i32)),
25473                 Mask, passThru, Subtarget, DAG);
25474           if (!isRoundModeCurDirection(Rnd))
25475             return SDValue();
25476         }
25477         return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,
25478                                                 Src2),
25479                                     Mask, passThru, Subtarget, DAG);
25480       }
25481 
25482       assert(Op.getNumOperands() == (6U + HasRounding) &&
25483              "Unexpected intrinsic form");
25484       SDValue RoundingMode = Op.getOperand(5);
25485       unsigned Opc = IntrData->Opc0;
25486       if (HasRounding) {
25487         SDValue Sae = Op.getOperand(6);
25488         if (isRoundModeSAE(Sae))
25489           Opc = IntrWithRoundingModeOpcode;
25490         else if (!isRoundModeCurDirection(Sae))
25491           return SDValue();
25492       }
25493       return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1,
25494                                               Src2, RoundingMode),
25495                                   Mask, passThru, Subtarget, DAG);
25496     }
25497     case INTR_TYPE_SCALAR_MASK_RND: {
25498       SDValue Src1 = Op.getOperand(1);
25499       SDValue Src2 = Op.getOperand(2);
25500       SDValue passThru = Op.getOperand(3);
25501       SDValue Mask = Op.getOperand(4);
25502       SDValue Rnd = Op.getOperand(5);
25503 
25504       SDValue NewOp;
25505       unsigned RC = 0;
25506       if (isRoundModeCurDirection(Rnd))
25507         NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
25508       else if (isRoundModeSAEToX(Rnd, RC))
25509         NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
25510                             DAG.getTargetConstant(RC, dl, MVT::i32));
25511       else
25512         return SDValue();
25513 
25514       return getScalarMaskingNode(NewOp, Mask, passThru, Subtarget, DAG);
25515     }
25516     case INTR_TYPE_SCALAR_MASK_SAE: {
25517       SDValue Src1 = Op.getOperand(1);
25518       SDValue Src2 = Op.getOperand(2);
25519       SDValue passThru = Op.getOperand(3);
25520       SDValue Mask = Op.getOperand(4);
25521       SDValue Sae = Op.getOperand(5);
25522       unsigned Opc;
25523       if (isRoundModeCurDirection(Sae))
25524         Opc = IntrData->Opc0;
25525       else if (isRoundModeSAE(Sae))
25526         Opc = IntrData->Opc1;
25527       else
25528         return SDValue();
25529 
25530       return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
25531                                   Mask, passThru, Subtarget, DAG);
25532     }
25533     case INTR_TYPE_2OP_MASK: {
25534       SDValue Src1 = Op.getOperand(1);
25535       SDValue Src2 = Op.getOperand(2);
25536       SDValue PassThru = Op.getOperand(3);
25537       SDValue Mask = Op.getOperand(4);
25538       SDValue NewOp;
25539       if (IntrData->Opc1 != 0) {
25540         SDValue Rnd = Op.getOperand(5);
25541         unsigned RC = 0;
25542         if (isRoundModeSAEToX(Rnd, RC))
25543           NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
25544                               DAG.getTargetConstant(RC, dl, MVT::i32));
25545         else if (!isRoundModeCurDirection(Rnd))
25546           return SDValue();
25547       }
25548       if (!NewOp)
25549         NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
25550       return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
25551     }
25552     case INTR_TYPE_2OP_MASK_SAE: {
25553       SDValue Src1 = Op.getOperand(1);
25554       SDValue Src2 = Op.getOperand(2);
25555       SDValue PassThru = Op.getOperand(3);
25556       SDValue Mask = Op.getOperand(4);
25557 
25558       unsigned Opc = IntrData->Opc0;
25559       if (IntrData->Opc1 != 0) {
25560         SDValue Sae = Op.getOperand(5);
25561         if (isRoundModeSAE(Sae))
25562           Opc = IntrData->Opc1;
25563         else if (!isRoundModeCurDirection(Sae))
25564           return SDValue();
25565       }
25566 
25567       return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
25568                                   Mask, PassThru, Subtarget, DAG);
25569     }
25570     case INTR_TYPE_3OP_SCALAR_MASK_SAE: {
25571       SDValue Src1 = Op.getOperand(1);
25572       SDValue Src2 = Op.getOperand(2);
25573       SDValue Src3 = Op.getOperand(3);
25574       SDValue PassThru = Op.getOperand(4);
25575       SDValue Mask = Op.getOperand(5);
25576       SDValue Sae = Op.getOperand(6);
25577       unsigned Opc;
25578       if (isRoundModeCurDirection(Sae))
25579         Opc = IntrData->Opc0;
25580       else if (isRoundModeSAE(Sae))
25581         Opc = IntrData->Opc1;
25582       else
25583         return SDValue();
25584 
25585       return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
25586                                   Mask, PassThru, Subtarget, DAG);
25587     }
25588     case INTR_TYPE_3OP_MASK_SAE: {
25589       SDValue Src1 = Op.getOperand(1);
25590       SDValue Src2 = Op.getOperand(2);
25591       SDValue Src3 = Op.getOperand(3);
25592       SDValue PassThru = Op.getOperand(4);
25593       SDValue Mask = Op.getOperand(5);
25594 
25595       unsigned Opc = IntrData->Opc0;
25596       if (IntrData->Opc1 != 0) {
25597         SDValue Sae = Op.getOperand(6);
25598         if (isRoundModeSAE(Sae))
25599           Opc = IntrData->Opc1;
25600         else if (!isRoundModeCurDirection(Sae))
25601           return SDValue();
25602       }
25603       return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
25604                                   Mask, PassThru, Subtarget, DAG);
25605     }
25606     case BLENDV: {
25607       SDValue Src1 = Op.getOperand(1);
25608       SDValue Src2 = Op.getOperand(2);
25609       SDValue Src3 = Op.getOperand(3);
25610 
25611       EVT MaskVT = Src3.getValueType().changeVectorElementTypeToInteger();
25612       Src3 = DAG.getBitcast(MaskVT, Src3);
25613 
25614       // Reverse the operands to match VSELECT order.
25615       return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1);
25616     }
25617     case VPERM_2OP : {
25618       SDValue Src1 = Op.getOperand(1);
25619       SDValue Src2 = Op.getOperand(2);
25620 
25621       // Swap Src1 and Src2 in the node creation
25622       return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
25623     }
25624     case CFMA_OP_MASKZ:
25625     case CFMA_OP_MASK: {
25626       SDValue Src1 = Op.getOperand(1);
25627       SDValue Src2 = Op.getOperand(2);
25628       SDValue Src3 = Op.getOperand(3);
25629       SDValue Mask = Op.getOperand(4);
25630       MVT VT = Op.getSimpleValueType();
25631 
25632       SDValue PassThru = Src3;
25633       if (IntrData->Type == CFMA_OP_MASKZ)
25634         PassThru = getZeroVector(VT, Subtarget, DAG, dl);
25635 
25636       // We add rounding mode to the Node when
25637       //   - RC Opcode is specified and
25638       //   - RC is not "current direction".
25639       SDValue NewOp;
25640       if (IntrData->Opc1 != 0) {
25641         SDValue Rnd = Op.getOperand(5);
25642         unsigned RC = 0;
25643         if (isRoundModeSAEToX(Rnd, RC))
25644           NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3,
25645                               DAG.getTargetConstant(RC, dl, MVT::i32));
25646         else if (!isRoundModeCurDirection(Rnd))
25647           return SDValue();
25648       }
25649       if (!NewOp)
25650         NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3);
25651       return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
25652     }
25653     case IFMA_OP:
25654       // NOTE: We need to swizzle the operands to pass the multiply operands
25655       // first.
25656       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25657                          Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
25658     case FPCLASSS: {
25659       SDValue Src1 = Op.getOperand(1);
25660       SDValue Imm = Op.getOperand(2);
25661       SDValue Mask = Op.getOperand(3);
25662       SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
25663       SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
25664                                                  Subtarget, DAG);
25665       // Need to fill with zeros to ensure the bitcast will produce zeroes
25666       // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25667       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
25668                                 DAG.getConstant(0, dl, MVT::v8i1),
25669                                 FPclassMask, DAG.getIntPtrConstant(0, dl));
25670       return DAG.getBitcast(MVT::i8, Ins);
25671     }
25672 
25673     case CMP_MASK_CC: {
25674       MVT MaskVT = Op.getSimpleValueType();
25675       SDValue CC = Op.getOperand(3);
25676       SDValue Mask = Op.getOperand(4);
25677       // We specify 2 possible opcodes for intrinsics with rounding modes.
25678       // First, we check if the intrinsic may have non-default rounding mode,
25679       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25680       if (IntrData->Opc1 != 0) {
25681         SDValue Sae = Op.getOperand(5);
25682         if (isRoundModeSAE(Sae))
25683           return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1),
25684                              Op.getOperand(2), CC, Mask, Sae);
25685         if (!isRoundModeCurDirection(Sae))
25686           return SDValue();
25687       }
25688       //default rounding mode
25689       return DAG.getNode(IntrData->Opc0, dl, MaskVT,
25690                          {Op.getOperand(1), Op.getOperand(2), CC, Mask});
25691     }
25692     case CMP_MASK_SCALAR_CC: {
25693       SDValue Src1 = Op.getOperand(1);
25694       SDValue Src2 = Op.getOperand(2);
25695       SDValue CC = Op.getOperand(3);
25696       SDValue Mask = Op.getOperand(4);
25697 
25698       SDValue Cmp;
25699       if (IntrData->Opc1 != 0) {
25700         SDValue Sae = Op.getOperand(5);
25701         if (isRoundModeSAE(Sae))
25702           Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae);
25703         else if (!isRoundModeCurDirection(Sae))
25704           return SDValue();
25705       }
25706       //default rounding mode
25707       if (!Cmp.getNode())
25708         Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);
25709 
25710       SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
25711                                              Subtarget, DAG);
25712       // Need to fill with zeros to ensure the bitcast will produce zeroes
25713       // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25714       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
25715                                 DAG.getConstant(0, dl, MVT::v8i1),
25716                                 CmpMask, DAG.getIntPtrConstant(0, dl));
25717       return DAG.getBitcast(MVT::i8, Ins);
25718     }
25719     case COMI: { // Comparison intrinsics
25720       ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1;
25721       SDValue LHS = Op.getOperand(1);
25722       SDValue RHS = Op.getOperand(2);
25723       // Some conditions require the operands to be swapped.
25724       if (CC == ISD::SETLT || CC == ISD::SETLE)
25725         std::swap(LHS, RHS);
25726 
25727       SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
25728       SDValue SetCC;
25729       switch (CC) {
25730       case ISD::SETEQ: { // (ZF = 0 and PF = 0)
25731         SetCC = getSETCC(X86::COND_E, Comi, dl, DAG);
25732         SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
25733         SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP);
25734         break;
25735       }
25736       case ISD::SETNE: { // (ZF = 1 or PF = 1)
25737         SetCC = getSETCC(X86::COND_NE, Comi, dl, DAG);
25738         SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
25739         SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP);
25740         break;
25741       }
25742       case ISD::SETGT: // (CF = 0 and ZF = 0)
25743       case ISD::SETLT: { // Condition opposite to GT. Operands swapped above.
25744         SetCC = getSETCC(X86::COND_A, Comi, dl, DAG);
25745         break;
25746       }
25747       case ISD::SETGE: // CF = 0
25748       case ISD::SETLE: // Condition opposite to GE. Operands swapped above.
25749         SetCC = getSETCC(X86::COND_AE, Comi, dl, DAG);
25750         break;
25751       default:
25752         llvm_unreachable("Unexpected illegal condition!");
25753       }
25754       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
25755     }
25756     case COMI_RM: { // Comparison intrinsics with Sae
25757       SDValue LHS = Op.getOperand(1);
25758       SDValue RHS = Op.getOperand(2);
25759       unsigned CondVal = Op.getConstantOperandVal(3);
25760       SDValue Sae = Op.getOperand(4);
25761 
25762       SDValue FCmp;
25763       if (isRoundModeCurDirection(Sae))
25764         FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS,
25765                            DAG.getTargetConstant(CondVal, dl, MVT::i8));
25766       else if (isRoundModeSAE(Sae))
25767         FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS,
25768                            DAG.getTargetConstant(CondVal, dl, MVT::i8), Sae);
25769       else
25770         return SDValue();
25771       // Need to fill with zeros to ensure the bitcast will produce zeroes
25772       // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25773       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
25774                                 DAG.getConstant(0, dl, MVT::v16i1),
25775                                 FCmp, DAG.getIntPtrConstant(0, dl));
25776       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32,
25777                          DAG.getBitcast(MVT::i16, Ins));
25778     }
25779     case VSHIFT: {
25780       SDValue SrcOp = Op.getOperand(1);
25781       SDValue ShAmt = Op.getOperand(2);
25782       assert(ShAmt.getValueType() == MVT::i32 &&
25783              "Unexpected VSHIFT amount type");
25784 
25785       // Catch shift-by-constant.
25786       if (auto *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
25787         return getTargetVShiftByConstNode(IntrData->Opc0, dl,
25788                                           Op.getSimpleValueType(), SrcOp,
25789                                           CShAmt->getZExtValue(), DAG);
25790 
25791       ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
25792       return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(),
25793                                  SrcOp, ShAmt, 0, Subtarget, DAG);
25794     }
25795     case COMPRESS_EXPAND_IN_REG: {
25796       SDValue Mask = Op.getOperand(3);
25797       SDValue DataToCompress = Op.getOperand(1);
25798       SDValue PassThru = Op.getOperand(2);
25799       if (ISD::isBuildVectorAllOnes(Mask.getNode())) // return data as is
25800         return Op.getOperand(1);
25801 
25802       // Avoid false dependency.
25803       if (PassThru.isUndef())
25804         PassThru = getZeroVector(VT, Subtarget, DAG, dl);
25805 
25806       return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru,
25807                          Mask);
25808     }
25809     case FIXUPIMM:
25810     case FIXUPIMM_MASKZ: {
25811       SDValue Src1 = Op.getOperand(1);
25812       SDValue Src2 = Op.getOperand(2);
25813       SDValue Src3 = Op.getOperand(3);
25814       SDValue Imm = Op.getOperand(4);
25815       SDValue Mask = Op.getOperand(5);
25816       SDValue Passthru = (IntrData->Type == FIXUPIMM)
25817                              ? Src1
25818                              : getZeroVector(VT, Subtarget, DAG, dl);
25819 
25820       unsigned Opc = IntrData->Opc0;
25821       if (IntrData->Opc1 != 0) {
25822         SDValue Sae = Op.getOperand(6);
25823         if (isRoundModeSAE(Sae))
25824           Opc = IntrData->Opc1;
25825         else if (!isRoundModeCurDirection(Sae))
25826           return SDValue();
25827       }
25828 
25829       SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm);
25830 
25831       if (Opc == X86ISD::VFIXUPIMM || Opc == X86ISD::VFIXUPIMM_SAE)
25832         return getVectorMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
25833 
25834       return getScalarMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
25835     }
25836     case ROUNDP: {
25837       assert(IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode");
25838       // Clear the upper bits of the rounding immediate so that the legacy
25839       // intrinsic can't trigger the scaling behavior of VRNDSCALE.
25840       auto Round = cast<ConstantSDNode>(Op.getOperand(2));
25841       SDValue RoundingMode =
25842           DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);
25843       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25844                          Op.getOperand(1), RoundingMode);
25845     }
25846     case ROUNDS: {
25847       assert(IntrData->Opc0 == X86ISD::VRNDSCALES && "Unexpected opcode");
25848       // Clear the upper bits of the rounding immediate so that the legacy
25849       // intrinsic can't trigger the scaling behavior of VRNDSCALE.
25850       auto Round = cast<ConstantSDNode>(Op.getOperand(3));
25851       SDValue RoundingMode =
25852           DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);
25853       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25854                          Op.getOperand(1), Op.getOperand(2), RoundingMode);
25855     }
25856     case BEXTRI: {
25857       assert(IntrData->Opc0 == X86ISD::BEXTRI && "Unexpected opcode");
25858 
25859       uint64_t Imm = Op.getConstantOperandVal(2);
25860       SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl,
25861                                               Op.getValueType());
25862       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25863                          Op.getOperand(1), Control);
25864     }
25865     // ADC/SBB
25866     case ADX: {
25867       SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
25868       SDVTList VTs = DAG.getVTList(Op.getOperand(2).getValueType(), MVT::i32);
25869 
25870       SDValue Res;
25871       // If the carry in is zero, then we should just use ADD/SUB instead of
25872       // ADC/SBB.
25873       if (isNullConstant(Op.getOperand(1))) {
25874         Res = DAG.getNode(IntrData->Opc1, dl, VTs, Op.getOperand(2),
25875                           Op.getOperand(3));
25876       } else {
25877         SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
25878                                     DAG.getConstant(-1, dl, MVT::i8));
25879         Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2),
25880                           Op.getOperand(3), GenCF.getValue(1));
25881       }
25882       SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
25883       SDValue Results[] = { SetCC, Res };
25884       return DAG.getMergeValues(Results, dl);
25885     }
25886     case CVTPD2PS_MASK:
25887     case CVTPD2DQ_MASK:
25888     case CVTQQ2PS_MASK:
25889     case TRUNCATE_TO_REG: {
25890       SDValue Src = Op.getOperand(1);
25891       SDValue PassThru = Op.getOperand(2);
25892       SDValue Mask = Op.getOperand(3);
25893 
25894       if (isAllOnesConstant(Mask))
25895         return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
25896 
25897       MVT SrcVT = Src.getSimpleValueType();
25898       MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
25899       Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25900       return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(),
25901                          {Src, PassThru, Mask});
25902     }
25903     case CVTPS2PH_MASK: {
25904       SDValue Src = Op.getOperand(1);
25905       SDValue Rnd = Op.getOperand(2);
25906       SDValue PassThru = Op.getOperand(3);
25907       SDValue Mask = Op.getOperand(4);
25908 
25909       unsigned RC = 0;
25910       unsigned Opc = IntrData->Opc0;
25911       bool SAE = Src.getValueType().is512BitVector() &&
25912                  (isRoundModeSAEToX(Rnd, RC) || isRoundModeSAE(Rnd));
25913       if (SAE) {
25914         Opc = X86ISD::CVTPS2PH_SAE;
25915         Rnd = DAG.getTargetConstant(RC, dl, MVT::i32);
25916       }
25917 
25918       if (isAllOnesConstant(Mask))
25919         return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd);
25920 
25921       if (SAE)
25922         Opc = X86ISD::MCVTPS2PH_SAE;
25923       else
25924         Opc = IntrData->Opc1;
25925       MVT SrcVT = Src.getSimpleValueType();
25926       MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
25927       Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25928       return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd, PassThru, Mask);
25929     }
25930     case CVTNEPS2BF16_MASK: {
25931       SDValue Src = Op.getOperand(1);
25932       SDValue PassThru = Op.getOperand(2);
25933       SDValue Mask = Op.getOperand(3);
25934 
25935       if (ISD::isBuildVectorAllOnes(Mask.getNode()))
25936         return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
25937 
25938       // Break false dependency.
25939       if (PassThru.isUndef())
25940         PassThru = DAG.getConstant(0, dl, PassThru.getValueType());
25941 
25942       return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru,
25943                          Mask);
25944     }
25945     default:
25946       break;
25947     }
25948   }
25949 
25950   switch (IntNo) {
25951   default: return SDValue();    // Don't custom lower most intrinsics.
25952 
25953   // ptest and testp intrinsics. The intrinsic these come from are designed to
25954   // return an integer value, not just an instruction so lower it to the ptest
25955   // or testp pattern and a setcc for the result.
25956   case Intrinsic::x86_avx512_ktestc_b:
25957   case Intrinsic::x86_avx512_ktestc_w:
25958   case Intrinsic::x86_avx512_ktestc_d:
25959   case Intrinsic::x86_avx512_ktestc_q:
25960   case Intrinsic::x86_avx512_ktestz_b:
25961   case Intrinsic::x86_avx512_ktestz_w:
25962   case Intrinsic::x86_avx512_ktestz_d:
25963   case Intrinsic::x86_avx512_ktestz_q:
25964   case Intrinsic::x86_sse41_ptestz:
25965   case Intrinsic::x86_sse41_ptestc:
25966   case Intrinsic::x86_sse41_ptestnzc:
25967   case Intrinsic::x86_avx_ptestz_256:
25968   case Intrinsic::x86_avx_ptestc_256:
25969   case Intrinsic::x86_avx_ptestnzc_256:
25970   case Intrinsic::x86_avx_vtestz_ps:
25971   case Intrinsic::x86_avx_vtestc_ps:
25972   case Intrinsic::x86_avx_vtestnzc_ps:
25973   case Intrinsic::x86_avx_vtestz_pd:
25974   case Intrinsic::x86_avx_vtestc_pd:
25975   case Intrinsic::x86_avx_vtestnzc_pd:
25976   case Intrinsic::x86_avx_vtestz_ps_256:
25977   case Intrinsic::x86_avx_vtestc_ps_256:
25978   case Intrinsic::x86_avx_vtestnzc_ps_256:
25979   case Intrinsic::x86_avx_vtestz_pd_256:
25980   case Intrinsic::x86_avx_vtestc_pd_256:
25981   case Intrinsic::x86_avx_vtestnzc_pd_256: {
25982     unsigned TestOpc = X86ISD::PTEST;
25983     X86::CondCode X86CC;
25984     switch (IntNo) {
25985     default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
25986     case Intrinsic::x86_avx512_ktestc_b:
25987     case Intrinsic::x86_avx512_ktestc_w:
25988     case Intrinsic::x86_avx512_ktestc_d:
25989     case Intrinsic::x86_avx512_ktestc_q:
25990       // CF = 1
25991       TestOpc = X86ISD::KTEST;
25992       X86CC = X86::COND_B;
25993       break;
25994     case Intrinsic::x86_avx512_ktestz_b:
25995     case Intrinsic::x86_avx512_ktestz_w:
25996     case Intrinsic::x86_avx512_ktestz_d:
25997     case Intrinsic::x86_avx512_ktestz_q:
25998       TestOpc = X86ISD::KTEST;
25999       X86CC = X86::COND_E;
26000       break;
26001     case Intrinsic::x86_avx_vtestz_ps:
26002     case Intrinsic::x86_avx_vtestz_pd:
26003     case Intrinsic::x86_avx_vtestz_ps_256:
26004     case Intrinsic::x86_avx_vtestz_pd_256:
26005       TestOpc = X86ISD::TESTP;
26006       [[fallthrough]];
26007     case Intrinsic::x86_sse41_ptestz:
26008     case Intrinsic::x86_avx_ptestz_256:
26009       // ZF = 1
26010       X86CC = X86::COND_E;
26011       break;
26012     case Intrinsic::x86_avx_vtestc_ps:
26013     case Intrinsic::x86_avx_vtestc_pd:
26014     case Intrinsic::x86_avx_vtestc_ps_256:
26015     case Intrinsic::x86_avx_vtestc_pd_256:
26016       TestOpc = X86ISD::TESTP;
26017       [[fallthrough]];
26018     case Intrinsic::x86_sse41_ptestc:
26019     case Intrinsic::x86_avx_ptestc_256:
26020       // CF = 1
26021       X86CC = X86::COND_B;
26022       break;
26023     case Intrinsic::x86_avx_vtestnzc_ps:
26024     case Intrinsic::x86_avx_vtestnzc_pd:
26025     case Intrinsic::x86_avx_vtestnzc_ps_256:
26026     case Intrinsic::x86_avx_vtestnzc_pd_256:
26027       TestOpc = X86ISD::TESTP;
26028       [[fallthrough]];
26029     case Intrinsic::x86_sse41_ptestnzc:
26030     case Intrinsic::x86_avx_ptestnzc_256:
26031       // ZF and CF = 0
26032       X86CC = X86::COND_A;
26033       break;
26034     }
26035 
26036     SDValue LHS = Op.getOperand(1);
26037     SDValue RHS = Op.getOperand(2);
26038     SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
26039     SDValue SetCC = getSETCC(X86CC, Test, dl, DAG);
26040     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26041   }
26042 
26043   case Intrinsic::x86_sse42_pcmpistria128:
26044   case Intrinsic::x86_sse42_pcmpestria128:
26045   case Intrinsic::x86_sse42_pcmpistric128:
26046   case Intrinsic::x86_sse42_pcmpestric128:
26047   case Intrinsic::x86_sse42_pcmpistrio128:
26048   case Intrinsic::x86_sse42_pcmpestrio128:
26049   case Intrinsic::x86_sse42_pcmpistris128:
26050   case Intrinsic::x86_sse42_pcmpestris128:
26051   case Intrinsic::x86_sse42_pcmpistriz128:
26052   case Intrinsic::x86_sse42_pcmpestriz128: {
26053     unsigned Opcode;
26054     X86::CondCode X86CC;
26055     switch (IntNo) {
26056     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
26057     case Intrinsic::x86_sse42_pcmpistria128:
26058       Opcode = X86ISD::PCMPISTR;
26059       X86CC = X86::COND_A;
26060       break;
26061     case Intrinsic::x86_sse42_pcmpestria128:
26062       Opcode = X86ISD::PCMPESTR;
26063       X86CC = X86::COND_A;
26064       break;
26065     case Intrinsic::x86_sse42_pcmpistric128:
26066       Opcode = X86ISD::PCMPISTR;
26067       X86CC = X86::COND_B;
26068       break;
26069     case Intrinsic::x86_sse42_pcmpestric128:
26070       Opcode = X86ISD::PCMPESTR;
26071       X86CC = X86::COND_B;
26072       break;
26073     case Intrinsic::x86_sse42_pcmpistrio128:
26074       Opcode = X86ISD::PCMPISTR;
26075       X86CC = X86::COND_O;
26076       break;
26077     case Intrinsic::x86_sse42_pcmpestrio128:
26078       Opcode = X86ISD::PCMPESTR;
26079       X86CC = X86::COND_O;
26080       break;
26081     case Intrinsic::x86_sse42_pcmpistris128:
26082       Opcode = X86ISD::PCMPISTR;
26083       X86CC = X86::COND_S;
26084       break;
26085     case Intrinsic::x86_sse42_pcmpestris128:
26086       Opcode = X86ISD::PCMPESTR;
26087       X86CC = X86::COND_S;
26088       break;
26089     case Intrinsic::x86_sse42_pcmpistriz128:
26090       Opcode = X86ISD::PCMPISTR;
26091       X86CC = X86::COND_E;
26092       break;
26093     case Intrinsic::x86_sse42_pcmpestriz128:
26094       Opcode = X86ISD::PCMPESTR;
26095       X86CC = X86::COND_E;
26096       break;
26097     }
26098     SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26099     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26100     SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
26101     SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
26102     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26103   }
26104 
26105   case Intrinsic::x86_sse42_pcmpistri128:
26106   case Intrinsic::x86_sse42_pcmpestri128: {
26107     unsigned Opcode;
26108     if (IntNo == Intrinsic::x86_sse42_pcmpistri128)
26109       Opcode = X86ISD::PCMPISTR;
26110     else
26111       Opcode = X86ISD::PCMPESTR;
26112 
26113     SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26114     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26115     return DAG.getNode(Opcode, dl, VTs, NewOps);
26116   }
26117 
26118   case Intrinsic::x86_sse42_pcmpistrm128:
26119   case Intrinsic::x86_sse42_pcmpestrm128: {
26120     unsigned Opcode;
26121     if (IntNo == Intrinsic::x86_sse42_pcmpistrm128)
26122       Opcode = X86ISD::PCMPISTR;
26123     else
26124       Opcode = X86ISD::PCMPESTR;
26125 
26126     SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26127     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26128     return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1);
26129   }
26130 
26131   case Intrinsic::eh_sjlj_lsda: {
26132     MachineFunction &MF = DAG.getMachineFunction();
26133     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26134     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
26135     auto &Context = MF.getMMI().getContext();
26136     MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
26137                                             Twine(MF.getFunctionNumber()));
26138     return DAG.getNode(getGlobalWrapperKind(nullptr, /*OpFlags=*/0), dl, VT,
26139                        DAG.getMCSymbol(S, PtrVT));
26140   }
26141 
26142   case Intrinsic::x86_seh_lsda: {
26143     // Compute the symbol for the LSDA. We know it'll get emitted later.
26144     MachineFunction &MF = DAG.getMachineFunction();
26145     SDValue Op1 = Op.getOperand(1);
26146     auto *Fn = cast<Function>(cast<GlobalAddressSDNode>(Op1)->getGlobal());
26147     MCSymbol *LSDASym = MF.getMMI().getContext().getOrCreateLSDASymbol(
26148         GlobalValue::dropLLVMManglingEscape(Fn->getName()));
26149 
26150     // Generate a simple absolute symbol reference. This intrinsic is only
26151     // supported on 32-bit Windows, which isn't PIC.
26152     SDValue Result = DAG.getMCSymbol(LSDASym, VT);
26153     return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
26154   }
26155 
26156   case Intrinsic::eh_recoverfp: {
26157     SDValue FnOp = Op.getOperand(1);
26158     SDValue IncomingFPOp = Op.getOperand(2);
26159     GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
26160     auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
26161     if (!Fn)
26162       report_fatal_error(
26163           "llvm.eh.recoverfp must take a function as the first argument");
26164     return recoverFramePointer(DAG, Fn, IncomingFPOp);
26165   }
26166 
26167   case Intrinsic::localaddress: {
26168     // Returns one of the stack, base, or frame pointer registers, depending on
26169     // which is used to reference local variables.
26170     MachineFunction &MF = DAG.getMachineFunction();
26171     const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
26172     unsigned Reg;
26173     if (RegInfo->hasBasePointer(MF))
26174       Reg = RegInfo->getBaseRegister();
26175     else { // Handles the SP or FP case.
26176       bool CantUseFP = RegInfo->hasStackRealignment(MF);
26177       if (CantUseFP)
26178         Reg = RegInfo->getPtrSizedStackRegister(MF);
26179       else
26180         Reg = RegInfo->getPtrSizedFrameRegister(MF);
26181     }
26182     return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
26183   }
26184   case Intrinsic::x86_avx512_vp2intersect_q_512:
26185   case Intrinsic::x86_avx512_vp2intersect_q_256:
26186   case Intrinsic::x86_avx512_vp2intersect_q_128:
26187   case Intrinsic::x86_avx512_vp2intersect_d_512:
26188   case Intrinsic::x86_avx512_vp2intersect_d_256:
26189   case Intrinsic::x86_avx512_vp2intersect_d_128: {
26190     MVT MaskVT = Op.getSimpleValueType();
26191 
26192     SDVTList VTs = DAG.getVTList(MVT::Untyped, MVT::Other);
26193     SDLoc DL(Op);
26194 
26195     SDValue Operation =
26196         DAG.getNode(X86ISD::VP2INTERSECT, DL, VTs,
26197                     Op->getOperand(1), Op->getOperand(2));
26198 
26199     SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
26200                                                  MaskVT, Operation);
26201     SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
26202                                                  MaskVT, Operation);
26203     return DAG.getMergeValues({Result0, Result1}, DL);
26204   }
26205   case Intrinsic::x86_mmx_pslli_w:
26206   case Intrinsic::x86_mmx_pslli_d:
26207   case Intrinsic::x86_mmx_pslli_q:
26208   case Intrinsic::x86_mmx_psrli_w:
26209   case Intrinsic::x86_mmx_psrli_d:
26210   case Intrinsic::x86_mmx_psrli_q:
26211   case Intrinsic::x86_mmx_psrai_w:
26212   case Intrinsic::x86_mmx_psrai_d: {
26213     SDLoc DL(Op);
26214     SDValue ShAmt = Op.getOperand(2);
26215     // If the argument is a constant, convert it to a target constant.
26216     if (auto *C = dyn_cast<ConstantSDNode>(ShAmt)) {
26217       // Clamp out of bounds shift amounts since they will otherwise be masked
26218       // to 8-bits which may make it no longer out of bounds.
26219       unsigned ShiftAmount = C->getAPIntValue().getLimitedValue(255);
26220       if (ShiftAmount == 0)
26221         return Op.getOperand(1);
26222 
26223       return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
26224                          Op.getOperand(0), Op.getOperand(1),
26225                          DAG.getTargetConstant(ShiftAmount, DL, MVT::i32));
26226     }
26227 
26228     unsigned NewIntrinsic;
26229     switch (IntNo) {
26230     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
26231     case Intrinsic::x86_mmx_pslli_w:
26232       NewIntrinsic = Intrinsic::x86_mmx_psll_w;
26233       break;
26234     case Intrinsic::x86_mmx_pslli_d:
26235       NewIntrinsic = Intrinsic::x86_mmx_psll_d;
26236       break;
26237     case Intrinsic::x86_mmx_pslli_q:
26238       NewIntrinsic = Intrinsic::x86_mmx_psll_q;
26239       break;
26240     case Intrinsic::x86_mmx_psrli_w:
26241       NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
26242       break;
26243     case Intrinsic::x86_mmx_psrli_d:
26244       NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
26245       break;
26246     case Intrinsic::x86_mmx_psrli_q:
26247       NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
26248       break;
26249     case Intrinsic::x86_mmx_psrai_w:
26250       NewIntrinsic = Intrinsic::x86_mmx_psra_w;
26251       break;
26252     case Intrinsic::x86_mmx_psrai_d:
26253       NewIntrinsic = Intrinsic::x86_mmx_psra_d;
26254       break;
26255     }
26256 
26257     // The vector shift intrinsics with scalars uses 32b shift amounts but
26258     // the sse2/mmx shift instructions reads 64 bits. Copy the 32 bits to an
26259     // MMX register.
26260     ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt);
26261     return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
26262                        DAG.getTargetConstant(NewIntrinsic, DL,
26263                                              getPointerTy(DAG.getDataLayout())),
26264                        Op.getOperand(1), ShAmt);
26265   }
26266   case Intrinsic::thread_pointer: {
26267     if (Subtarget.isTargetELF()) {
26268       SDLoc dl(Op);
26269       EVT PtrVT = getPointerTy(DAG.getDataLayout());
26270       // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
26271       Value *Ptr = Constant::getNullValue(PointerType::get(
26272           *DAG.getContext(), Subtarget.is64Bit() ? X86AS::FS : X86AS::GS));
26273       return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
26274                          DAG.getIntPtrConstant(0, dl), MachinePointerInfo(Ptr));
26275     }
26276     report_fatal_error(
26277         "Target OS doesn't support __builtin_thread_pointer() yet.");
26278   }
26279   }
26280 }
26281 
26282 static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26283                                  SDValue Src, SDValue Mask, SDValue Base,
26284                                  SDValue Index, SDValue ScaleOp, SDValue Chain,
26285                                  const X86Subtarget &Subtarget) {
26286   SDLoc dl(Op);
26287   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26288   // Scale must be constant.
26289   if (!C)
26290     return SDValue();
26291   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26292   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26293                                         TLI.getPointerTy(DAG.getDataLayout()));
26294   EVT MaskVT = Mask.getValueType().changeVectorElementTypeToInteger();
26295   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);
26296   // If source is undef or we know it won't be used, use a zero vector
26297   // to break register dependency.
26298   // TODO: use undef instead and let BreakFalseDeps deal with it?
26299   if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
26300     Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
26301 
26302   // Cast mask to an integer type.
26303   Mask = DAG.getBitcast(MaskVT, Mask);
26304 
26305   MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26306 
26307   SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
26308   SDValue Res =
26309       DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
26310                               MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26311   return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
26312 }
26313 
26314 static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
26315                              SDValue Src, SDValue Mask, SDValue Base,
26316                              SDValue Index, SDValue ScaleOp, SDValue Chain,
26317                              const X86Subtarget &Subtarget) {
26318   MVT VT = Op.getSimpleValueType();
26319   SDLoc dl(Op);
26320   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26321   // Scale must be constant.
26322   if (!C)
26323     return SDValue();
26324   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26325   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26326                                         TLI.getPointerTy(DAG.getDataLayout()));
26327   unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
26328                               VT.getVectorNumElements());
26329   MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
26330 
26331   // We support two versions of the gather intrinsics. One with scalar mask and
26332   // one with vXi1 mask. Convert scalar to vXi1 if necessary.
26333   if (Mask.getValueType() != MaskVT)
26334     Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26335 
26336   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);
26337   // If source is undef or we know it won't be used, use a zero vector
26338   // to break register dependency.
26339   // TODO: use undef instead and let BreakFalseDeps deal with it?
26340   if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
26341     Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
26342 
26343   MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26344 
26345   SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
26346   SDValue Res =
26347       DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
26348                               MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26349   return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
26350 }
26351 
26352 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26353                                SDValue Src, SDValue Mask, SDValue Base,
26354                                SDValue Index, SDValue ScaleOp, SDValue Chain,
26355                                const X86Subtarget &Subtarget) {
26356   SDLoc dl(Op);
26357   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26358   // Scale must be constant.
26359   if (!C)
26360     return SDValue();
26361   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26362   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26363                                         TLI.getPointerTy(DAG.getDataLayout()));
26364   unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
26365                               Src.getSimpleValueType().getVectorNumElements());
26366   MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
26367 
26368   // We support two versions of the scatter intrinsics. One with scalar mask and
26369   // one with vXi1 mask. Convert scalar to vXi1 if necessary.
26370   if (Mask.getValueType() != MaskVT)
26371     Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26372 
26373   MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26374 
26375   SDVTList VTs = DAG.getVTList(MVT::Other);
26376   SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
26377   SDValue Res =
26378       DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
26379                               MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26380   return Res;
26381 }
26382 
26383 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26384                                SDValue Mask, SDValue Base, SDValue Index,
26385                                SDValue ScaleOp, SDValue Chain,
26386                                const X86Subtarget &Subtarget) {
26387   SDLoc dl(Op);
26388   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26389   // Scale must be constant.
26390   if (!C)
26391     return SDValue();
26392   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26393   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26394                                         TLI.getPointerTy(DAG.getDataLayout()));
26395   SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
26396   SDValue Segment = DAG.getRegister(0, MVT::i32);
26397   MVT MaskVT =
26398     MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
26399   SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26400   SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
26401   SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
26402   return SDValue(Res, 0);
26403 }
26404 
26405 /// Handles the lowering of builtin intrinsics with chain that return their
26406 /// value into registers EDX:EAX.
26407 /// If operand ScrReg is a valid register identifier, then operand 2 of N is
26408 /// copied to SrcReg. The assumption is that SrcReg is an implicit input to
26409 /// TargetOpcode.
26410 /// Returns a Glue value which can be used to add extra copy-from-reg if the
26411 /// expanded intrinsics implicitly defines extra registers (i.e. not just
26412 /// EDX:EAX).
26413 static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
26414                                         SelectionDAG &DAG,
26415                                         unsigned TargetOpcode,
26416                                         unsigned SrcReg,
26417                                         const X86Subtarget &Subtarget,
26418                                         SmallVectorImpl<SDValue> &Results) {
26419   SDValue Chain = N->getOperand(0);
26420   SDValue Glue;
26421 
26422   if (SrcReg) {
26423     assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
26424     Chain = DAG.getCopyToReg(Chain, DL, SrcReg, N->getOperand(2), Glue);
26425     Glue = Chain.getValue(1);
26426   }
26427 
26428   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
26429   SDValue N1Ops[] = {Chain, Glue};
26430   SDNode *N1 = DAG.getMachineNode(
26431       TargetOpcode, DL, Tys, ArrayRef<SDValue>(N1Ops, Glue.getNode() ? 2 : 1));
26432   Chain = SDValue(N1, 0);
26433 
26434   // Reads the content of XCR and returns it in registers EDX:EAX.
26435   SDValue LO, HI;
26436   if (Subtarget.is64Bit()) {
26437     LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
26438     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
26439                             LO.getValue(2));
26440   } else {
26441     LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
26442     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
26443                             LO.getValue(2));
26444   }
26445   Chain = HI.getValue(1);
26446   Glue = HI.getValue(2);
26447 
26448   if (Subtarget.is64Bit()) {
26449     // Merge the two 32-bit values into a 64-bit one.
26450     SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
26451                               DAG.getConstant(32, DL, MVT::i8));
26452     Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
26453     Results.push_back(Chain);
26454     return Glue;
26455   }
26456 
26457   // Use a buildpair to merge the two 32-bit values into a 64-bit one.
26458   SDValue Ops[] = { LO, HI };
26459   SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
26460   Results.push_back(Pair);
26461   Results.push_back(Chain);
26462   return Glue;
26463 }
26464 
26465 /// Handles the lowering of builtin intrinsics that read the time stamp counter
26466 /// (x86_rdtsc and x86_rdtscp). This function is also used to custom lower
26467 /// READCYCLECOUNTER nodes.
26468 static void getReadTimeStampCounter(SDNode *N, const SDLoc &DL, unsigned Opcode,
26469                                     SelectionDAG &DAG,
26470                                     const X86Subtarget &Subtarget,
26471                                     SmallVectorImpl<SDValue> &Results) {
26472   // The processor's time-stamp counter (a 64-bit MSR) is stored into the
26473   // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
26474   // and the EAX register is loaded with the low-order 32 bits.
26475   SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
26476                                              /* NoRegister */0, Subtarget,
26477                                              Results);
26478   if (Opcode != X86::RDTSCP)
26479     return;
26480 
26481   SDValue Chain = Results[1];
26482   // Instruction RDTSCP loads the IA32:TSC_AUX_MSR (address C000_0103H) into
26483   // the ECX register. Add 'ecx' explicitly to the chain.
26484   SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
26485   Results[1] = ecx;
26486   Results.push_back(ecx.getValue(1));
26487 }
26488 
26489 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
26490                                      SelectionDAG &DAG) {
26491   SmallVector<SDValue, 3> Results;
26492   SDLoc DL(Op);
26493   getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget,
26494                           Results);
26495   return DAG.getMergeValues(Results, DL);
26496 }
26497 
26498 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
26499   MachineFunction &MF = DAG.getMachineFunction();
26500   SDValue Chain = Op.getOperand(0);
26501   SDValue RegNode = Op.getOperand(2);
26502   WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
26503   if (!EHInfo)
26504     report_fatal_error("EH registrations only live in functions using WinEH");
26505 
26506   // Cast the operand to an alloca, and remember the frame index.
26507   auto *FINode = dyn_cast<FrameIndexSDNode>(RegNode);
26508   if (!FINode)
26509     report_fatal_error("llvm.x86.seh.ehregnode expects a static alloca");
26510   EHInfo->EHRegNodeFrameIndex = FINode->getIndex();
26511 
26512   // Return the chain operand without making any DAG nodes.
26513   return Chain;
26514 }
26515 
26516 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
26517   MachineFunction &MF = DAG.getMachineFunction();
26518   SDValue Chain = Op.getOperand(0);
26519   SDValue EHGuard = Op.getOperand(2);
26520   WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
26521   if (!EHInfo)
26522     report_fatal_error("EHGuard only live in functions using WinEH");
26523 
26524   // Cast the operand to an alloca, and remember the frame index.
26525   auto *FINode = dyn_cast<FrameIndexSDNode>(EHGuard);
26526   if (!FINode)
26527     report_fatal_error("llvm.x86.seh.ehguard expects a static alloca");
26528   EHInfo->EHGuardFrameIndex = FINode->getIndex();
26529 
26530   // Return the chain operand without making any DAG nodes.
26531   return Chain;
26532 }
26533 
26534 /// Emit Truncating Store with signed or unsigned saturation.
26535 static SDValue
26536 EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val,
26537                 SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
26538                 SelectionDAG &DAG) {
26539   SDVTList VTs = DAG.getVTList(MVT::Other);
26540   SDValue Undef = DAG.getUNDEF(Ptr.getValueType());
26541   SDValue Ops[] = { Chain, Val, Ptr, Undef };
26542   unsigned Opc = SignedSat ? X86ISD::VTRUNCSTORES : X86ISD::VTRUNCSTOREUS;
26543   return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
26544 }
26545 
26546 /// Emit Masked Truncating Store with signed or unsigned saturation.
26547 static SDValue EmitMaskedTruncSStore(bool SignedSat, SDValue Chain,
26548                                      const SDLoc &DL,
26549                       SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
26550                       MachineMemOperand *MMO, SelectionDAG &DAG) {
26551   SDVTList VTs = DAG.getVTList(MVT::Other);
26552   SDValue Ops[] = { Chain, Val, Ptr, Mask };
26553   unsigned Opc = SignedSat ? X86ISD::VMTRUNCSTORES : X86ISD::VMTRUNCSTOREUS;
26554   return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
26555 }
26556 
26557 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
26558                                       SelectionDAG &DAG) {
26559   unsigned IntNo = Op.getConstantOperandVal(1);
26560   const IntrinsicData *IntrData = getIntrinsicWithChain(IntNo);
26561   if (!IntrData) {
26562     switch (IntNo) {
26563 
26564     case Intrinsic::swift_async_context_addr: {
26565       SDLoc dl(Op);
26566       auto &MF = DAG.getMachineFunction();
26567       auto X86FI = MF.getInfo<X86MachineFunctionInfo>();
26568       if (Subtarget.is64Bit()) {
26569         MF.getFrameInfo().setFrameAddressIsTaken(true);
26570         X86FI->setHasSwiftAsyncContext(true);
26571         SDValue Chain = Op->getOperand(0);
26572         SDValue CopyRBP = DAG.getCopyFromReg(Chain, dl, X86::RBP, MVT::i64);
26573         SDValue Result =
26574             SDValue(DAG.getMachineNode(X86::SUB64ri32, dl, MVT::i64, CopyRBP,
26575                                        DAG.getTargetConstant(8, dl, MVT::i32)),
26576                     0);
26577         // Return { result, chain }.
26578         return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
26579                            CopyRBP.getValue(1));
26580       } else {
26581         // 32-bit so no special extended frame, create or reuse an existing
26582         // stack slot.
26583         if (!X86FI->getSwiftAsyncContextFrameIdx())
26584           X86FI->setSwiftAsyncContextFrameIdx(
26585               MF.getFrameInfo().CreateStackObject(4, Align(4), false));
26586         SDValue Result =
26587             DAG.getFrameIndex(*X86FI->getSwiftAsyncContextFrameIdx(), MVT::i32);
26588         // Return { result, chain }.
26589         return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
26590                            Op->getOperand(0));
26591       }
26592     }
26593 
26594     case llvm::Intrinsic::x86_seh_ehregnode:
26595       return MarkEHRegistrationNode(Op, DAG);
26596     case llvm::Intrinsic::x86_seh_ehguard:
26597       return MarkEHGuard(Op, DAG);
26598     case llvm::Intrinsic::x86_rdpkru: {
26599       SDLoc dl(Op);
26600       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26601       // Create a RDPKRU node and pass 0 to the ECX parameter.
26602       return DAG.getNode(X86ISD::RDPKRU, dl, VTs, Op.getOperand(0),
26603                          DAG.getConstant(0, dl, MVT::i32));
26604     }
26605     case llvm::Intrinsic::x86_wrpkru: {
26606       SDLoc dl(Op);
26607       // Create a WRPKRU node, pass the input to the EAX parameter,  and pass 0
26608       // to the EDX and ECX parameters.
26609       return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,
26610                          Op.getOperand(0), Op.getOperand(2),
26611                          DAG.getConstant(0, dl, MVT::i32),
26612                          DAG.getConstant(0, dl, MVT::i32));
26613     }
26614     case llvm::Intrinsic::asan_check_memaccess: {
26615       // Mark this as adjustsStack because it will be lowered to a call.
26616       DAG.getMachineFunction().getFrameInfo().setAdjustsStack(true);
26617       // Don't do anything here, we will expand these intrinsics out later.
26618       return Op;
26619     }
26620     case llvm::Intrinsic::x86_flags_read_u32:
26621     case llvm::Intrinsic::x86_flags_read_u64:
26622     case llvm::Intrinsic::x86_flags_write_u32:
26623     case llvm::Intrinsic::x86_flags_write_u64: {
26624       // We need a frame pointer because this will get lowered to a PUSH/POP
26625       // sequence.
26626       MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
26627       MFI.setHasCopyImplyingStackAdjustment(true);
26628       // Don't do anything here, we will expand these intrinsics out later
26629       // during FinalizeISel in EmitInstrWithCustomInserter.
26630       return Op;
26631     }
26632     case Intrinsic::x86_lwpins32:
26633     case Intrinsic::x86_lwpins64:
26634     case Intrinsic::x86_umwait:
26635     case Intrinsic::x86_tpause: {
26636       SDLoc dl(Op);
26637       SDValue Chain = Op->getOperand(0);
26638       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26639       unsigned Opcode;
26640 
26641       switch (IntNo) {
26642       default: llvm_unreachable("Impossible intrinsic");
26643       case Intrinsic::x86_umwait:
26644         Opcode = X86ISD::UMWAIT;
26645         break;
26646       case Intrinsic::x86_tpause:
26647         Opcode = X86ISD::TPAUSE;
26648         break;
26649       case Intrinsic::x86_lwpins32:
26650       case Intrinsic::x86_lwpins64:
26651         Opcode = X86ISD::LWPINS;
26652         break;
26653       }
26654 
26655       SDValue Operation =
26656           DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
26657                       Op->getOperand(3), Op->getOperand(4));
26658       SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
26659       return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
26660                          Operation.getValue(1));
26661     }
26662     case Intrinsic::x86_enqcmd:
26663     case Intrinsic::x86_enqcmds: {
26664       SDLoc dl(Op);
26665       SDValue Chain = Op.getOperand(0);
26666       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26667       unsigned Opcode;
26668       switch (IntNo) {
26669       default: llvm_unreachable("Impossible intrinsic!");
26670       case Intrinsic::x86_enqcmd:
26671         Opcode = X86ISD::ENQCMD;
26672         break;
26673       case Intrinsic::x86_enqcmds:
26674         Opcode = X86ISD::ENQCMDS;
26675         break;
26676       }
26677       SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2),
26678                                       Op.getOperand(3));
26679       SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
26680       return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
26681                          Operation.getValue(1));
26682     }
26683     case Intrinsic::x86_aesenc128kl:
26684     case Intrinsic::x86_aesdec128kl:
26685     case Intrinsic::x86_aesenc256kl:
26686     case Intrinsic::x86_aesdec256kl: {
26687       SDLoc DL(Op);
26688       SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::i32, MVT::Other);
26689       SDValue Chain = Op.getOperand(0);
26690       unsigned Opcode;
26691 
26692       switch (IntNo) {
26693       default: llvm_unreachable("Impossible intrinsic");
26694       case Intrinsic::x86_aesenc128kl:
26695         Opcode = X86ISD::AESENC128KL;
26696         break;
26697       case Intrinsic::x86_aesdec128kl:
26698         Opcode = X86ISD::AESDEC128KL;
26699         break;
26700       case Intrinsic::x86_aesenc256kl:
26701         Opcode = X86ISD::AESENC256KL;
26702         break;
26703       case Intrinsic::x86_aesdec256kl:
26704         Opcode = X86ISD::AESDEC256KL;
26705         break;
26706       }
26707 
26708       MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26709       MachineMemOperand *MMO = MemIntr->getMemOperand();
26710       EVT MemVT = MemIntr->getMemoryVT();
26711       SDValue Operation = DAG.getMemIntrinsicNode(
26712           Opcode, DL, VTs, {Chain, Op.getOperand(2), Op.getOperand(3)}, MemVT,
26713           MMO);
26714       SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(1), DL, DAG);
26715 
26716       return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
26717                          {ZF, Operation.getValue(0), Operation.getValue(2)});
26718     }
26719     case Intrinsic::x86_aesencwide128kl:
26720     case Intrinsic::x86_aesdecwide128kl:
26721     case Intrinsic::x86_aesencwide256kl:
26722     case Intrinsic::x86_aesdecwide256kl: {
26723       SDLoc DL(Op);
26724       SDVTList VTs = DAG.getVTList(
26725           {MVT::i32, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64,
26726            MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::Other});
26727       SDValue Chain = Op.getOperand(0);
26728       unsigned Opcode;
26729 
26730       switch (IntNo) {
26731       default: llvm_unreachable("Impossible intrinsic");
26732       case Intrinsic::x86_aesencwide128kl:
26733         Opcode = X86ISD::AESENCWIDE128KL;
26734         break;
26735       case Intrinsic::x86_aesdecwide128kl:
26736         Opcode = X86ISD::AESDECWIDE128KL;
26737         break;
26738       case Intrinsic::x86_aesencwide256kl:
26739         Opcode = X86ISD::AESENCWIDE256KL;
26740         break;
26741       case Intrinsic::x86_aesdecwide256kl:
26742         Opcode = X86ISD::AESDECWIDE256KL;
26743         break;
26744       }
26745 
26746       MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26747       MachineMemOperand *MMO = MemIntr->getMemOperand();
26748       EVT MemVT = MemIntr->getMemoryVT();
26749       SDValue Operation = DAG.getMemIntrinsicNode(
26750           Opcode, DL, VTs,
26751           {Chain, Op.getOperand(2), Op.getOperand(3), Op.getOperand(4),
26752            Op.getOperand(5), Op.getOperand(6), Op.getOperand(7),
26753            Op.getOperand(8), Op.getOperand(9), Op.getOperand(10)},
26754           MemVT, MMO);
26755       SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(0), DL, DAG);
26756 
26757       return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
26758                          {ZF, Operation.getValue(1), Operation.getValue(2),
26759                           Operation.getValue(3), Operation.getValue(4),
26760                           Operation.getValue(5), Operation.getValue(6),
26761                           Operation.getValue(7), Operation.getValue(8),
26762                           Operation.getValue(9)});
26763     }
26764     case Intrinsic::x86_testui: {
26765       SDLoc dl(Op);
26766       SDValue Chain = Op.getOperand(0);
26767       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26768       SDValue Operation = DAG.getNode(X86ISD::TESTUI, dl, VTs, Chain);
26769       SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
26770       return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
26771                          Operation.getValue(1));
26772     }
26773     case Intrinsic::x86_atomic_bts_rm:
26774     case Intrinsic::x86_atomic_btc_rm:
26775     case Intrinsic::x86_atomic_btr_rm: {
26776       SDLoc DL(Op);
26777       MVT VT = Op.getSimpleValueType();
26778       SDValue Chain = Op.getOperand(0);
26779       SDValue Op1 = Op.getOperand(2);
26780       SDValue Op2 = Op.getOperand(3);
26781       unsigned Opc = IntNo == Intrinsic::x86_atomic_bts_rm   ? X86ISD::LBTS_RM
26782                      : IntNo == Intrinsic::x86_atomic_btc_rm ? X86ISD::LBTC_RM
26783                                                              : X86ISD::LBTR_RM;
26784       MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
26785       SDValue Res =
26786           DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
26787                                   {Chain, Op1, Op2}, VT, MMO);
26788       Chain = Res.getValue(1);
26789       Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
26790       return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain);
26791     }
26792     case Intrinsic::x86_atomic_bts:
26793     case Intrinsic::x86_atomic_btc:
26794     case Intrinsic::x86_atomic_btr: {
26795       SDLoc DL(Op);
26796       MVT VT = Op.getSimpleValueType();
26797       SDValue Chain = Op.getOperand(0);
26798       SDValue Op1 = Op.getOperand(2);
26799       SDValue Op2 = Op.getOperand(3);
26800       unsigned Opc = IntNo == Intrinsic::x86_atomic_bts   ? X86ISD::LBTS
26801                      : IntNo == Intrinsic::x86_atomic_btc ? X86ISD::LBTC
26802                                                           : X86ISD::LBTR;
26803       SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32);
26804       MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
26805       SDValue Res =
26806           DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
26807                                   {Chain, Op1, Op2, Size}, VT, MMO);
26808       Chain = Res.getValue(1);
26809       Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
26810       unsigned Imm = Op2->getAsZExtVal();
26811       if (Imm)
26812         Res = DAG.getNode(ISD::SHL, DL, VT, Res,
26813                           DAG.getShiftAmountConstant(Imm, VT, DL));
26814       return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain);
26815     }
26816     case Intrinsic::x86_cmpccxadd32:
26817     case Intrinsic::x86_cmpccxadd64: {
26818       SDLoc DL(Op);
26819       SDValue Chain = Op.getOperand(0);
26820       SDValue Addr = Op.getOperand(2);
26821       SDValue Src1 = Op.getOperand(3);
26822       SDValue Src2 = Op.getOperand(4);
26823       SDValue CC = Op.getOperand(5);
26824       MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
26825       SDValue Operation = DAG.getMemIntrinsicNode(
26826           X86ISD::CMPCCXADD, DL, Op->getVTList(), {Chain, Addr, Src1, Src2, CC},
26827           MVT::i32, MMO);
26828       return Operation;
26829     }
26830     case Intrinsic::x86_aadd32:
26831     case Intrinsic::x86_aadd64:
26832     case Intrinsic::x86_aand32:
26833     case Intrinsic::x86_aand64:
26834     case Intrinsic::x86_aor32:
26835     case Intrinsic::x86_aor64:
26836     case Intrinsic::x86_axor32:
26837     case Intrinsic::x86_axor64: {
26838       SDLoc DL(Op);
26839       SDValue Chain = Op.getOperand(0);
26840       SDValue Op1 = Op.getOperand(2);
26841       SDValue Op2 = Op.getOperand(3);
26842       MVT VT = Op2.getSimpleValueType();
26843       unsigned Opc = 0;
26844       switch (IntNo) {
26845       default:
26846         llvm_unreachable("Unknown Intrinsic");
26847       case Intrinsic::x86_aadd32:
26848       case Intrinsic::x86_aadd64:
26849         Opc = X86ISD::AADD;
26850         break;
26851       case Intrinsic::x86_aand32:
26852       case Intrinsic::x86_aand64:
26853         Opc = X86ISD::AAND;
26854         break;
26855       case Intrinsic::x86_aor32:
26856       case Intrinsic::x86_aor64:
26857         Opc = X86ISD::AOR;
26858         break;
26859       case Intrinsic::x86_axor32:
26860       case Intrinsic::x86_axor64:
26861         Opc = X86ISD::AXOR;
26862         break;
26863       }
26864       MachineMemOperand *MMO = cast<MemSDNode>(Op)->getMemOperand();
26865       return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(),
26866                                      {Chain, Op1, Op2}, VT, MMO);
26867     }
26868     case Intrinsic::x86_atomic_add_cc:
26869     case Intrinsic::x86_atomic_sub_cc:
26870     case Intrinsic::x86_atomic_or_cc:
26871     case Intrinsic::x86_atomic_and_cc:
26872     case Intrinsic::x86_atomic_xor_cc: {
26873       SDLoc DL(Op);
26874       SDValue Chain = Op.getOperand(0);
26875       SDValue Op1 = Op.getOperand(2);
26876       SDValue Op2 = Op.getOperand(3);
26877       X86::CondCode CC = (X86::CondCode)Op.getConstantOperandVal(4);
26878       MVT VT = Op2.getSimpleValueType();
26879       unsigned Opc = 0;
26880       switch (IntNo) {
26881       default:
26882         llvm_unreachable("Unknown Intrinsic");
26883       case Intrinsic::x86_atomic_add_cc:
26884         Opc = X86ISD::LADD;
26885         break;
26886       case Intrinsic::x86_atomic_sub_cc:
26887         Opc = X86ISD::LSUB;
26888         break;
26889       case Intrinsic::x86_atomic_or_cc:
26890         Opc = X86ISD::LOR;
26891         break;
26892       case Intrinsic::x86_atomic_and_cc:
26893         Opc = X86ISD::LAND;
26894         break;
26895       case Intrinsic::x86_atomic_xor_cc:
26896         Opc = X86ISD::LXOR;
26897         break;
26898       }
26899       MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
26900       SDValue LockArith =
26901           DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
26902                                   {Chain, Op1, Op2}, VT, MMO);
26903       Chain = LockArith.getValue(1);
26904       return DAG.getMergeValues({getSETCC(CC, LockArith, DL, DAG), Chain}, DL);
26905     }
26906     }
26907     return SDValue();
26908   }
26909 
26910   SDLoc dl(Op);
26911   switch(IntrData->Type) {
26912   default: llvm_unreachable("Unknown Intrinsic Type");
26913   case RDSEED:
26914   case RDRAND: {
26915     // Emit the node with the right value type.
26916     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other);
26917     SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
26918 
26919     // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
26920     // Otherwise return the value from Rand, which is always 0, casted to i32.
26921     SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
26922                      DAG.getConstant(1, dl, Op->getValueType(1)),
26923                      DAG.getTargetConstant(X86::COND_B, dl, MVT::i8),
26924                      SDValue(Result.getNode(), 1)};
26925     SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
26926 
26927     // Return { result, isValid, chain }.
26928     return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
26929                        SDValue(Result.getNode(), 2));
26930   }
26931   case GATHER_AVX2: {
26932     SDValue Chain = Op.getOperand(0);
26933     SDValue Src   = Op.getOperand(2);
26934     SDValue Base  = Op.getOperand(3);
26935     SDValue Index = Op.getOperand(4);
26936     SDValue Mask  = Op.getOperand(5);
26937     SDValue Scale = Op.getOperand(6);
26938     return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
26939                              Scale, Chain, Subtarget);
26940   }
26941   case GATHER: {
26942   //gather(v1, mask, index, base, scale);
26943     SDValue Chain = Op.getOperand(0);
26944     SDValue Src   = Op.getOperand(2);
26945     SDValue Base  = Op.getOperand(3);
26946     SDValue Index = Op.getOperand(4);
26947     SDValue Mask  = Op.getOperand(5);
26948     SDValue Scale = Op.getOperand(6);
26949     return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale,
26950                          Chain, Subtarget);
26951   }
26952   case SCATTER: {
26953   //scatter(base, mask, index, v1, scale);
26954     SDValue Chain = Op.getOperand(0);
26955     SDValue Base  = Op.getOperand(2);
26956     SDValue Mask  = Op.getOperand(3);
26957     SDValue Index = Op.getOperand(4);
26958     SDValue Src   = Op.getOperand(5);
26959     SDValue Scale = Op.getOperand(6);
26960     return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
26961                           Scale, Chain, Subtarget);
26962   }
26963   case PREFETCH: {
26964     const APInt &HintVal = Op.getConstantOperandAPInt(6);
26965     assert((HintVal == 2 || HintVal == 3) &&
26966            "Wrong prefetch hint in intrinsic: should be 2 or 3");
26967     unsigned Opcode = (HintVal == 2 ? IntrData->Opc1 : IntrData->Opc0);
26968     SDValue Chain = Op.getOperand(0);
26969     SDValue Mask  = Op.getOperand(2);
26970     SDValue Index = Op.getOperand(3);
26971     SDValue Base  = Op.getOperand(4);
26972     SDValue Scale = Op.getOperand(5);
26973     return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain,
26974                            Subtarget);
26975   }
26976   // Read Time Stamp Counter (RDTSC) and Processor ID (RDTSCP).
26977   case RDTSC: {
26978     SmallVector<SDValue, 2> Results;
26979     getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget,
26980                             Results);
26981     return DAG.getMergeValues(Results, dl);
26982   }
26983   // Read Performance Monitoring Counters.
26984   case RDPMC:
26985   // Read Processor Register.
26986   case RDPRU:
26987   // GetExtended Control Register.
26988   case XGETBV: {
26989     SmallVector<SDValue, 2> Results;
26990 
26991     // RDPMC uses ECX to select the index of the performance counter to read.
26992     // RDPRU uses ECX to select the processor register to read.
26993     // XGETBV uses ECX to select the index of the XCR register to return.
26994     // The result is stored into registers EDX:EAX.
26995     expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX,
26996                                 Subtarget, Results);
26997     return DAG.getMergeValues(Results, dl);
26998   }
26999   // XTEST intrinsics.
27000   case XTEST: {
27001     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
27002     SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
27003 
27004     SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
27005     SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
27006     return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
27007                        Ret, SDValue(InTrans.getNode(), 1));
27008   }
27009   case TRUNCATE_TO_MEM_VI8:
27010   case TRUNCATE_TO_MEM_VI16:
27011   case TRUNCATE_TO_MEM_VI32: {
27012     SDValue Mask = Op.getOperand(4);
27013     SDValue DataToTruncate = Op.getOperand(3);
27014     SDValue Addr = Op.getOperand(2);
27015     SDValue Chain = Op.getOperand(0);
27016 
27017     MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
27018     assert(MemIntr && "Expected MemIntrinsicSDNode!");
27019 
27020     EVT MemVT  = MemIntr->getMemoryVT();
27021 
27022     uint16_t TruncationOp = IntrData->Opc0;
27023     switch (TruncationOp) {
27024     case X86ISD::VTRUNC: {
27025       if (isAllOnesConstant(Mask)) // return just a truncate store
27026         return DAG.getTruncStore(Chain, dl, DataToTruncate, Addr, MemVT,
27027                                  MemIntr->getMemOperand());
27028 
27029       MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
27030       SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27031       SDValue Offset = DAG.getUNDEF(VMask.getValueType());
27032 
27033       return DAG.getMaskedStore(Chain, dl, DataToTruncate, Addr, Offset, VMask,
27034                                 MemVT, MemIntr->getMemOperand(), ISD::UNINDEXED,
27035                                 true /* truncating */);
27036     }
27037     case X86ISD::VTRUNCUS:
27038     case X86ISD::VTRUNCS: {
27039       bool IsSigned = (TruncationOp == X86ISD::VTRUNCS);
27040       if (isAllOnesConstant(Mask))
27041         return EmitTruncSStore(IsSigned, Chain, dl, DataToTruncate, Addr, MemVT,
27042                                MemIntr->getMemOperand(), DAG);
27043 
27044       MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
27045       SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27046 
27047       return EmitMaskedTruncSStore(IsSigned, Chain, dl, DataToTruncate, Addr,
27048                                    VMask, MemVT, MemIntr->getMemOperand(), DAG);
27049     }
27050     default:
27051       llvm_unreachable("Unsupported truncstore intrinsic");
27052     }
27053   }
27054   }
27055 }
27056 
27057 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
27058                                            SelectionDAG &DAG) const {
27059   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
27060   MFI.setReturnAddressIsTaken(true);
27061 
27062   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
27063     return SDValue();
27064 
27065   unsigned Depth = Op.getConstantOperandVal(0);
27066   SDLoc dl(Op);
27067   EVT PtrVT = getPointerTy(DAG.getDataLayout());
27068 
27069   if (Depth > 0) {
27070     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
27071     const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27072     SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
27073     return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
27074                        DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset),
27075                        MachinePointerInfo());
27076   }
27077 
27078   // Just load the return address.
27079   SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
27080   return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI,
27081                      MachinePointerInfo());
27082 }
27083 
27084 SDValue X86TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
27085                                                  SelectionDAG &DAG) const {
27086   DAG.getMachineFunction().getFrameInfo().setReturnAddressIsTaken(true);
27087   return getReturnAddressFrameIndex(DAG);
27088 }
27089 
27090 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
27091   MachineFunction &MF = DAG.getMachineFunction();
27092   MachineFrameInfo &MFI = MF.getFrameInfo();
27093   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
27094   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27095   EVT VT = Op.getValueType();
27096 
27097   MFI.setFrameAddressIsTaken(true);
27098 
27099   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
27100     // Depth > 0 makes no sense on targets which use Windows unwind codes.  It
27101     // is not possible to crawl up the stack without looking at the unwind codes
27102     // simultaneously.
27103     int FrameAddrIndex = FuncInfo->getFAIndex();
27104     if (!FrameAddrIndex) {
27105       // Set up a frame object for the return address.
27106       unsigned SlotSize = RegInfo->getSlotSize();
27107       FrameAddrIndex = MF.getFrameInfo().CreateFixedObject(
27108           SlotSize, /*SPOffset=*/0, /*IsImmutable=*/false);
27109       FuncInfo->setFAIndex(FrameAddrIndex);
27110     }
27111     return DAG.getFrameIndex(FrameAddrIndex, VT);
27112   }
27113 
27114   unsigned FrameReg =
27115       RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
27116   SDLoc dl(Op);  // FIXME probably not meaningful
27117   unsigned Depth = Op.getConstantOperandVal(0);
27118   assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
27119           (FrameReg == X86::EBP && VT == MVT::i32)) &&
27120          "Invalid Frame Register!");
27121   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
27122   while (Depth--)
27123     FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
27124                             MachinePointerInfo());
27125   return FrameAddr;
27126 }
27127 
27128 // FIXME? Maybe this could be a TableGen attribute on some registers and
27129 // this table could be generated automatically from RegInfo.
27130 Register X86TargetLowering::getRegisterByName(const char* RegName, LLT VT,
27131                                               const MachineFunction &MF) const {
27132   const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
27133 
27134   Register Reg = StringSwitch<unsigned>(RegName)
27135                      .Case("esp", X86::ESP)
27136                      .Case("rsp", X86::RSP)
27137                      .Case("ebp", X86::EBP)
27138                      .Case("rbp", X86::RBP)
27139                      .Case("r14", X86::R14)
27140                      .Case("r15", X86::R15)
27141                      .Default(0);
27142 
27143   if (Reg == X86::EBP || Reg == X86::RBP) {
27144     if (!TFI.hasFP(MF))
27145       report_fatal_error("register " + StringRef(RegName) +
27146                          " is allocatable: function has no frame pointer");
27147 #ifndef NDEBUG
27148     else {
27149       const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27150       Register FrameReg = RegInfo->getPtrSizedFrameRegister(MF);
27151       assert((FrameReg == X86::EBP || FrameReg == X86::RBP) &&
27152              "Invalid Frame Register!");
27153     }
27154 #endif
27155   }
27156 
27157   if (Reg)
27158     return Reg;
27159 
27160   report_fatal_error("Invalid register name global variable");
27161 }
27162 
27163 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
27164                                                      SelectionDAG &DAG) const {
27165   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27166   return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op));
27167 }
27168 
27169 Register X86TargetLowering::getExceptionPointerRegister(
27170     const Constant *PersonalityFn) const {
27171   if (classifyEHPersonality(PersonalityFn) == EHPersonality::CoreCLR)
27172     return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
27173 
27174   return Subtarget.isTarget64BitLP64() ? X86::RAX : X86::EAX;
27175 }
27176 
27177 Register X86TargetLowering::getExceptionSelectorRegister(
27178     const Constant *PersonalityFn) const {
27179   // Funclet personalities don't use selectors (the runtime does the selection).
27180   if (isFuncletEHPersonality(classifyEHPersonality(PersonalityFn)))
27181     return X86::NoRegister;
27182   return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
27183 }
27184 
27185 bool X86TargetLowering::needsFixedCatchObjects() const {
27186   return Subtarget.isTargetWin64();
27187 }
27188 
27189 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
27190   SDValue Chain     = Op.getOperand(0);
27191   SDValue Offset    = Op.getOperand(1);
27192   SDValue Handler   = Op.getOperand(2);
27193   SDLoc dl      (Op);
27194 
27195   EVT PtrVT = getPointerTy(DAG.getDataLayout());
27196   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27197   Register FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
27198   assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||
27199           (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&
27200          "Invalid Frame Register!");
27201   SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
27202   Register StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
27203 
27204   SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
27205                                  DAG.getIntPtrConstant(RegInfo->getSlotSize(),
27206                                                        dl));
27207   StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
27208   Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
27209   Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
27210 
27211   return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
27212                      DAG.getRegister(StoreAddrReg, PtrVT));
27213 }
27214 
27215 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
27216                                                SelectionDAG &DAG) const {
27217   SDLoc DL(Op);
27218   // If the subtarget is not 64bit, we may need the global base reg
27219   // after isel expand pseudo, i.e., after CGBR pass ran.
27220   // Therefore, ask for the GlobalBaseReg now, so that the pass
27221   // inserts the code for us in case we need it.
27222   // Otherwise, we will end up in a situation where we will
27223   // reference a virtual register that is not defined!
27224   if (!Subtarget.is64Bit()) {
27225     const X86InstrInfo *TII = Subtarget.getInstrInfo();
27226     (void)TII->getGlobalBaseReg(&DAG.getMachineFunction());
27227   }
27228   return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
27229                      DAG.getVTList(MVT::i32, MVT::Other),
27230                      Op.getOperand(0), Op.getOperand(1));
27231 }
27232 
27233 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
27234                                                 SelectionDAG &DAG) const {
27235   SDLoc DL(Op);
27236   return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
27237                      Op.getOperand(0), Op.getOperand(1));
27238 }
27239 
27240 SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
27241                                                        SelectionDAG &DAG) const {
27242   SDLoc DL(Op);
27243   return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
27244                      Op.getOperand(0));
27245 }
27246 
27247 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
27248   return Op.getOperand(0);
27249 }
27250 
27251 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
27252                                                 SelectionDAG &DAG) const {
27253   SDValue Root = Op.getOperand(0);
27254   SDValue Trmp = Op.getOperand(1); // trampoline
27255   SDValue FPtr = Op.getOperand(2); // nested function
27256   SDValue Nest = Op.getOperand(3); // 'nest' parameter value
27257   SDLoc dl (Op);
27258 
27259   const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
27260   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
27261 
27262   if (Subtarget.is64Bit()) {
27263     SDValue OutChains[6];
27264 
27265     // Large code-model.
27266     const unsigned char JMP64r  = 0xFF; // 64-bit jmp through register opcode.
27267     const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
27268 
27269     const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
27270     const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7;
27271 
27272     const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
27273 
27274     // Load the pointer to the nested function into R11.
27275     unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
27276     SDValue Addr = Trmp;
27277     OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27278                                 Addr, MachinePointerInfo(TrmpAddr));
27279 
27280     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27281                        DAG.getConstant(2, dl, MVT::i64));
27282     OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr,
27283                                 MachinePointerInfo(TrmpAddr, 2), Align(2));
27284 
27285     // Load the 'nest' parameter value into R10.
27286     // R10 is specified in X86CallingConv.td
27287     OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
27288     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27289                        DAG.getConstant(10, dl, MVT::i64));
27290     OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27291                                 Addr, MachinePointerInfo(TrmpAddr, 10));
27292 
27293     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27294                        DAG.getConstant(12, dl, MVT::i64));
27295     OutChains[3] = DAG.getStore(Root, dl, Nest, Addr,
27296                                 MachinePointerInfo(TrmpAddr, 12), Align(2));
27297 
27298     // Jump to the nested function.
27299     OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
27300     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27301                        DAG.getConstant(20, dl, MVT::i64));
27302     OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27303                                 Addr, MachinePointerInfo(TrmpAddr, 20));
27304 
27305     unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
27306     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27307                        DAG.getConstant(22, dl, MVT::i64));
27308     OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8),
27309                                 Addr, MachinePointerInfo(TrmpAddr, 22));
27310 
27311     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
27312   } else {
27313     const Function *Func =
27314       cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
27315     CallingConv::ID CC = Func->getCallingConv();
27316     unsigned NestReg;
27317 
27318     switch (CC) {
27319     default:
27320       llvm_unreachable("Unsupported calling convention");
27321     case CallingConv::C:
27322     case CallingConv::X86_StdCall: {
27323       // Pass 'nest' parameter in ECX.
27324       // Must be kept in sync with X86CallingConv.td
27325       NestReg = X86::ECX;
27326 
27327       // Check that ECX wasn't needed by an 'inreg' parameter.
27328       FunctionType *FTy = Func->getFunctionType();
27329       const AttributeList &Attrs = Func->getAttributes();
27330 
27331       if (!Attrs.isEmpty() && !Func->isVarArg()) {
27332         unsigned InRegCount = 0;
27333         unsigned Idx = 0;
27334 
27335         for (FunctionType::param_iterator I = FTy->param_begin(),
27336              E = FTy->param_end(); I != E; ++I, ++Idx)
27337           if (Attrs.hasParamAttr(Idx, Attribute::InReg)) {
27338             const DataLayout &DL = DAG.getDataLayout();
27339             // FIXME: should only count parameters that are lowered to integers.
27340             InRegCount += (DL.getTypeSizeInBits(*I) + 31) / 32;
27341           }
27342 
27343         if (InRegCount > 2) {
27344           report_fatal_error("Nest register in use - reduce number of inreg"
27345                              " parameters!");
27346         }
27347       }
27348       break;
27349     }
27350     case CallingConv::X86_FastCall:
27351     case CallingConv::X86_ThisCall:
27352     case CallingConv::Fast:
27353     case CallingConv::Tail:
27354     case CallingConv::SwiftTail:
27355       // Pass 'nest' parameter in EAX.
27356       // Must be kept in sync with X86CallingConv.td
27357       NestReg = X86::EAX;
27358       break;
27359     }
27360 
27361     SDValue OutChains[4];
27362     SDValue Addr, Disp;
27363 
27364     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27365                        DAG.getConstant(10, dl, MVT::i32));
27366     Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
27367 
27368     // This is storing the opcode for MOV32ri.
27369     const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
27370     const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7;
27371     OutChains[0] =
27372         DAG.getStore(Root, dl, DAG.getConstant(MOV32ri | N86Reg, dl, MVT::i8),
27373                      Trmp, MachinePointerInfo(TrmpAddr));
27374 
27375     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27376                        DAG.getConstant(1, dl, MVT::i32));
27377     OutChains[1] = DAG.getStore(Root, dl, Nest, Addr,
27378                                 MachinePointerInfo(TrmpAddr, 1), Align(1));
27379 
27380     const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode.
27381     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27382                        DAG.getConstant(5, dl, MVT::i32));
27383     OutChains[2] =
27384         DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), Addr,
27385                      MachinePointerInfo(TrmpAddr, 5), Align(1));
27386 
27387     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27388                        DAG.getConstant(6, dl, MVT::i32));
27389     OutChains[3] = DAG.getStore(Root, dl, Disp, Addr,
27390                                 MachinePointerInfo(TrmpAddr, 6), Align(1));
27391 
27392     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
27393   }
27394 }
27395 
27396 SDValue X86TargetLowering::LowerGET_ROUNDING(SDValue Op,
27397                                              SelectionDAG &DAG) const {
27398   /*
27399    The rounding mode is in bits 11:10 of FPSR, and has the following
27400    settings:
27401      00 Round to nearest
27402      01 Round to -inf
27403      10 Round to +inf
27404      11 Round to 0
27405 
27406   GET_ROUNDING, on the other hand, expects the following:
27407     -1 Undefined
27408      0 Round to 0
27409      1 Round to nearest
27410      2 Round to +inf
27411      3 Round to -inf
27412 
27413   To perform the conversion, we use a packed lookup table of the four 2-bit
27414   values that we can index by FPSP[11:10]
27415     0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10]
27416 
27417     (0x2d >> ((FPSR & 0xc00) >> 9)) & 3
27418   */
27419 
27420   MachineFunction &MF = DAG.getMachineFunction();
27421   MVT VT = Op.getSimpleValueType();
27422   SDLoc DL(Op);
27423 
27424   // Save FP Control Word to stack slot
27425   int SSFI = MF.getFrameInfo().CreateStackObject(2, Align(2), false);
27426   SDValue StackSlot =
27427       DAG.getFrameIndex(SSFI, getPointerTy(DAG.getDataLayout()));
27428 
27429   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, SSFI);
27430 
27431   SDValue Chain = Op.getOperand(0);
27432   SDValue Ops[] = {Chain, StackSlot};
27433   Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
27434                                   DAG.getVTList(MVT::Other), Ops, MVT::i16, MPI,
27435                                   Align(2), MachineMemOperand::MOStore);
27436 
27437   // Load FP Control Word from stack slot
27438   SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));
27439   Chain = CWD.getValue(1);
27440 
27441   // Mask and turn the control bits into a shift for the lookup table.
27442   SDValue Shift =
27443     DAG.getNode(ISD::SRL, DL, MVT::i16,
27444                 DAG.getNode(ISD::AND, DL, MVT::i16,
27445                             CWD, DAG.getConstant(0xc00, DL, MVT::i16)),
27446                 DAG.getConstant(9, DL, MVT::i8));
27447   Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift);
27448 
27449   SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
27450   SDValue RetVal =
27451     DAG.getNode(ISD::AND, DL, MVT::i32,
27452                 DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
27453                 DAG.getConstant(3, DL, MVT::i32));
27454 
27455   RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT);
27456 
27457   return DAG.getMergeValues({RetVal, Chain}, DL);
27458 }
27459 
27460 SDValue X86TargetLowering::LowerSET_ROUNDING(SDValue Op,
27461                                              SelectionDAG &DAG) const {
27462   MachineFunction &MF = DAG.getMachineFunction();
27463   SDLoc DL(Op);
27464   SDValue Chain = Op.getNode()->getOperand(0);
27465 
27466   // FP control word may be set only from data in memory. So we need to allocate
27467   // stack space to save/load FP control word.
27468   int OldCWFrameIdx = MF.getFrameInfo().CreateStackObject(4, Align(4), false);
27469   SDValue StackSlot =
27470       DAG.getFrameIndex(OldCWFrameIdx, getPointerTy(DAG.getDataLayout()));
27471   MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, OldCWFrameIdx);
27472   MachineMemOperand *MMO =
27473       MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 2, Align(2));
27474 
27475   // Store FP control word into memory.
27476   SDValue Ops[] = {Chain, StackSlot};
27477   Chain = DAG.getMemIntrinsicNode(
27478       X86ISD::FNSTCW16m, DL, DAG.getVTList(MVT::Other), Ops, MVT::i16, MMO);
27479 
27480   // Load FP Control Word from stack slot and clear RM field (bits 11:10).
27481   SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI);
27482   Chain = CWD.getValue(1);
27483   CWD = DAG.getNode(ISD::AND, DL, MVT::i16, CWD.getValue(0),
27484                     DAG.getConstant(0xf3ff, DL, MVT::i16));
27485 
27486   // Calculate new rounding mode.
27487   SDValue NewRM = Op.getNode()->getOperand(1);
27488   SDValue RMBits;
27489   if (auto *CVal = dyn_cast<ConstantSDNode>(NewRM)) {
27490     uint64_t RM = CVal->getZExtValue();
27491     int FieldVal;
27492     switch (static_cast<RoundingMode>(RM)) {
27493     case RoundingMode::NearestTiesToEven: FieldVal = X86::rmToNearest; break;
27494     case RoundingMode::TowardNegative:    FieldVal = X86::rmDownward; break;
27495     case RoundingMode::TowardPositive:    FieldVal = X86::rmUpward; break;
27496     case RoundingMode::TowardZero:        FieldVal = X86::rmTowardZero; break;
27497     default:
27498       llvm_unreachable("rounding mode is not supported by X86 hardware");
27499     }
27500     RMBits = DAG.getConstant(FieldVal, DL, MVT::i16);
27501   } else {
27502     // Need to convert argument into bits of control word:
27503     //    0 Round to 0       -> 11
27504     //    1 Round to nearest -> 00
27505     //    2 Round to +inf    -> 10
27506     //    3 Round to -inf    -> 01
27507     // The 2-bit value needs then to be shifted so that it occupies bits 11:10.
27508     // To make the conversion, put all these values into a value 0xc9 and shift
27509     // it left depending on the rounding mode:
27510     //    (0xc9 << 4) & 0xc00 = X86::rmTowardZero
27511     //    (0xc9 << 6) & 0xc00 = X86::rmToNearest
27512     //    ...
27513     // (0xc9 << (2 * NewRM + 4)) & 0xc00
27514     SDValue ShiftValue =
27515         DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
27516                     DAG.getNode(ISD::ADD, DL, MVT::i32,
27517                                 DAG.getNode(ISD::SHL, DL, MVT::i32, NewRM,
27518                                             DAG.getConstant(1, DL, MVT::i8)),
27519                                 DAG.getConstant(4, DL, MVT::i32)));
27520     SDValue Shifted =
27521         DAG.getNode(ISD::SHL, DL, MVT::i16, DAG.getConstant(0xc9, DL, MVT::i16),
27522                     ShiftValue);
27523     RMBits = DAG.getNode(ISD::AND, DL, MVT::i16, Shifted,
27524                          DAG.getConstant(0xc00, DL, MVT::i16));
27525   }
27526 
27527   // Update rounding mode bits and store the new FP Control Word into stack.
27528   CWD = DAG.getNode(ISD::OR, DL, MVT::i16, CWD, RMBits);
27529   Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(2));
27530 
27531   // Load FP control word from the slot.
27532   SDValue OpsLD[] = {Chain, StackSlot};
27533   MachineMemOperand *MMOL =
27534       MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 2, Align(2));
27535   Chain = DAG.getMemIntrinsicNode(
27536       X86ISD::FLDCW16m, DL, DAG.getVTList(MVT::Other), OpsLD, MVT::i16, MMOL);
27537 
27538   // If target supports SSE, set MXCSR as well. Rounding mode is encoded in the
27539   // same way but in bits 14:13.
27540   if (Subtarget.hasSSE1()) {
27541     // Store MXCSR into memory.
27542     Chain = DAG.getNode(
27543         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27544         DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
27545         StackSlot);
27546 
27547     // Load MXCSR from stack slot and clear RM field (bits 14:13).
27548     SDValue CWD = DAG.getLoad(MVT::i32, DL, Chain, StackSlot, MPI);
27549     Chain = CWD.getValue(1);
27550     CWD = DAG.getNode(ISD::AND, DL, MVT::i32, CWD.getValue(0),
27551                       DAG.getConstant(0xffff9fff, DL, MVT::i32));
27552 
27553     // Shift X87 RM bits from 11:10 to 14:13.
27554     RMBits = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, RMBits);
27555     RMBits = DAG.getNode(ISD::SHL, DL, MVT::i32, RMBits,
27556                          DAG.getConstant(3, DL, MVT::i8));
27557 
27558     // Update rounding mode bits and store the new FP Control Word into stack.
27559     CWD = DAG.getNode(ISD::OR, DL, MVT::i32, CWD, RMBits);
27560     Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(4));
27561 
27562     // Load MXCSR from the slot.
27563     Chain = DAG.getNode(
27564         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27565         DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
27566         StackSlot);
27567   }
27568 
27569   return Chain;
27570 }
27571 
27572 const unsigned X87StateSize = 28;
27573 const unsigned FPStateSize = 32;
27574 [[maybe_unused]] const unsigned FPStateSizeInBits = FPStateSize * 8;
27575 
27576 SDValue X86TargetLowering::LowerGET_FPENV_MEM(SDValue Op,
27577                                               SelectionDAG &DAG) const {
27578   MachineFunction &MF = DAG.getMachineFunction();
27579   SDLoc DL(Op);
27580   SDValue Chain = Op->getOperand(0);
27581   SDValue Ptr = Op->getOperand(1);
27582   auto *Node = cast<FPStateAccessSDNode>(Op);
27583   EVT MemVT = Node->getMemoryVT();
27584   assert(MemVT.getSizeInBits() == FPStateSizeInBits);
27585   MachineMemOperand *MMO = cast<FPStateAccessSDNode>(Op)->getMemOperand();
27586 
27587   // Get x87 state, if it presents.
27588   if (Subtarget.hasX87()) {
27589     Chain =
27590         DAG.getMemIntrinsicNode(X86ISD::FNSTENVm, DL, DAG.getVTList(MVT::Other),
27591                                 {Chain, Ptr}, MemVT, MMO);
27592 
27593     // FNSTENV changes the exception mask, so load back the stored environment.
27594     MachineMemOperand::Flags NewFlags =
27595         MachineMemOperand::MOLoad |
27596         (MMO->getFlags() & ~MachineMemOperand::MOStore);
27597     MMO = MF.getMachineMemOperand(MMO, NewFlags);
27598     Chain =
27599         DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other),
27600                                 {Chain, Ptr}, MemVT, MMO);
27601   }
27602 
27603   // If target supports SSE, get MXCSR as well.
27604   if (Subtarget.hasSSE1()) {
27605     // Get pointer to the MXCSR location in memory.
27606     MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27607     SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
27608                                     DAG.getConstant(X87StateSize, DL, PtrVT));
27609     // Store MXCSR into memory.
27610     Chain = DAG.getNode(
27611         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27612         DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
27613         MXCSRAddr);
27614   }
27615 
27616   return Chain;
27617 }
27618 
27619 static SDValue createSetFPEnvNodes(SDValue Ptr, SDValue Chain, SDLoc DL,
27620                                    EVT MemVT, MachineMemOperand *MMO,
27621                                    SelectionDAG &DAG,
27622                                    const X86Subtarget &Subtarget) {
27623   // Set x87 state, if it presents.
27624   if (Subtarget.hasX87())
27625     Chain =
27626         DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other),
27627                                 {Chain, Ptr}, MemVT, MMO);
27628   // If target supports SSE, set MXCSR as well.
27629   if (Subtarget.hasSSE1()) {
27630     // Get pointer to the MXCSR location in memory.
27631     MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27632     SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
27633                                     DAG.getConstant(X87StateSize, DL, PtrVT));
27634     // Load MXCSR from memory.
27635     Chain = DAG.getNode(
27636         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27637         DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
27638         MXCSRAddr);
27639   }
27640   return Chain;
27641 }
27642 
27643 SDValue X86TargetLowering::LowerSET_FPENV_MEM(SDValue Op,
27644                                               SelectionDAG &DAG) const {
27645   SDLoc DL(Op);
27646   SDValue Chain = Op->getOperand(0);
27647   SDValue Ptr = Op->getOperand(1);
27648   auto *Node = cast<FPStateAccessSDNode>(Op);
27649   EVT MemVT = Node->getMemoryVT();
27650   assert(MemVT.getSizeInBits() == FPStateSizeInBits);
27651   MachineMemOperand *MMO = cast<FPStateAccessSDNode>(Op)->getMemOperand();
27652   return createSetFPEnvNodes(Ptr, Chain, DL, MemVT, MMO, DAG, Subtarget);
27653 }
27654 
27655 SDValue X86TargetLowering::LowerRESET_FPENV(SDValue Op,
27656                                             SelectionDAG &DAG) const {
27657   MachineFunction &MF = DAG.getMachineFunction();
27658   SDLoc DL(Op);
27659   SDValue Chain = Op.getNode()->getOperand(0);
27660 
27661   IntegerType *ItemTy = Type::getInt32Ty(*DAG.getContext());
27662   ArrayType *FPEnvTy = ArrayType::get(ItemTy, 8);
27663   SmallVector<Constant *, 8> FPEnvVals;
27664 
27665   // x87 FPU Control Word: mask all floating-point exceptions, sets rounding to
27666   // nearest. FPU precision is set to 53 bits on Windows and 64 bits otherwise
27667   // for compatibility with glibc.
27668   unsigned X87CW = Subtarget.isTargetWindowsMSVC() ? 0x27F : 0x37F;
27669   FPEnvVals.push_back(ConstantInt::get(ItemTy, X87CW));
27670   Constant *Zero = ConstantInt::get(ItemTy, 0);
27671   for (unsigned I = 0; I < 6; ++I)
27672     FPEnvVals.push_back(Zero);
27673 
27674   // MXCSR: mask all floating-point exceptions, sets rounding to nearest, clear
27675   // all exceptions, sets DAZ and FTZ to 0.
27676   FPEnvVals.push_back(ConstantInt::get(ItemTy, 0x1F80));
27677   Constant *FPEnvBits = ConstantArray::get(FPEnvTy, FPEnvVals);
27678   MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27679   SDValue Env = DAG.getConstantPool(FPEnvBits, PtrVT);
27680   MachinePointerInfo MPI =
27681       MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
27682   MachineMemOperand *MMO = MF.getMachineMemOperand(
27683       MPI, MachineMemOperand::MOStore, X87StateSize, Align(4));
27684 
27685   return createSetFPEnvNodes(Env, Chain, DL, MVT::i32, MMO, DAG, Subtarget);
27686 }
27687 
27688 /// Lower a vector CTLZ using native supported vector CTLZ instruction.
27689 //
27690 // i8/i16 vector implemented using dword LZCNT vector instruction
27691 // ( sub(trunc(lzcnt(zext32(x)))) ). In case zext32(x) is illegal,
27692 // split the vector, perform operation on it's Lo a Hi part and
27693 // concatenate the results.
27694 static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
27695                                          const X86Subtarget &Subtarget) {
27696   assert(Op.getOpcode() == ISD::CTLZ);
27697   SDLoc dl(Op);
27698   MVT VT = Op.getSimpleValueType();
27699   MVT EltVT = VT.getVectorElementType();
27700   unsigned NumElems = VT.getVectorNumElements();
27701 
27702   assert((EltVT == MVT::i8 || EltVT == MVT::i16) &&
27703           "Unsupported element type");
27704 
27705   // Split vector, it's Lo and Hi parts will be handled in next iteration.
27706   if (NumElems > 16 ||
27707       (NumElems == 16 && !Subtarget.canExtendTo512DQ()))
27708     return splitVectorIntUnary(Op, DAG);
27709 
27710   MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
27711   assert((NewVT.is256BitVector() || NewVT.is512BitVector()) &&
27712           "Unsupported value type for operation");
27713 
27714   // Use native supported vector instruction vplzcntd.
27715   Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0));
27716   SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
27717   SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);
27718   SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);
27719 
27720   return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta);
27721 }
27722 
27723 // Lower CTLZ using a PSHUFB lookup table implementation.
27724 static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
27725                                        const X86Subtarget &Subtarget,
27726                                        SelectionDAG &DAG) {
27727   MVT VT = Op.getSimpleValueType();
27728   int NumElts = VT.getVectorNumElements();
27729   int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8);
27730   MVT CurrVT = MVT::getVectorVT(MVT::i8, NumBytes);
27731 
27732   // Per-nibble leading zero PSHUFB lookup table.
27733   const int LUT[16] = {/* 0 */ 4, /* 1 */ 3, /* 2 */ 2, /* 3 */ 2,
27734                        /* 4 */ 1, /* 5 */ 1, /* 6 */ 1, /* 7 */ 1,
27735                        /* 8 */ 0, /* 9 */ 0, /* a */ 0, /* b */ 0,
27736                        /* c */ 0, /* d */ 0, /* e */ 0, /* f */ 0};
27737 
27738   SmallVector<SDValue, 64> LUTVec;
27739   for (int i = 0; i < NumBytes; ++i)
27740     LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
27741   SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
27742 
27743   // Begin by bitcasting the input to byte vector, then split those bytes
27744   // into lo/hi nibbles and use the PSHUFB LUT to perform CLTZ on each of them.
27745   // If the hi input nibble is zero then we add both results together, otherwise
27746   // we just take the hi result (by masking the lo result to zero before the
27747   // add).
27748   SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
27749   SDValue Zero = DAG.getConstant(0, DL, CurrVT);
27750 
27751   SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
27752   SDValue Lo = Op0;
27753   SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
27754   SDValue HiZ;
27755   if (CurrVT.is512BitVector()) {
27756     MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());
27757     HiZ = DAG.getSetCC(DL, MaskVT, Hi, Zero, ISD::SETEQ);
27758     HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
27759   } else {
27760     HiZ = DAG.getSetCC(DL, CurrVT, Hi, Zero, ISD::SETEQ);
27761   }
27762 
27763   Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
27764   Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
27765   Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ);
27766   SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
27767 
27768   // Merge result back from vXi8 back to VT, working on the lo/hi halves
27769   // of the current vector width in the same way we did for the nibbles.
27770   // If the upper half of the input element is zero then add the halves'
27771   // leading zero counts together, otherwise just use the upper half's.
27772   // Double the width of the result until we are at target width.
27773   while (CurrVT != VT) {
27774     int CurrScalarSizeInBits = CurrVT.getScalarSizeInBits();
27775     int CurrNumElts = CurrVT.getVectorNumElements();
27776     MVT NextSVT = MVT::getIntegerVT(CurrScalarSizeInBits * 2);
27777     MVT NextVT = MVT::getVectorVT(NextSVT, CurrNumElts / 2);
27778     SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
27779 
27780     // Check if the upper half of the input element is zero.
27781     if (CurrVT.is512BitVector()) {
27782       MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());
27783       HiZ = DAG.getSetCC(DL, MaskVT, DAG.getBitcast(CurrVT, Op0),
27784                          DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
27785       HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
27786     } else {
27787       HiZ = DAG.getSetCC(DL, CurrVT, DAG.getBitcast(CurrVT, Op0),
27788                          DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
27789     }
27790     HiZ = DAG.getBitcast(NextVT, HiZ);
27791 
27792     // Move the upper/lower halves to the lower bits as we'll be extending to
27793     // NextVT. Mask the lower result to zero if HiZ is true and add the results
27794     // together.
27795     SDValue ResNext = Res = DAG.getBitcast(NextVT, Res);
27796     SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
27797     SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
27798     R1 = DAG.getNode(ISD::AND, DL, NextVT, ResNext, R1);
27799     Res = DAG.getNode(ISD::ADD, DL, NextVT, R0, R1);
27800     CurrVT = NextVT;
27801   }
27802 
27803   return Res;
27804 }
27805 
27806 static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
27807                                const X86Subtarget &Subtarget,
27808                                SelectionDAG &DAG) {
27809   MVT VT = Op.getSimpleValueType();
27810 
27811   if (Subtarget.hasCDI() &&
27812       // vXi8 vectors need to be promoted to 512-bits for vXi32.
27813       (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8))
27814     return LowerVectorCTLZ_AVX512CDI(Op, DAG, Subtarget);
27815 
27816   // Decompose 256-bit ops into smaller 128-bit ops.
27817   if (VT.is256BitVector() && !Subtarget.hasInt256())
27818     return splitVectorIntUnary(Op, DAG);
27819 
27820   // Decompose 512-bit ops into smaller 256-bit ops.
27821   if (VT.is512BitVector() && !Subtarget.hasBWI())
27822     return splitVectorIntUnary(Op, DAG);
27823 
27824   assert(Subtarget.hasSSSE3() && "Expected SSSE3 support for PSHUFB");
27825   return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
27826 }
27827 
27828 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
27829                          SelectionDAG &DAG) {
27830   MVT VT = Op.getSimpleValueType();
27831   MVT OpVT = VT;
27832   unsigned NumBits = VT.getSizeInBits();
27833   SDLoc dl(Op);
27834   unsigned Opc = Op.getOpcode();
27835 
27836   if (VT.isVector())
27837     return LowerVectorCTLZ(Op, dl, Subtarget, DAG);
27838 
27839   Op = Op.getOperand(0);
27840   if (VT == MVT::i8) {
27841     // Zero extend to i32 since there is not an i8 bsr.
27842     OpVT = MVT::i32;
27843     Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
27844   }
27845 
27846   // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
27847   SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
27848   Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
27849 
27850   if (Opc == ISD::CTLZ) {
27851     // If src is zero (i.e. bsr sets ZF), returns NumBits.
27852     SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
27853                      DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
27854                      Op.getValue(1)};
27855     Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
27856   }
27857 
27858   // Finally xor with NumBits-1.
27859   Op = DAG.getNode(ISD::XOR, dl, OpVT, Op,
27860                    DAG.getConstant(NumBits - 1, dl, OpVT));
27861 
27862   if (VT == MVT::i8)
27863     Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
27864   return Op;
27865 }
27866 
27867 static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
27868                          SelectionDAG &DAG) {
27869   MVT VT = Op.getSimpleValueType();
27870   unsigned NumBits = VT.getScalarSizeInBits();
27871   SDValue N0 = Op.getOperand(0);
27872   SDLoc dl(Op);
27873 
27874   assert(!VT.isVector() && Op.getOpcode() == ISD::CTTZ &&
27875          "Only scalar CTTZ requires custom lowering");
27876 
27877   // Issue a bsf (scan bits forward) which also sets EFLAGS.
27878   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
27879   Op = DAG.getNode(X86ISD::BSF, dl, VTs, N0);
27880 
27881   // If src is known never zero we can skip the CMOV.
27882   if (DAG.isKnownNeverZero(N0))
27883     return Op;
27884 
27885   // If src is zero (i.e. bsf sets ZF), returns NumBits.
27886   SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
27887                    DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
27888                    Op.getValue(1)};
27889   return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
27890 }
27891 
27892 static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
27893                            const X86Subtarget &Subtarget) {
27894   MVT VT = Op.getSimpleValueType();
27895   if (VT == MVT::i16 || VT == MVT::i32)
27896     return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
27897 
27898   if (VT == MVT::v32i16 || VT == MVT::v64i8)
27899     return splitVectorIntBinary(Op, DAG);
27900 
27901   assert(Op.getSimpleValueType().is256BitVector() &&
27902          Op.getSimpleValueType().isInteger() &&
27903          "Only handle AVX 256-bit vector integer operation");
27904   return splitVectorIntBinary(Op, DAG);
27905 }
27906 
27907 static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
27908                                   const X86Subtarget &Subtarget) {
27909   MVT VT = Op.getSimpleValueType();
27910   SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
27911   unsigned Opcode = Op.getOpcode();
27912   SDLoc DL(Op);
27913 
27914   if (VT == MVT::v32i16 || VT == MVT::v64i8 ||
27915       (VT.is256BitVector() && !Subtarget.hasInt256())) {
27916     assert(Op.getSimpleValueType().isInteger() &&
27917            "Only handle AVX vector integer operation");
27918     return splitVectorIntBinary(Op, DAG);
27919   }
27920 
27921   // Avoid the generic expansion with min/max if we don't have pminu*/pmaxu*.
27922   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27923   EVT SetCCResultType =
27924       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
27925 
27926   unsigned BitWidth = VT.getScalarSizeInBits();
27927   if (Opcode == ISD::USUBSAT) {
27928     if (!TLI.isOperationLegal(ISD::UMAX, VT) || useVPTERNLOG(Subtarget, VT)) {
27929       // Handle a special-case with a bit-hack instead of cmp+select:
27930       // usubsat X, SMIN --> (X ^ SMIN) & (X s>> BW-1)
27931       // If the target can use VPTERNLOG, DAGToDAG will match this as
27932       // "vpsra + vpternlog" which is better than "vpmax + vpsub" with a
27933       // "broadcast" constant load.
27934       ConstantSDNode *C = isConstOrConstSplat(Y, true);
27935       if (C && C->getAPIntValue().isSignMask()) {
27936         SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT);
27937         SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT);
27938         SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask);
27939         SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt);
27940         return DAG.getNode(ISD::AND, DL, VT, Xor, Sra);
27941       }
27942     }
27943     if (!TLI.isOperationLegal(ISD::UMAX, VT)) {
27944       // usubsat X, Y --> (X >u Y) ? X - Y : 0
27945       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
27946       SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
27947       // TODO: Move this to DAGCombiner?
27948       if (SetCCResultType == VT &&
27949           DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits())
27950         return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub);
27951       return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT));
27952     }
27953   }
27954 
27955   if ((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
27956       (!VT.isVector() || VT == MVT::v2i64)) {
27957     APInt MinVal = APInt::getSignedMinValue(BitWidth);
27958     APInt MaxVal = APInt::getSignedMaxValue(BitWidth);
27959     SDValue Zero = DAG.getConstant(0, DL, VT);
27960     SDValue Result =
27961         DAG.getNode(Opcode == ISD::SADDSAT ? ISD::SADDO : ISD::SSUBO, DL,
27962                     DAG.getVTList(VT, SetCCResultType), X, Y);
27963     SDValue SumDiff = Result.getValue(0);
27964     SDValue Overflow = Result.getValue(1);
27965     SDValue SatMin = DAG.getConstant(MinVal, DL, VT);
27966     SDValue SatMax = DAG.getConstant(MaxVal, DL, VT);
27967     SDValue SumNeg =
27968         DAG.getSetCC(DL, SetCCResultType, SumDiff, Zero, ISD::SETLT);
27969     Result = DAG.getSelect(DL, VT, SumNeg, SatMax, SatMin);
27970     return DAG.getSelect(DL, VT, Overflow, Result, SumDiff);
27971   }
27972 
27973   // Use default expansion.
27974   return SDValue();
27975 }
27976 
27977 static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
27978                         SelectionDAG &DAG) {
27979   MVT VT = Op.getSimpleValueType();
27980   if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) {
27981     // Since X86 does not have CMOV for 8-bit integer, we don't convert
27982     // 8-bit integer abs to NEG and CMOV.
27983     SDLoc DL(Op);
27984     SDValue N0 = Op.getOperand(0);
27985     SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
27986                               DAG.getConstant(0, DL, VT), N0);
27987     SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_NS, DL, MVT::i8),
27988                      SDValue(Neg.getNode(), 1)};
27989     return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
27990   }
27991 
27992   // ABS(vXi64 X) --> VPBLENDVPD(X, 0-X, X).
27993   if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) {
27994     SDLoc DL(Op);
27995     SDValue Src = Op.getOperand(0);
27996     SDValue Sub =
27997         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
27998     return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Sub, Src);
27999   }
28000 
28001   if (VT.is256BitVector() && !Subtarget.hasInt256()) {
28002     assert(VT.isInteger() &&
28003            "Only handle AVX 256-bit vector integer operation");
28004     return splitVectorIntUnary(Op, DAG);
28005   }
28006 
28007   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
28008     return splitVectorIntUnary(Op, DAG);
28009 
28010   // Default to expand.
28011   return SDValue();
28012 }
28013 
28014 static SDValue LowerAVG(SDValue Op, const X86Subtarget &Subtarget,
28015                         SelectionDAG &DAG) {
28016   MVT VT = Op.getSimpleValueType();
28017 
28018   // For AVX1 cases, split to use legal ops.
28019   if (VT.is256BitVector() && !Subtarget.hasInt256())
28020     return splitVectorIntBinary(Op, DAG);
28021 
28022   if (VT == MVT::v32i16 || VT == MVT::v64i8)
28023     return splitVectorIntBinary(Op, DAG);
28024 
28025   // Default to expand.
28026   return SDValue();
28027 }
28028 
28029 static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget,
28030                            SelectionDAG &DAG) {
28031   MVT VT = Op.getSimpleValueType();
28032 
28033   // For AVX1 cases, split to use legal ops.
28034   if (VT.is256BitVector() && !Subtarget.hasInt256())
28035     return splitVectorIntBinary(Op, DAG);
28036 
28037   if (VT == MVT::v32i16 || VT == MVT::v64i8)
28038     return splitVectorIntBinary(Op, DAG);
28039 
28040   // Default to expand.
28041   return SDValue();
28042 }
28043 
28044 static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
28045                                       SelectionDAG &DAG) {
28046   assert((Op.getOpcode() == ISD::FMAXIMUM || Op.getOpcode() == ISD::FMINIMUM) &&
28047          "Expected FMAXIMUM or FMINIMUM opcode");
28048   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28049   EVT VT = Op.getValueType();
28050   SDValue X = Op.getOperand(0);
28051   SDValue Y = Op.getOperand(1);
28052   SDLoc DL(Op);
28053   uint64_t SizeInBits = VT.getScalarSizeInBits();
28054   APInt PreferredZero = APInt::getZero(SizeInBits);
28055   APInt OppositeZero = PreferredZero;
28056   EVT IVT = VT.changeTypeToInteger();
28057   X86ISD::NodeType MinMaxOp;
28058   if (Op.getOpcode() == ISD::FMAXIMUM) {
28059     MinMaxOp = X86ISD::FMAX;
28060     OppositeZero.setSignBit();
28061   } else {
28062     PreferredZero.setSignBit();
28063     MinMaxOp = X86ISD::FMIN;
28064   }
28065   EVT SetCCType =
28066       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28067 
28068   // The tables below show the expected result of Max in cases of NaN and
28069   // signed zeros.
28070   //
28071   //                 Y                       Y
28072   //             Num   xNaN              +0     -0
28073   //          ---------------         ---------------
28074   //     Num  |  Max |   Y  |     +0  |  +0  |  +0  |
28075   // X        ---------------  X      ---------------
28076   //    xNaN  |   X  |  X/Y |     -0  |  +0  |  -0  |
28077   //          ---------------         ---------------
28078   //
28079   // It is achieved by means of FMAX/FMIN with preliminary checks and operand
28080   // reordering.
28081   //
28082   // We check if any of operands is NaN and return NaN. Then we check if any of
28083   // operands is zero or negative zero (for fmaximum and fminimum respectively)
28084   // to ensure the correct zero is returned.
28085   auto MatchesZero = [](SDValue Op, APInt Zero) {
28086     Op = peekThroughBitcasts(Op);
28087     if (auto *CstOp = dyn_cast<ConstantFPSDNode>(Op))
28088       return CstOp->getValueAPF().bitcastToAPInt() == Zero;
28089     if (auto *CstOp = dyn_cast<ConstantSDNode>(Op))
28090       return CstOp->getAPIntValue() == Zero;
28091     if (Op->getOpcode() == ISD::BUILD_VECTOR ||
28092         Op->getOpcode() == ISD::SPLAT_VECTOR) {
28093       for (const SDValue &OpVal : Op->op_values()) {
28094         if (OpVal.isUndef())
28095           continue;
28096         auto *CstOp = dyn_cast<ConstantFPSDNode>(OpVal);
28097         if (!CstOp)
28098           return false;
28099         if (!CstOp->getValueAPF().isZero())
28100           continue;
28101         if (CstOp->getValueAPF().bitcastToAPInt() != Zero)
28102           return false;
28103       }
28104       return true;
28105     }
28106     return false;
28107   };
28108 
28109   bool IsXNeverNaN = DAG.isKnownNeverNaN(X);
28110   bool IsYNeverNaN = DAG.isKnownNeverNaN(Y);
28111   bool IgnoreSignedZero = DAG.getTarget().Options.NoSignedZerosFPMath ||
28112                           Op->getFlags().hasNoSignedZeros() ||
28113                           DAG.isKnownNeverZeroFloat(X) ||
28114                           DAG.isKnownNeverZeroFloat(Y);
28115   SDValue NewX, NewY;
28116   if (IgnoreSignedZero || MatchesZero(Y, PreferredZero) ||
28117       MatchesZero(X, OppositeZero)) {
28118     // Operands are already in right order or order does not matter.
28119     NewX = X;
28120     NewY = Y;
28121   } else if (MatchesZero(X, PreferredZero) || MatchesZero(Y, OppositeZero)) {
28122     NewX = Y;
28123     NewY = X;
28124   } else if (!VT.isVector() && (VT == MVT::f16 || Subtarget.hasDQI()) &&
28125              (Op->getFlags().hasNoNaNs() || IsXNeverNaN || IsYNeverNaN)) {
28126     if (IsXNeverNaN)
28127       std::swap(X, Y);
28128     // VFPCLASSS consumes a vector type. So provide a minimal one corresponded
28129     // xmm register.
28130     MVT VectorType = MVT::getVectorVT(VT.getSimpleVT(), 128 / SizeInBits);
28131     SDValue VX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VectorType, X);
28132     // Bits of classes:
28133     // Bits  Imm8[0] Imm8[1] Imm8[2] Imm8[3] Imm8[4]  Imm8[5]  Imm8[6] Imm8[7]
28134     // Class    QNAN PosZero NegZero  PosINF  NegINF Denormal Negative    SNAN
28135     SDValue Imm = DAG.getTargetConstant(MinMaxOp == X86ISD::FMAX ? 0b11 : 0b101,
28136                                         DL, MVT::i32);
28137     SDValue IsNanZero = DAG.getNode(X86ISD::VFPCLASSS, DL, MVT::v1i1, VX, Imm);
28138     SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1,
28139                               DAG.getConstant(0, DL, MVT::v8i1), IsNanZero,
28140                               DAG.getIntPtrConstant(0, DL));
28141     SDValue NeedSwap = DAG.getBitcast(MVT::i8, Ins);
28142     NewX = DAG.getSelect(DL, VT, NeedSwap, Y, X);
28143     NewY = DAG.getSelect(DL, VT, NeedSwap, X, Y);
28144     return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
28145   } else {
28146     SDValue IsXSigned;
28147     if (Subtarget.is64Bit() || VT != MVT::f64) {
28148       SDValue XInt = DAG.getNode(ISD::BITCAST, DL, IVT, X);
28149       SDValue ZeroCst = DAG.getConstant(0, DL, IVT);
28150       IsXSigned = DAG.getSetCC(DL, SetCCType, XInt, ZeroCst, ISD::SETLT);
28151     } else {
28152       assert(VT == MVT::f64);
28153       SDValue Ins = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2f64,
28154                                 DAG.getConstantFP(0, DL, MVT::v2f64), X,
28155                                 DAG.getIntPtrConstant(0, DL));
28156       SDValue VX = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, Ins);
28157       SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX,
28158                                DAG.getIntPtrConstant(1, DL));
28159       Hi = DAG.getBitcast(MVT::i32, Hi);
28160       SDValue ZeroCst = DAG.getConstant(0, DL, MVT::i32);
28161       EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(),
28162                                              *DAG.getContext(), MVT::i32);
28163       IsXSigned = DAG.getSetCC(DL, SetCCType, Hi, ZeroCst, ISD::SETLT);
28164     }
28165     if (MinMaxOp == X86ISD::FMAX) {
28166       NewX = DAG.getSelect(DL, VT, IsXSigned, X, Y);
28167       NewY = DAG.getSelect(DL, VT, IsXSigned, Y, X);
28168     } else {
28169       NewX = DAG.getSelect(DL, VT, IsXSigned, Y, X);
28170       NewY = DAG.getSelect(DL, VT, IsXSigned, X, Y);
28171     }
28172   }
28173 
28174   bool IgnoreNaN = DAG.getTarget().Options.NoNaNsFPMath ||
28175                    Op->getFlags().hasNoNaNs() || (IsXNeverNaN && IsYNeverNaN);
28176 
28177   // If we did no ordering operands for signed zero handling and we need
28178   // to process NaN and we know that the second operand is not NaN then put
28179   // it in first operand and we will not need to post handle NaN after max/min.
28180   if (IgnoreSignedZero && !IgnoreNaN && DAG.isKnownNeverNaN(NewY))
28181     std::swap(NewX, NewY);
28182 
28183   SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
28184 
28185   if (IgnoreNaN || DAG.isKnownNeverNaN(NewX))
28186     return MinMax;
28187 
28188   SDValue IsNaN = DAG.getSetCC(DL, SetCCType, NewX, NewX, ISD::SETUO);
28189   return DAG.getSelect(DL, VT, IsNaN, NewX, MinMax);
28190 }
28191 
28192 static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
28193                         SelectionDAG &DAG) {
28194   MVT VT = Op.getSimpleValueType();
28195 
28196   // For AVX1 cases, split to use legal ops.
28197   if (VT.is256BitVector() && !Subtarget.hasInt256())
28198     return splitVectorIntBinary(Op, DAG);
28199 
28200   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.useBWIRegs())
28201     return splitVectorIntBinary(Op, DAG);
28202 
28203   SDLoc dl(Op);
28204   bool IsSigned = Op.getOpcode() == ISD::ABDS;
28205   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28206 
28207   // TODO: Move to TargetLowering expandABD() once we have ABD promotion.
28208   if (VT.isScalarInteger()) {
28209     unsigned WideBits = std::max<unsigned>(2 * VT.getScalarSizeInBits(), 32u);
28210     MVT WideVT = MVT::getIntegerVT(WideBits);
28211     if (TLI.isTypeLegal(WideVT)) {
28212       // abds(lhs, rhs) -> trunc(abs(sub(sext(lhs), sext(rhs))))
28213       // abdu(lhs, rhs) -> trunc(abs(sub(zext(lhs), zext(rhs))))
28214       unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28215       SDValue LHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(0));
28216       SDValue RHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(1));
28217       SDValue Diff = DAG.getNode(ISD::SUB, dl, WideVT, LHS, RHS);
28218       SDValue AbsDiff = DAG.getNode(ISD::ABS, dl, WideVT, Diff);
28219       return DAG.getNode(ISD::TRUNCATE, dl, VT, AbsDiff);
28220     }
28221   }
28222 
28223   // TODO: Move to TargetLowering expandABD().
28224   if (!Subtarget.hasSSE41() &&
28225       ((IsSigned && VT == MVT::v16i8) || VT == MVT::v4i32)) {
28226     SDValue LHS = DAG.getFreeze(Op.getOperand(0));
28227     SDValue RHS = DAG.getFreeze(Op.getOperand(1));
28228     ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
28229     SDValue Cmp = DAG.getSetCC(dl, VT, LHS, RHS, CC);
28230     SDValue Diff0 = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
28231     SDValue Diff1 = DAG.getNode(ISD::SUB, dl, VT, RHS, LHS);
28232     return getBitSelect(dl, VT, Diff0, Diff1, Cmp, DAG);
28233   }
28234 
28235   // Default to expand.
28236   return SDValue();
28237 }
28238 
28239 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
28240                         SelectionDAG &DAG) {
28241   SDLoc dl(Op);
28242   MVT VT = Op.getSimpleValueType();
28243 
28244   // Decompose 256-bit ops into 128-bit ops.
28245   if (VT.is256BitVector() && !Subtarget.hasInt256())
28246     return splitVectorIntBinary(Op, DAG);
28247 
28248   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
28249     return splitVectorIntBinary(Op, DAG);
28250 
28251   SDValue A = Op.getOperand(0);
28252   SDValue B = Op.getOperand(1);
28253 
28254   // Lower v16i8/v32i8/v64i8 mul as sign-extension to v8i16/v16i16/v32i16
28255   // vector pairs, multiply and truncate.
28256   if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) {
28257     unsigned NumElts = VT.getVectorNumElements();
28258 
28259     if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28260         (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28261       MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
28262       return DAG.getNode(
28263           ISD::TRUNCATE, dl, VT,
28264           DAG.getNode(ISD::MUL, dl, ExVT,
28265                       DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, A),
28266                       DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, B)));
28267     }
28268 
28269     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
28270 
28271     // Extract the lo/hi parts to any extend to i16.
28272     // We're going to mask off the low byte of each result element of the
28273     // pmullw, so it doesn't matter what's in the high byte of each 16-bit
28274     // element.
28275     SDValue Undef = DAG.getUNDEF(VT);
28276     SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));
28277     SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));
28278 
28279     SDValue BLo, BHi;
28280     if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
28281       // If the RHS is a constant, manually unpackl/unpackh.
28282       SmallVector<SDValue, 16> LoOps, HiOps;
28283       for (unsigned i = 0; i != NumElts; i += 16) {
28284         for (unsigned j = 0; j != 8; ++j) {
28285           LoOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j), dl,
28286                                                MVT::i16));
28287           HiOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j + 8), dl,
28288                                                MVT::i16));
28289         }
28290       }
28291 
28292       BLo = DAG.getBuildVector(ExVT, dl, LoOps);
28293       BHi = DAG.getBuildVector(ExVT, dl, HiOps);
28294     } else {
28295       BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef));
28296       BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef));
28297     }
28298 
28299     // Multiply, mask the lower 8bits of the lo/hi results and pack.
28300     SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
28301     SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
28302     return getPack(DAG, Subtarget, dl, VT, RLo, RHi);
28303   }
28304 
28305   // Lower v4i32 mul as 2x shuffle, 2x pmuludq, 2x shuffle.
28306   if (VT == MVT::v4i32) {
28307     assert(Subtarget.hasSSE2() && !Subtarget.hasSSE41() &&
28308            "Should not custom lower when pmulld is available!");
28309 
28310     // Extract the odd parts.
28311     static const int UnpackMask[] = { 1, -1, 3, -1 };
28312     SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
28313     SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
28314 
28315     // Multiply the even parts.
28316     SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
28317                                 DAG.getBitcast(MVT::v2i64, A),
28318                                 DAG.getBitcast(MVT::v2i64, B));
28319     // Now multiply odd parts.
28320     SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
28321                                DAG.getBitcast(MVT::v2i64, Aodds),
28322                                DAG.getBitcast(MVT::v2i64, Bodds));
28323 
28324     Evens = DAG.getBitcast(VT, Evens);
28325     Odds = DAG.getBitcast(VT, Odds);
28326 
28327     // Merge the two vectors back together with a shuffle. This expands into 2
28328     // shuffles.
28329     static const int ShufMask[] = { 0, 4, 2, 6 };
28330     return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
28331   }
28332 
28333   assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
28334          "Only know how to lower V2I64/V4I64/V8I64 multiply");
28335   assert(!Subtarget.hasDQI() && "DQI should use MULLQ");
28336 
28337   //  Ahi = psrlqi(a, 32);
28338   //  Bhi = psrlqi(b, 32);
28339   //
28340   //  AloBlo = pmuludq(a, b);
28341   //  AloBhi = pmuludq(a, Bhi);
28342   //  AhiBlo = pmuludq(Ahi, b);
28343   //
28344   //  Hi = psllqi(AloBhi + AhiBlo, 32);
28345   //  return AloBlo + Hi;
28346   KnownBits AKnown = DAG.computeKnownBits(A);
28347   KnownBits BKnown = DAG.computeKnownBits(B);
28348 
28349   APInt LowerBitsMask = APInt::getLowBitsSet(64, 32);
28350   bool ALoIsZero = LowerBitsMask.isSubsetOf(AKnown.Zero);
28351   bool BLoIsZero = LowerBitsMask.isSubsetOf(BKnown.Zero);
28352 
28353   APInt UpperBitsMask = APInt::getHighBitsSet(64, 32);
28354   bool AHiIsZero = UpperBitsMask.isSubsetOf(AKnown.Zero);
28355   bool BHiIsZero = UpperBitsMask.isSubsetOf(BKnown.Zero);
28356 
28357   SDValue Zero = DAG.getConstant(0, dl, VT);
28358 
28359   // Only multiply lo/hi halves that aren't known to be zero.
28360   SDValue AloBlo = Zero;
28361   if (!ALoIsZero && !BLoIsZero)
28362     AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
28363 
28364   SDValue AloBhi = Zero;
28365   if (!ALoIsZero && !BHiIsZero) {
28366     SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
28367     AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
28368   }
28369 
28370   SDValue AhiBlo = Zero;
28371   if (!AHiIsZero && !BLoIsZero) {
28372     SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
28373     AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
28374   }
28375 
28376   SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);
28377   Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG);
28378 
28379   return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi);
28380 }
28381 
28382 static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl,
28383                                      MVT VT, bool IsSigned,
28384                                      const X86Subtarget &Subtarget,
28385                                      SelectionDAG &DAG,
28386                                      SDValue *Low = nullptr) {
28387   unsigned NumElts = VT.getVectorNumElements();
28388 
28389   // For vXi8 we will unpack the low and high half of each 128 bit lane to widen
28390   // to a vXi16 type. Do the multiplies, shift the results and pack the half
28391   // lane results back together.
28392 
28393   // We'll take different approaches for signed and unsigned.
28394   // For unsigned we'll use punpcklbw/punpckhbw to put zero extend the bytes
28395   // and use pmullw to calculate the full 16-bit product.
28396   // For signed we'll use punpcklbw/punpckbw to extend the bytes to words and
28397   // shift them left into the upper byte of each word. This allows us to use
28398   // pmulhw to calculate the full 16-bit product. This trick means we don't
28399   // need to sign extend the bytes to use pmullw.
28400 
28401   MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
28402   SDValue Zero = DAG.getConstant(0, dl, VT);
28403 
28404   SDValue ALo, AHi;
28405   if (IsSigned) {
28406     ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A));
28407     AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A));
28408   } else {
28409     ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero));
28410     AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero));
28411   }
28412 
28413   SDValue BLo, BHi;
28414   if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
28415     // If the RHS is a constant, manually unpackl/unpackh and extend.
28416     SmallVector<SDValue, 16> LoOps, HiOps;
28417     for (unsigned i = 0; i != NumElts; i += 16) {
28418       for (unsigned j = 0; j != 8; ++j) {
28419         SDValue LoOp = B.getOperand(i + j);
28420         SDValue HiOp = B.getOperand(i + j + 8);
28421 
28422         if (IsSigned) {
28423           LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16);
28424           HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16);
28425           LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp,
28426                              DAG.getConstant(8, dl, MVT::i16));
28427           HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp,
28428                              DAG.getConstant(8, dl, MVT::i16));
28429         } else {
28430           LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);
28431           HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);
28432         }
28433 
28434         LoOps.push_back(LoOp);
28435         HiOps.push_back(HiOp);
28436       }
28437     }
28438 
28439     BLo = DAG.getBuildVector(ExVT, dl, LoOps);
28440     BHi = DAG.getBuildVector(ExVT, dl, HiOps);
28441   } else if (IsSigned) {
28442     BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
28443     BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
28444   } else {
28445     BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
28446     BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero));
28447   }
28448 
28449   // Multiply, lshr the upper 8bits to the lower 8bits of the lo/hi results and
28450   // pack back to vXi8.
28451   unsigned MulOpc = IsSigned ? ISD::MULHS : ISD::MUL;
28452   SDValue RLo = DAG.getNode(MulOpc, dl, ExVT, ALo, BLo);
28453   SDValue RHi = DAG.getNode(MulOpc, dl, ExVT, AHi, BHi);
28454 
28455   if (Low)
28456     *Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi);
28457 
28458   return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true);
28459 }
28460 
28461 static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
28462                          SelectionDAG &DAG) {
28463   SDLoc dl(Op);
28464   MVT VT = Op.getSimpleValueType();
28465   bool IsSigned = Op->getOpcode() == ISD::MULHS;
28466   unsigned NumElts = VT.getVectorNumElements();
28467   SDValue A = Op.getOperand(0);
28468   SDValue B = Op.getOperand(1);
28469 
28470   // Decompose 256-bit ops into 128-bit ops.
28471   if (VT.is256BitVector() && !Subtarget.hasInt256())
28472     return splitVectorIntBinary(Op, DAG);
28473 
28474   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
28475     return splitVectorIntBinary(Op, DAG);
28476 
28477   if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) {
28478     assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) ||
28479            (VT == MVT::v8i32 && Subtarget.hasInt256()) ||
28480            (VT == MVT::v16i32 && Subtarget.hasAVX512()));
28481 
28482     // PMULxD operations multiply each even value (starting at 0) of LHS with
28483     // the related value of RHS and produce a widen result.
28484     // E.g., PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
28485     // => <2 x i64> <ae|cg>
28486     //
28487     // In other word, to have all the results, we need to perform two PMULxD:
28488     // 1. one with the even values.
28489     // 2. one with the odd values.
28490     // To achieve #2, with need to place the odd values at an even position.
28491     //
28492     // Place the odd value at an even position (basically, shift all values 1
28493     // step to the left):
28494     const int Mask[] = {1, -1,  3, -1,  5, -1,  7, -1,
28495                         9, -1, 11, -1, 13, -1, 15, -1};
28496     // <a|b|c|d> => <b|undef|d|undef>
28497     SDValue Odd0 =
28498         DAG.getVectorShuffle(VT, dl, A, A, ArrayRef(&Mask[0], NumElts));
28499     // <e|f|g|h> => <f|undef|h|undef>
28500     SDValue Odd1 =
28501         DAG.getVectorShuffle(VT, dl, B, B, ArrayRef(&Mask[0], NumElts));
28502 
28503     // Emit two multiplies, one for the lower 2 ints and one for the higher 2
28504     // ints.
28505     MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2);
28506     unsigned Opcode =
28507         (IsSigned && Subtarget.hasSSE41()) ? X86ISD::PMULDQ : X86ISD::PMULUDQ;
28508     // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
28509     // => <2 x i64> <ae|cg>
28510     SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
28511                                                   DAG.getBitcast(MulVT, A),
28512                                                   DAG.getBitcast(MulVT, B)));
28513     // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
28514     // => <2 x i64> <bf|dh>
28515     SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
28516                                                   DAG.getBitcast(MulVT, Odd0),
28517                                                   DAG.getBitcast(MulVT, Odd1)));
28518 
28519     // Shuffle it back into the right order.
28520     SmallVector<int, 16> ShufMask(NumElts);
28521     for (int i = 0; i != (int)NumElts; ++i)
28522       ShufMask[i] = (i / 2) * 2 + ((i % 2) * NumElts) + 1;
28523 
28524     SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
28525 
28526     // If we have a signed multiply but no PMULDQ fix up the result of an
28527     // unsigned multiply.
28528     if (IsSigned && !Subtarget.hasSSE41()) {
28529       SDValue Zero = DAG.getConstant(0, dl, VT);
28530       SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
28531                                DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B);
28532       SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
28533                                DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A);
28534 
28535       SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
28536       Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup);
28537     }
28538 
28539     return Res;
28540   }
28541 
28542   // Only i8 vectors should need custom lowering after this.
28543   assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
28544          (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
28545          "Unsupported vector type");
28546 
28547   // Lower v16i8/v32i8 as extension to v8i16/v16i16 vector pairs, multiply,
28548   // logical shift down the upper half and pack back to i8.
28549 
28550   // With SSE41 we can use sign/zero extend, but for pre-SSE41 we unpack
28551   // and then ashr/lshr the upper bits down to the lower bits before multiply.
28552 
28553   if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28554       (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28555     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
28556     unsigned ExAVX = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28557     SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
28558     SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
28559     SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
28560     Mul = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28561     return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
28562   }
28563 
28564   return LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG);
28565 }
28566 
28567 // Custom lowering for SMULO/UMULO.
28568 static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget,
28569                          SelectionDAG &DAG) {
28570   MVT VT = Op.getSimpleValueType();
28571 
28572   // Scalars defer to LowerXALUO.
28573   if (!VT.isVector())
28574     return LowerXALUO(Op, DAG);
28575 
28576   SDLoc dl(Op);
28577   bool IsSigned = Op->getOpcode() == ISD::SMULO;
28578   SDValue A = Op.getOperand(0);
28579   SDValue B = Op.getOperand(1);
28580   EVT OvfVT = Op->getValueType(1);
28581 
28582   if ((VT == MVT::v32i8 && !Subtarget.hasInt256()) ||
28583       (VT == MVT::v64i8 && !Subtarget.hasBWI())) {
28584     // Extract the LHS Lo/Hi vectors
28585     SDValue LHSLo, LHSHi;
28586     std::tie(LHSLo, LHSHi) = splitVector(A, DAG, dl);
28587 
28588     // Extract the RHS Lo/Hi vectors
28589     SDValue RHSLo, RHSHi;
28590     std::tie(RHSLo, RHSHi) = splitVector(B, DAG, dl);
28591 
28592     EVT LoOvfVT, HiOvfVT;
28593     std::tie(LoOvfVT, HiOvfVT) = DAG.GetSplitDestVTs(OvfVT);
28594     SDVTList LoVTs = DAG.getVTList(LHSLo.getValueType(), LoOvfVT);
28595     SDVTList HiVTs = DAG.getVTList(LHSHi.getValueType(), HiOvfVT);
28596 
28597     // Issue the split operations.
28598     SDValue Lo = DAG.getNode(Op.getOpcode(), dl, LoVTs, LHSLo, RHSLo);
28599     SDValue Hi = DAG.getNode(Op.getOpcode(), dl, HiVTs, LHSHi, RHSHi);
28600 
28601     // Join the separate data results and the overflow results.
28602     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
28603     SDValue Ovf = DAG.getNode(ISD::CONCAT_VECTORS, dl, OvfVT, Lo.getValue(1),
28604                               Hi.getValue(1));
28605 
28606     return DAG.getMergeValues({Res, Ovf}, dl);
28607   }
28608 
28609   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28610   EVT SetccVT =
28611       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28612 
28613   if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28614       (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28615     unsigned NumElts = VT.getVectorNumElements();
28616     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
28617     unsigned ExAVX = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28618     SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
28619     SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
28620     SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
28621 
28622     SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
28623 
28624     SDValue Ovf;
28625     if (IsSigned) {
28626       SDValue High, LowSign;
28627       if (OvfVT.getVectorElementType() == MVT::i1 &&
28628           (Subtarget.hasBWI() || Subtarget.canExtendTo512DQ())) {
28629         // Rather the truncating try to do the compare on vXi16 or vXi32.
28630         // Shift the high down filling with sign bits.
28631         High = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Mul, 8, DAG);
28632         // Fill all 16 bits with the sign bit from the low.
28633         LowSign =
28634             getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExVT, Mul, 8, DAG);
28635         LowSign = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, LowSign,
28636                                              15, DAG);
28637         SetccVT = OvfVT;
28638         if (!Subtarget.hasBWI()) {
28639           // We can't do a vXi16 compare so sign extend to v16i32.
28640           High = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, High);
28641           LowSign = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, LowSign);
28642         }
28643       } else {
28644         // Otherwise do the compare at vXi8.
28645         High = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28646         High = DAG.getNode(ISD::TRUNCATE, dl, VT, High);
28647         LowSign =
28648             DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT));
28649       }
28650 
28651       Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE);
28652     } else {
28653       SDValue High =
28654           getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28655       if (OvfVT.getVectorElementType() == MVT::i1 &&
28656           (Subtarget.hasBWI() || Subtarget.canExtendTo512DQ())) {
28657         // Rather the truncating try to do the compare on vXi16 or vXi32.
28658         SetccVT = OvfVT;
28659         if (!Subtarget.hasBWI()) {
28660           // We can't do a vXi16 compare so sign extend to v16i32.
28661           High = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, High);
28662         }
28663       } else {
28664         // Otherwise do the compare at vXi8.
28665         High = DAG.getNode(ISD::TRUNCATE, dl, VT, High);
28666       }
28667 
28668       Ovf =
28669           DAG.getSetCC(dl, SetccVT, High,
28670                        DAG.getConstant(0, dl, High.getValueType()), ISD::SETNE);
28671     }
28672 
28673     Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT);
28674 
28675     return DAG.getMergeValues({Low, Ovf}, dl);
28676   }
28677 
28678   SDValue Low;
28679   SDValue High =
28680       LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG, &Low);
28681 
28682   SDValue Ovf;
28683   if (IsSigned) {
28684     // SMULO overflows if the high bits don't match the sign of the low.
28685     SDValue LowSign =
28686         DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT));
28687     Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE);
28688   } else {
28689     // UMULO overflows if the high bits are non-zero.
28690     Ovf =
28691         DAG.getSetCC(dl, SetccVT, High, DAG.getConstant(0, dl, VT), ISD::SETNE);
28692   }
28693 
28694   Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT);
28695 
28696   return DAG.getMergeValues({Low, Ovf}, dl);
28697 }
28698 
28699 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
28700   assert(Subtarget.isTargetWin64() && "Unexpected target");
28701   EVT VT = Op.getValueType();
28702   assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
28703          "Unexpected return type for lowering");
28704 
28705   if (isa<ConstantSDNode>(Op->getOperand(1))) {
28706     SmallVector<SDValue> Result;
28707     if (expandDIVREMByConstant(Op.getNode(), Result, MVT::i64, DAG))
28708       return DAG.getNode(ISD::BUILD_PAIR, SDLoc(Op), VT, Result[0], Result[1]);
28709   }
28710 
28711   RTLIB::Libcall LC;
28712   bool isSigned;
28713   switch (Op->getOpcode()) {
28714   default: llvm_unreachable("Unexpected request for libcall!");
28715   case ISD::SDIV:      isSigned = true;  LC = RTLIB::SDIV_I128;    break;
28716   case ISD::UDIV:      isSigned = false; LC = RTLIB::UDIV_I128;    break;
28717   case ISD::SREM:      isSigned = true;  LC = RTLIB::SREM_I128;    break;
28718   case ISD::UREM:      isSigned = false; LC = RTLIB::UREM_I128;    break;
28719   }
28720 
28721   SDLoc dl(Op);
28722   SDValue InChain = DAG.getEntryNode();
28723 
28724   TargetLowering::ArgListTy Args;
28725   TargetLowering::ArgListEntry Entry;
28726   for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) {
28727     EVT ArgVT = Op->getOperand(i).getValueType();
28728     assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
28729            "Unexpected argument type for lowering");
28730     SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
28731     int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
28732     MachinePointerInfo MPI =
28733         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
28734     Entry.Node = StackPtr;
28735     InChain =
28736         DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16));
28737     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
28738     Entry.Ty = PointerType::get(ArgTy,0);
28739     Entry.IsSExt = false;
28740     Entry.IsZExt = false;
28741     Args.push_back(Entry);
28742   }
28743 
28744   SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
28745                                          getPointerTy(DAG.getDataLayout()));
28746 
28747   TargetLowering::CallLoweringInfo CLI(DAG);
28748   CLI.setDebugLoc(dl)
28749       .setChain(InChain)
28750       .setLibCallee(
28751           getLibcallCallingConv(LC),
28752           static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee,
28753           std::move(Args))
28754       .setInRegister()
28755       .setSExtResult(isSigned)
28756       .setZExtResult(!isSigned);
28757 
28758   std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
28759   return DAG.getBitcast(VT, CallInfo.first);
28760 }
28761 
28762 SDValue X86TargetLowering::LowerWin64_FP_TO_INT128(SDValue Op,
28763                                                    SelectionDAG &DAG,
28764                                                    SDValue &Chain) const {
28765   assert(Subtarget.isTargetWin64() && "Unexpected target");
28766   EVT VT = Op.getValueType();
28767   bool IsStrict = Op->isStrictFPOpcode();
28768 
28769   SDValue Arg = Op.getOperand(IsStrict ? 1 : 0);
28770   EVT ArgVT = Arg.getValueType();
28771 
28772   assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
28773          "Unexpected return type for lowering");
28774 
28775   RTLIB::Libcall LC;
28776   if (Op->getOpcode() == ISD::FP_TO_SINT ||
28777       Op->getOpcode() == ISD::STRICT_FP_TO_SINT)
28778     LC = RTLIB::getFPTOSINT(ArgVT, VT);
28779   else
28780     LC = RTLIB::getFPTOUINT(ArgVT, VT);
28781   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
28782 
28783   SDLoc dl(Op);
28784   MakeLibCallOptions CallOptions;
28785   Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
28786 
28787   SDValue Result;
28788   // Expect the i128 argument returned as a v2i64 in xmm0, cast back to the
28789   // expected VT (i128).
28790   std::tie(Result, Chain) =
28791       makeLibCall(DAG, LC, MVT::v2i64, Arg, CallOptions, dl, Chain);
28792   Result = DAG.getBitcast(VT, Result);
28793   return Result;
28794 }
28795 
28796 SDValue X86TargetLowering::LowerWin64_INT128_TO_FP(SDValue Op,
28797                                                    SelectionDAG &DAG) const {
28798   assert(Subtarget.isTargetWin64() && "Unexpected target");
28799   EVT VT = Op.getValueType();
28800   bool IsStrict = Op->isStrictFPOpcode();
28801 
28802   SDValue Arg = Op.getOperand(IsStrict ? 1 : 0);
28803   EVT ArgVT = Arg.getValueType();
28804 
28805   assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
28806          "Unexpected argument type for lowering");
28807 
28808   RTLIB::Libcall LC;
28809   if (Op->getOpcode() == ISD::SINT_TO_FP ||
28810       Op->getOpcode() == ISD::STRICT_SINT_TO_FP)
28811     LC = RTLIB::getSINTTOFP(ArgVT, VT);
28812   else
28813     LC = RTLIB::getUINTTOFP(ArgVT, VT);
28814   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
28815 
28816   SDLoc dl(Op);
28817   MakeLibCallOptions CallOptions;
28818   SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
28819 
28820   // Pass the i128 argument as an indirect argument on the stack.
28821   SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
28822   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
28823   MachinePointerInfo MPI =
28824       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
28825   Chain = DAG.getStore(Chain, dl, Arg, StackPtr, MPI, Align(16));
28826 
28827   SDValue Result;
28828   std::tie(Result, Chain) =
28829       makeLibCall(DAG, LC, VT, StackPtr, CallOptions, dl, Chain);
28830   return IsStrict ? DAG.getMergeValues({Result, Chain}, dl) : Result;
28831 }
28832 
28833 // Return true if the required (according to Opcode) shift-imm form is natively
28834 // supported by the Subtarget
28835 static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget,
28836                                         unsigned Opcode) {
28837   if (!VT.isSimple())
28838     return false;
28839 
28840   if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))
28841     return false;
28842 
28843   if (VT.getScalarSizeInBits() < 16)
28844     return false;
28845 
28846   if (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
28847       (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI()))
28848     return true;
28849 
28850   bool LShift = (VT.is128BitVector() && Subtarget.hasSSE2()) ||
28851                 (VT.is256BitVector() && Subtarget.hasInt256());
28852 
28853   bool AShift = LShift && (Subtarget.hasAVX512() ||
28854                            (VT != MVT::v2i64 && VT != MVT::v4i64));
28855   return (Opcode == ISD::SRA) ? AShift : LShift;
28856 }
28857 
28858 // The shift amount is a variable, but it is the same for all vector lanes.
28859 // These instructions are defined together with shift-immediate.
28860 static
28861 bool supportedVectorShiftWithBaseAmnt(EVT VT, const X86Subtarget &Subtarget,
28862                                       unsigned Opcode) {
28863   return supportedVectorShiftWithImm(VT, Subtarget, Opcode);
28864 }
28865 
28866 // Return true if the required (according to Opcode) variable-shift form is
28867 // natively supported by the Subtarget
28868 static bool supportedVectorVarShift(EVT VT, const X86Subtarget &Subtarget,
28869                                     unsigned Opcode) {
28870   if (!VT.isSimple())
28871     return false;
28872 
28873   if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))
28874     return false;
28875 
28876   if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16)
28877     return false;
28878 
28879   // vXi16 supported only on AVX-512, BWI
28880   if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI())
28881     return false;
28882 
28883   if (Subtarget.hasAVX512() &&
28884       (Subtarget.useAVX512Regs() || !VT.is512BitVector()))
28885     return true;
28886 
28887   bool LShift = VT.is128BitVector() || VT.is256BitVector();
28888   bool AShift = LShift &&  VT != MVT::v2i64 && VT != MVT::v4i64;
28889   return (Opcode == ISD::SRA) ? AShift : LShift;
28890 }
28891 
28892 static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
28893                                            const X86Subtarget &Subtarget) {
28894   MVT VT = Op.getSimpleValueType();
28895   SDLoc dl(Op);
28896   SDValue R = Op.getOperand(0);
28897   SDValue Amt = Op.getOperand(1);
28898   unsigned X86Opc = getTargetVShiftUniformOpcode(Op.getOpcode(), false);
28899 
28900   auto ArithmeticShiftRight64 = [&](uint64_t ShiftAmt) {
28901     assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type");
28902     MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);
28903     SDValue Ex = DAG.getBitcast(ExVT, R);
28904 
28905     // ashr(R, 63) === cmp_slt(R, 0)
28906     if (ShiftAmt == 63 && Subtarget.hasSSE42()) {
28907       assert((VT != MVT::v4i64 || Subtarget.hasInt256()) &&
28908              "Unsupported PCMPGT op");
28909       return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R);
28910     }
28911 
28912     if (ShiftAmt >= 32) {
28913       // Splat sign to upper i32 dst, and SRA upper i32 src to lower i32.
28914       SDValue Upper =
28915           getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG);
28916       SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
28917                                                  ShiftAmt - 32, DAG);
28918       if (VT == MVT::v2i64)
28919         Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {5, 1, 7, 3});
28920       if (VT == MVT::v4i64)
28921         Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
28922                                   {9, 1, 11, 3, 13, 5, 15, 7});
28923     } else {
28924       // SRA upper i32, SRL whole i64 and select lower i32.
28925       SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
28926                                                  ShiftAmt, DAG);
28927       SDValue Lower =
28928           getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);
28929       Lower = DAG.getBitcast(ExVT, Lower);
28930       if (VT == MVT::v2i64)
28931         Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {4, 1, 6, 3});
28932       if (VT == MVT::v4i64)
28933         Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
28934                                   {8, 1, 10, 3, 12, 5, 14, 7});
28935     }
28936     return DAG.getBitcast(VT, Ex);
28937   };
28938 
28939   // Optimize shl/srl/sra with constant shift amount.
28940   APInt APIntShiftAmt;
28941   if (!X86::isConstantSplat(Amt, APIntShiftAmt))
28942     return SDValue();
28943 
28944   // If the shift amount is out of range, return undef.
28945   if (APIntShiftAmt.uge(VT.getScalarSizeInBits()))
28946     return DAG.getUNDEF(VT);
28947 
28948   uint64_t ShiftAmt = APIntShiftAmt.getZExtValue();
28949 
28950   if (supportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode())) {
28951     // Hardware support for vector shifts is sparse which makes us scalarize the
28952     // vector operations in many cases. Also, on sandybridge ADD is faster than
28953     // shl: (shl V, 1) -> (add (freeze V), (freeze V))
28954     if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1) {
28955       // R may be undef at run-time, but (shl R, 1) must be an even number (LSB
28956       // must be 0). (add undef, undef) however can be any value. To make this
28957       // safe, we must freeze R to ensure that register allocation uses the same
28958       // register for an undefined value. This ensures that the result will
28959       // still be even and preserves the original semantics.
28960       R = DAG.getFreeze(R);
28961       return DAG.getNode(ISD::ADD, dl, VT, R, R);
28962     }
28963 
28964     return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG);
28965   }
28966 
28967   // i64 SRA needs to be performed as partial shifts.
28968   if (((!Subtarget.hasXOP() && VT == MVT::v2i64) ||
28969        (Subtarget.hasInt256() && VT == MVT::v4i64)) &&
28970       Op.getOpcode() == ISD::SRA)
28971     return ArithmeticShiftRight64(ShiftAmt);
28972 
28973   if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) ||
28974       (Subtarget.hasBWI() && VT == MVT::v64i8)) {
28975     unsigned NumElts = VT.getVectorNumElements();
28976     MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
28977 
28978     // Simple i8 add case
28979     if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1) {
28980       // R may be undef at run-time, but (shl R, 1) must be an even number (LSB
28981       // must be 0). (add undef, undef) however can be any value. To make this
28982       // safe, we must freeze R to ensure that register allocation uses the same
28983       // register for an undefined value. This ensures that the result will
28984       // still be even and preserves the original semantics.
28985       R = DAG.getFreeze(R);
28986       return DAG.getNode(ISD::ADD, dl, VT, R, R);
28987     }
28988 
28989     // ashr(R, 7)  === cmp_slt(R, 0)
28990     if (Op.getOpcode() == ISD::SRA && ShiftAmt == 7) {
28991       SDValue Zeros = DAG.getConstant(0, dl, VT);
28992       if (VT.is512BitVector()) {
28993         assert(VT == MVT::v64i8 && "Unexpected element type!");
28994         SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);
28995         return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP);
28996       }
28997       return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
28998     }
28999 
29000     // XOP can shift v16i8 directly instead of as shift v8i16 + mask.
29001     if (VT == MVT::v16i8 && Subtarget.hasXOP())
29002       return SDValue();
29003 
29004     if (Op.getOpcode() == ISD::SHL) {
29005       // Make a large shift.
29006       SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT, R,
29007                                                ShiftAmt, DAG);
29008       SHL = DAG.getBitcast(VT, SHL);
29009       // Zero out the rightmost bits.
29010       APInt Mask = APInt::getHighBitsSet(8, 8 - ShiftAmt);
29011       return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT));
29012     }
29013     if (Op.getOpcode() == ISD::SRL) {
29014       // Make a large shift.
29015       SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT, R,
29016                                                ShiftAmt, DAG);
29017       SRL = DAG.getBitcast(VT, SRL);
29018       // Zero out the leftmost bits.
29019       APInt Mask = APInt::getLowBitsSet(8, 8 - ShiftAmt);
29020       return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT));
29021     }
29022     if (Op.getOpcode() == ISD::SRA) {
29023       // ashr(R, Amt) === sub(xor(lshr(R, Amt), Mask), Mask)
29024       SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
29025 
29026       SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);
29027       Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
29028       Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
29029       return Res;
29030     }
29031     llvm_unreachable("Unknown shift opcode.");
29032   }
29033 
29034   return SDValue();
29035 }
29036 
29037 static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG,
29038                                           const X86Subtarget &Subtarget) {
29039   MVT VT = Op.getSimpleValueType();
29040   SDLoc dl(Op);
29041   SDValue R = Op.getOperand(0);
29042   SDValue Amt = Op.getOperand(1);
29043   unsigned Opcode = Op.getOpcode();
29044   unsigned X86OpcI = getTargetVShiftUniformOpcode(Opcode, false);
29045 
29046   int BaseShAmtIdx = -1;
29047   if (SDValue BaseShAmt = DAG.getSplatSourceVector(Amt, BaseShAmtIdx)) {
29048     if (supportedVectorShiftWithBaseAmnt(VT, Subtarget, Opcode))
29049       return getTargetVShiftNode(X86OpcI, dl, VT, R, BaseShAmt, BaseShAmtIdx,
29050                                  Subtarget, DAG);
29051 
29052     // vXi8 shifts - shift as v8i16 + mask result.
29053     if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) ||
29054          (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) ||
29055          VT == MVT::v64i8) &&
29056         !Subtarget.hasXOP()) {
29057       unsigned NumElts = VT.getVectorNumElements();
29058       MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
29059       if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, Opcode)) {
29060         unsigned LogicalOp = (Opcode == ISD::SHL ? ISD::SHL : ISD::SRL);
29061         unsigned LogicalX86Op = getTargetVShiftUniformOpcode(LogicalOp, false);
29062 
29063         // Create the mask using vXi16 shifts. For shift-rights we need to move
29064         // the upper byte down before splatting the vXi8 mask.
29065         SDValue BitMask = DAG.getConstant(-1, dl, ExtVT);
29066         BitMask = getTargetVShiftNode(LogicalX86Op, dl, ExtVT, BitMask,
29067                                       BaseShAmt, BaseShAmtIdx, Subtarget, DAG);
29068         if (Opcode != ISD::SHL)
29069           BitMask = getTargetVShiftByConstNode(LogicalX86Op, dl, ExtVT, BitMask,
29070                                                8, DAG);
29071         BitMask = DAG.getBitcast(VT, BitMask);
29072         BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask,
29073                                        SmallVector<int, 64>(NumElts, 0));
29074 
29075         SDValue Res = getTargetVShiftNode(LogicalX86Op, dl, ExtVT,
29076                                           DAG.getBitcast(ExtVT, R), BaseShAmt,
29077                                           BaseShAmtIdx, Subtarget, DAG);
29078         Res = DAG.getBitcast(VT, Res);
29079         Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask);
29080 
29081         if (Opcode == ISD::SRA) {
29082           // ashr(R, Amt) === sub(xor(lshr(R, Amt), SignMask), SignMask)
29083           // SignMask = lshr(SignBit, Amt) - safe to do this with PSRLW.
29084           SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT);
29085           SignMask =
29086               getTargetVShiftNode(LogicalX86Op, dl, ExtVT, SignMask, BaseShAmt,
29087                                   BaseShAmtIdx, Subtarget, DAG);
29088           SignMask = DAG.getBitcast(VT, SignMask);
29089           Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask);
29090           Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask);
29091         }
29092         return Res;
29093       }
29094     }
29095   }
29096 
29097   return SDValue();
29098 }
29099 
29100 // Convert a shift/rotate left amount to a multiplication scale factor.
29101 static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
29102                                        const X86Subtarget &Subtarget,
29103                                        SelectionDAG &DAG) {
29104   MVT VT = Amt.getSimpleValueType();
29105   if (!(VT == MVT::v8i16 || VT == MVT::v4i32 ||
29106         (Subtarget.hasInt256() && VT == MVT::v16i16) ||
29107         (Subtarget.hasAVX512() && VT == MVT::v32i16) ||
29108         (!Subtarget.hasAVX512() && VT == MVT::v16i8) ||
29109         (Subtarget.hasInt256() && VT == MVT::v32i8) ||
29110         (Subtarget.hasBWI() && VT == MVT::v64i8)))
29111     return SDValue();
29112 
29113   MVT SVT = VT.getVectorElementType();
29114   unsigned SVTBits = SVT.getSizeInBits();
29115   unsigned NumElems = VT.getVectorNumElements();
29116 
29117   APInt UndefElts;
29118   SmallVector<APInt> EltBits;
29119   if (getTargetConstantBitsFromNode(Amt, SVTBits, UndefElts, EltBits)) {
29120     APInt One(SVTBits, 1);
29121     SmallVector<SDValue> Elts(NumElems, DAG.getUNDEF(SVT));
29122     for (unsigned I = 0; I != NumElems; ++I) {
29123       if (UndefElts[I] || EltBits[I].uge(SVTBits))
29124         continue;
29125       uint64_t ShAmt = EltBits[I].getZExtValue();
29126       Elts[I] = DAG.getConstant(One.shl(ShAmt), dl, SVT);
29127     }
29128     return DAG.getBuildVector(VT, dl, Elts);
29129   }
29130 
29131   // If the target doesn't support variable shifts, use either FP conversion
29132   // or integer multiplication to avoid shifting each element individually.
29133   if (VT == MVT::v4i32) {
29134     Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT));
29135     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt,
29136                       DAG.getConstant(0x3f800000U, dl, VT));
29137     Amt = DAG.getBitcast(MVT::v4f32, Amt);
29138     return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt);
29139   }
29140 
29141   // AVX2 can more effectively perform this as a zext/trunc to/from v8i32.
29142   if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) {
29143     SDValue Z = DAG.getConstant(0, dl, VT);
29144     SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));
29145     SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));
29146     Lo = convertShiftLeftToScale(Lo, dl, Subtarget, DAG);
29147     Hi = convertShiftLeftToScale(Hi, dl, Subtarget, DAG);
29148     if (Subtarget.hasSSE41())
29149       return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
29150     return getPack(DAG, Subtarget, dl, VT, Lo, Hi);
29151   }
29152 
29153   return SDValue();
29154 }
29155 
29156 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
29157                           SelectionDAG &DAG) {
29158   MVT VT = Op.getSimpleValueType();
29159   SDLoc dl(Op);
29160   SDValue R = Op.getOperand(0);
29161   SDValue Amt = Op.getOperand(1);
29162   unsigned EltSizeInBits = VT.getScalarSizeInBits();
29163   bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
29164 
29165   unsigned Opc = Op.getOpcode();
29166   unsigned X86OpcV = getTargetVShiftUniformOpcode(Opc, true);
29167   unsigned X86OpcI = getTargetVShiftUniformOpcode(Opc, false);
29168 
29169   assert(VT.isVector() && "Custom lowering only for vector shifts!");
29170   assert(Subtarget.hasSSE2() && "Only custom lower when we have SSE2!");
29171 
29172   if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget))
29173     return V;
29174 
29175   if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget))
29176     return V;
29177 
29178   if (supportedVectorVarShift(VT, Subtarget, Opc))
29179     return Op;
29180 
29181   // i64 vector arithmetic shift can be emulated with the transform:
29182   // M = lshr(SIGN_MASK, Amt)
29183   // ashr(R, Amt) === sub(xor(lshr(R, Amt), M), M)
29184   if (((VT == MVT::v2i64 && !Subtarget.hasXOP()) ||
29185        (VT == MVT::v4i64 && Subtarget.hasInt256())) &&
29186       Opc == ISD::SRA) {
29187     SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
29188     SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
29189     R = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
29190     R = DAG.getNode(ISD::XOR, dl, VT, R, M);
29191     R = DAG.getNode(ISD::SUB, dl, VT, R, M);
29192     return R;
29193   }
29194 
29195   // XOP has 128-bit variable logical/arithmetic shifts.
29196   // +ve/-ve Amt = shift left/right.
29197   if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 ||
29198                              VT == MVT::v8i16 || VT == MVT::v16i8)) {
29199     if (Opc == ISD::SRL || Opc == ISD::SRA) {
29200       SDValue Zero = DAG.getConstant(0, dl, VT);
29201       Amt = DAG.getNode(ISD::SUB, dl, VT, Zero, Amt);
29202     }
29203     if (Opc == ISD::SHL || Opc == ISD::SRL)
29204       return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);
29205     if (Opc == ISD::SRA)
29206       return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);
29207   }
29208 
29209   // 2i64 vector logical shifts can efficiently avoid scalarization - do the
29210   // shifts per-lane and then shuffle the partial results back together.
29211   if (VT == MVT::v2i64 && Opc != ISD::SRA) {
29212     // Splat the shift amounts so the scalar shifts above will catch it.
29213     SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
29214     SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
29215     SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
29216     SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
29217     return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});
29218   }
29219 
29220   // If possible, lower this shift as a sequence of two shifts by
29221   // constant plus a BLENDing shuffle instead of scalarizing it.
29222   // Example:
29223   //   (v4i32 (srl A, (build_vector < X, Y, Y, Y>)))
29224   //
29225   // Could be rewritten as:
29226   //   (v4i32 (MOVSS (srl A, <Y,Y,Y,Y>), (srl A, <X,X,X,X>)))
29227   //
29228   // The advantage is that the two shifts from the example would be
29229   // lowered as X86ISD::VSRLI nodes in parallel before blending.
29230   if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
29231                       (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
29232     SDValue Amt1, Amt2;
29233     unsigned NumElts = VT.getVectorNumElements();
29234     SmallVector<int, 8> ShuffleMask;
29235     for (unsigned i = 0; i != NumElts; ++i) {
29236       SDValue A = Amt->getOperand(i);
29237       if (A.isUndef()) {
29238         ShuffleMask.push_back(SM_SentinelUndef);
29239         continue;
29240       }
29241       if (!Amt1 || Amt1 == A) {
29242         ShuffleMask.push_back(i);
29243         Amt1 = A;
29244         continue;
29245       }
29246       if (!Amt2 || Amt2 == A) {
29247         ShuffleMask.push_back(i + NumElts);
29248         Amt2 = A;
29249         continue;
29250       }
29251       break;
29252     }
29253 
29254     // Only perform this blend if we can perform it without loading a mask.
29255     if (ShuffleMask.size() == NumElts && Amt1 && Amt2 &&
29256         (VT != MVT::v16i16 ||
29257          is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) &&
29258         (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL ||
29259          canWidenShuffleElements(ShuffleMask))) {
29260       auto *Cst1 = dyn_cast<ConstantSDNode>(Amt1);
29261       auto *Cst2 = dyn_cast<ConstantSDNode>(Amt2);
29262       if (Cst1 && Cst2 && Cst1->getAPIntValue().ult(EltSizeInBits) &&
29263           Cst2->getAPIntValue().ult(EltSizeInBits)) {
29264         SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
29265                                                     Cst1->getZExtValue(), DAG);
29266         SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
29267                                                     Cst2->getZExtValue(), DAG);
29268         return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask);
29269       }
29270     }
29271   }
29272 
29273   // If possible, lower this packed shift into a vector multiply instead of
29274   // expanding it into a sequence of scalar shifts.
29275   // For v32i8 cases, it might be quicker to split/extend to vXi16 shifts.
29276   if (Opc == ISD::SHL && !(VT == MVT::v32i8 && (Subtarget.hasXOP() ||
29277                                                 Subtarget.canExtendTo512BW())))
29278     if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG))
29279       return DAG.getNode(ISD::MUL, dl, VT, R, Scale);
29280 
29281   // Constant ISD::SRL can be performed efficiently on vXi16 vectors as we
29282   // can replace with ISD::MULHU, creating scale factor from (NumEltBits - Amt).
29283   if (Opc == ISD::SRL && ConstantAmt &&
29284       (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
29285     SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
29286     SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
29287     if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
29288       SDValue Zero = DAG.getConstant(0, dl, VT);
29289       SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
29290       SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
29291       return DAG.getSelect(dl, VT, ZAmt, R, Res);
29292     }
29293   }
29294 
29295   // Constant ISD::SRA can be performed efficiently on vXi16 vectors as we
29296   // can replace with ISD::MULHS, creating scale factor from (NumEltBits - Amt).
29297   // TODO: Special case handling for shift by 0/1, really we can afford either
29298   // of these cases in pre-SSE41/XOP/AVX512 but not both.
29299   if (Opc == ISD::SRA && ConstantAmt &&
29300       (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) &&
29301       ((Subtarget.hasSSE41() && !Subtarget.hasXOP() &&
29302         !Subtarget.hasAVX512()) ||
29303        DAG.isKnownNeverZero(Amt))) {
29304     SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
29305     SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
29306     if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
29307       SDValue Amt0 =
29308           DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ);
29309       SDValue Amt1 =
29310           DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ);
29311       SDValue Sra1 =
29312           getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG);
29313       SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
29314       Res = DAG.getSelect(dl, VT, Amt0, R, Res);
29315       return DAG.getSelect(dl, VT, Amt1, Sra1, Res);
29316     }
29317   }
29318 
29319   // v4i32 Non Uniform Shifts.
29320   // If the shift amount is constant we can shift each lane using the SSE2
29321   // immediate shifts, else we need to zero-extend each lane to the lower i64
29322   // and shift using the SSE2 variable shifts.
29323   // The separate results can then be blended together.
29324   if (VT == MVT::v4i32) {
29325     SDValue Amt0, Amt1, Amt2, Amt3;
29326     if (ConstantAmt) {
29327       Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});
29328       Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});
29329       Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});
29330       Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});
29331     } else {
29332       // The SSE2 shifts use the lower i64 as the same shift amount for
29333       // all lanes and the upper i64 is ignored. On AVX we're better off
29334       // just zero-extending, but for SSE just duplicating the top 16-bits is
29335       // cheaper and has the same effect for out of range values.
29336       if (Subtarget.hasAVX()) {
29337         SDValue Z = DAG.getConstant(0, dl, VT);
29338         Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1});
29339         Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1});
29340         Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1});
29341         Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1});
29342       } else {
29343         SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);
29344         SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
29345                                              {4, 5, 6, 7, -1, -1, -1, -1});
29346         SDValue Msk02 = getV4X86ShuffleImm8ForMask({0, 1, 1, 1}, dl, DAG);
29347         SDValue Msk13 = getV4X86ShuffleImm8ForMask({2, 3, 3, 3}, dl, DAG);
29348         Amt0 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk02);
29349         Amt1 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk13);
29350         Amt2 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk02);
29351         Amt3 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk13);
29352       }
29353     }
29354 
29355     unsigned ShOpc = ConstantAmt ? Opc : X86OpcV;
29356     SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
29357     SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
29358     SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
29359     SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
29360 
29361     // Merge the shifted lane results optimally with/without PBLENDW.
29362     // TODO - ideally shuffle combining would handle this.
29363     if (Subtarget.hasSSE41()) {
29364       SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
29365       SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
29366       return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7});
29367     }
29368     SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
29369     SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
29370     return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7});
29371   }
29372 
29373   // It's worth extending once and using the vXi16/vXi32 shifts for smaller
29374   // types, but without AVX512 the extra overheads to get from vXi8 to vXi32
29375   // make the existing SSE solution better.
29376   // NOTE: We honor prefered vector width before promoting to 512-bits.
29377   if ((Subtarget.hasInt256() && VT == MVT::v8i16) ||
29378       (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) ||
29379       (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) ||
29380       (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) ||
29381       (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) {
29382     assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) &&
29383            "Unexpected vector type");
29384     MVT EvtSVT = Subtarget.hasBWI() ? MVT::i16 : MVT::i32;
29385     MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements());
29386     unsigned ExtOpc = Opc == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
29387     R = DAG.getNode(ExtOpc, dl, ExtVT, R);
29388     Amt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Amt);
29389     return DAG.getNode(ISD::TRUNCATE, dl, VT,
29390                        DAG.getNode(Opc, dl, ExtVT, R, Amt));
29391   }
29392 
29393   // Constant ISD::SRA/SRL can be performed efficiently on vXi8 vectors as we
29394   // extend to vXi16 to perform a MUL scale effectively as a MUL_LOHI.
29395   if (ConstantAmt && (Opc == ISD::SRA || Opc == ISD::SRL) &&
29396       (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
29397        (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
29398       !Subtarget.hasXOP()) {
29399     int NumElts = VT.getVectorNumElements();
29400     SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
29401 
29402     // Extend constant shift amount to vXi16 (it doesn't matter if the type
29403     // isn't legal).
29404     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
29405     Amt = DAG.getZExtOrTrunc(Amt, dl, ExVT);
29406     Amt = DAG.getNode(ISD::SUB, dl, ExVT, DAG.getConstant(8, dl, ExVT), Amt);
29407     Amt = DAG.getNode(ISD::SHL, dl, ExVT, DAG.getConstant(1, dl, ExVT), Amt);
29408     assert(ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) &&
29409            "Constant build vector expected");
29410 
29411     if (VT == MVT::v16i8 && Subtarget.hasInt256()) {
29412       bool IsSigned = Opc == ISD::SRA;
29413       R = DAG.getExtOrTrunc(IsSigned, R, dl, ExVT);
29414       R = DAG.getNode(ISD::MUL, dl, ExVT, R, Amt);
29415       R = DAG.getNode(X86ISD::VSRLI, dl, ExVT, R, Cst8);
29416       return DAG.getZExtOrTrunc(R, dl, VT);
29417     }
29418 
29419     SmallVector<SDValue, 16> LoAmt, HiAmt;
29420     for (int i = 0; i != NumElts; i += 16) {
29421       for (int j = 0; j != 8; ++j) {
29422         LoAmt.push_back(Amt.getOperand(i + j));
29423         HiAmt.push_back(Amt.getOperand(i + j + 8));
29424       }
29425     }
29426 
29427     MVT VT16 = MVT::getVectorVT(MVT::i16, NumElts / 2);
29428     SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt);
29429     SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt);
29430 
29431     SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
29432     SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
29433     LoR = DAG.getNode(X86OpcI, dl, VT16, LoR, Cst8);
29434     HiR = DAG.getNode(X86OpcI, dl, VT16, HiR, Cst8);
29435     LoR = DAG.getNode(ISD::MUL, dl, VT16, LoR, LoA);
29436     HiR = DAG.getNode(ISD::MUL, dl, VT16, HiR, HiA);
29437     LoR = DAG.getNode(X86ISD::VSRLI, dl, VT16, LoR, Cst8);
29438     HiR = DAG.getNode(X86ISD::VSRLI, dl, VT16, HiR, Cst8);
29439     return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR);
29440   }
29441 
29442   if (VT == MVT::v16i8 ||
29443       (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) ||
29444       (VT == MVT::v64i8 && Subtarget.hasBWI())) {
29445     MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);
29446 
29447     auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
29448       if (VT.is512BitVector()) {
29449         // On AVX512BW targets we make use of the fact that VSELECT lowers
29450         // to a masked blend which selects bytes based just on the sign bit
29451         // extracted to a mask.
29452         MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
29453         V0 = DAG.getBitcast(VT, V0);
29454         V1 = DAG.getBitcast(VT, V1);
29455         Sel = DAG.getBitcast(VT, Sel);
29456         Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel,
29457                            ISD::SETGT);
29458         return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
29459       } else if (Subtarget.hasSSE41()) {
29460         // On SSE41 targets we can use PBLENDVB which selects bytes based just
29461         // on the sign bit.
29462         V0 = DAG.getBitcast(VT, V0);
29463         V1 = DAG.getBitcast(VT, V1);
29464         Sel = DAG.getBitcast(VT, Sel);
29465         return DAG.getBitcast(SelVT,
29466                               DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1));
29467       }
29468       // On pre-SSE41 targets we test for the sign bit by comparing to
29469       // zero - a negative value will set all bits of the lanes to true
29470       // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
29471       SDValue Z = DAG.getConstant(0, dl, SelVT);
29472       SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
29473       return DAG.getSelect(dl, SelVT, C, V0, V1);
29474     };
29475 
29476     // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
29477     // We can safely do this using i16 shifts as we're only interested in
29478     // the 3 lower bits of each byte.
29479     Amt = DAG.getBitcast(ExtVT, Amt);
29480     Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExtVT, Amt, 5, DAG);
29481     Amt = DAG.getBitcast(VT, Amt);
29482 
29483     if (Opc == ISD::SHL || Opc == ISD::SRL) {
29484       // r = VSELECT(r, shift(r, 4), a);
29485       SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
29486       R = SignBitSelect(VT, Amt, M, R);
29487 
29488       // a += a
29489       Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29490 
29491       // r = VSELECT(r, shift(r, 2), a);
29492       M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT));
29493       R = SignBitSelect(VT, Amt, M, R);
29494 
29495       // a += a
29496       Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29497 
29498       // return VSELECT(r, shift(r, 1), a);
29499       M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT));
29500       R = SignBitSelect(VT, Amt, M, R);
29501       return R;
29502     }
29503 
29504     if (Opc == ISD::SRA) {
29505       // For SRA we need to unpack each byte to the higher byte of a i16 vector
29506       // so we can correctly sign extend. We don't care what happens to the
29507       // lower byte.
29508       SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
29509       SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
29510       SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
29511       SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
29512       ALo = DAG.getBitcast(ExtVT, ALo);
29513       AHi = DAG.getBitcast(ExtVT, AHi);
29514       RLo = DAG.getBitcast(ExtVT, RLo);
29515       RHi = DAG.getBitcast(ExtVT, RHi);
29516 
29517       // r = VSELECT(r, shift(r, 4), a);
29518       SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG);
29519       SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG);
29520       RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29521       RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29522 
29523       // a += a
29524       ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
29525       AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
29526 
29527       // r = VSELECT(r, shift(r, 2), a);
29528       MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 2, DAG);
29529       MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 2, DAG);
29530       RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29531       RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29532 
29533       // a += a
29534       ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
29535       AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
29536 
29537       // r = VSELECT(r, shift(r, 1), a);
29538       MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 1, DAG);
29539       MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 1, DAG);
29540       RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29541       RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29542 
29543       // Logical shift the result back to the lower byte, leaving a zero upper
29544       // byte meaning that we can safely pack with PACKUSWB.
29545       RLo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RLo, 8, DAG);
29546       RHi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RHi, 8, DAG);
29547       return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
29548     }
29549   }
29550 
29551   if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) {
29552     MVT ExtVT = MVT::v8i32;
29553     SDValue Z = DAG.getConstant(0, dl, VT);
29554     SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
29555     SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
29556     SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
29557     SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
29558     ALo = DAG.getBitcast(ExtVT, ALo);
29559     AHi = DAG.getBitcast(ExtVT, AHi);
29560     RLo = DAG.getBitcast(ExtVT, RLo);
29561     RHi = DAG.getBitcast(ExtVT, RHi);
29562     SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo);
29563     SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi);
29564     Lo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Lo, 16, DAG);
29565     Hi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Hi, 16, DAG);
29566     return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
29567   }
29568 
29569   if (VT == MVT::v8i16) {
29570     // If we have a constant shift amount, the non-SSE41 path is best as
29571     // avoiding bitcasts make it easier to constant fold and reduce to PBLENDW.
29572     bool UseSSE41 = Subtarget.hasSSE41() &&
29573                     !ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
29574 
29575     auto SignBitSelect = [&](SDValue Sel, SDValue V0, SDValue V1) {
29576       // On SSE41 targets we can use PBLENDVB which selects bytes based just on
29577       // the sign bit.
29578       if (UseSSE41) {
29579         MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);
29580         V0 = DAG.getBitcast(ExtVT, V0);
29581         V1 = DAG.getBitcast(ExtVT, V1);
29582         Sel = DAG.getBitcast(ExtVT, Sel);
29583         return DAG.getBitcast(
29584             VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1));
29585       }
29586       // On pre-SSE41 targets we splat the sign bit - a negative value will
29587       // set all bits of the lanes to true and VSELECT uses that in
29588       // its OR(AND(V0,C),AND(V1,~C)) lowering.
29589       SDValue C =
29590           getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG);
29591       return DAG.getSelect(dl, VT, C, V0, V1);
29592     };
29593 
29594     // Turn 'a' into a mask suitable for VSELECT: a = a << 12;
29595     if (UseSSE41) {
29596       // On SSE41 targets we need to replicate the shift mask in both
29597       // bytes for PBLENDVB.
29598       Amt = DAG.getNode(
29599           ISD::OR, dl, VT,
29600           getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG),
29601           getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG));
29602     } else {
29603       Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG);
29604     }
29605 
29606     // r = VSELECT(r, shift(r, 8), a);
29607     SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
29608     R = SignBitSelect(Amt, M, R);
29609 
29610     // a += a
29611     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29612 
29613     // r = VSELECT(r, shift(r, 4), a);
29614     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG);
29615     R = SignBitSelect(Amt, M, R);
29616 
29617     // a += a
29618     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29619 
29620     // r = VSELECT(r, shift(r, 2), a);
29621     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG);
29622     R = SignBitSelect(Amt, M, R);
29623 
29624     // a += a
29625     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29626 
29627     // return VSELECT(r, shift(r, 1), a);
29628     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG);
29629     R = SignBitSelect(Amt, M, R);
29630     return R;
29631   }
29632 
29633   // Decompose 256-bit shifts into 128-bit shifts.
29634   if (VT.is256BitVector())
29635     return splitVectorIntBinary(Op, DAG);
29636 
29637   if (VT == MVT::v32i16 || VT == MVT::v64i8)
29638     return splitVectorIntBinary(Op, DAG);
29639 
29640   return SDValue();
29641 }
29642 
29643 static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
29644                                 SelectionDAG &DAG) {
29645   MVT VT = Op.getSimpleValueType();
29646   assert((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) &&
29647          "Unexpected funnel shift opcode!");
29648 
29649   SDLoc DL(Op);
29650   SDValue Op0 = Op.getOperand(0);
29651   SDValue Op1 = Op.getOperand(1);
29652   SDValue Amt = Op.getOperand(2);
29653   unsigned EltSizeInBits = VT.getScalarSizeInBits();
29654   bool IsFSHR = Op.getOpcode() == ISD::FSHR;
29655 
29656   if (VT.isVector()) {
29657     APInt APIntShiftAmt;
29658     bool IsCstSplat = X86::isConstantSplat(Amt, APIntShiftAmt);
29659 
29660     if (Subtarget.hasVBMI2() && EltSizeInBits > 8) {
29661       if (IsFSHR)
29662         std::swap(Op0, Op1);
29663 
29664       if (IsCstSplat) {
29665         uint64_t ShiftAmt = APIntShiftAmt.urem(EltSizeInBits);
29666         SDValue Imm = DAG.getTargetConstant(ShiftAmt, DL, MVT::i8);
29667         return getAVX512Node(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT,
29668                              {Op0, Op1, Imm}, DAG, Subtarget);
29669       }
29670       return getAVX512Node(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,
29671                            {Op0, Op1, Amt}, DAG, Subtarget);
29672     }
29673     assert((VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8 ||
29674             VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16 ||
29675             VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) &&
29676            "Unexpected funnel shift type!");
29677 
29678     // fshl(x,y,z) -> unpack(y,x) << (z & (bw-1))) >> bw.
29679     // fshr(x,y,z) -> unpack(y,x) >> (z & (bw-1))).
29680     if (IsCstSplat) {
29681       // TODO: Can't use generic expansion as UNDEF amt elements can be
29682       // converted to other values when folded to shift amounts, losing the
29683       // splat.
29684       uint64_t ShiftAmt = APIntShiftAmt.urem(EltSizeInBits);
29685       uint64_t ShXAmt = IsFSHR ? (EltSizeInBits - ShiftAmt) : ShiftAmt;
29686       uint64_t ShYAmt = IsFSHR ? ShiftAmt : (EltSizeInBits - ShiftAmt);
29687       SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, Op0,
29688                                 DAG.getShiftAmountConstant(ShXAmt, VT, DL));
29689       SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Op1,
29690                                 DAG.getShiftAmountConstant(ShYAmt, VT, DL));
29691       return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
29692     }
29693 
29694     SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
29695     SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
29696     bool IsCst = ISD::isBuildVectorOfConstantSDNodes(AmtMod.getNode());
29697 
29698     // Constant vXi16 funnel shifts can be efficiently handled by default.
29699     if (IsCst && EltSizeInBits == 16)
29700       return SDValue();
29701 
29702     unsigned ShiftOpc = IsFSHR ? ISD::SRL : ISD::SHL;
29703     unsigned NumElts = VT.getVectorNumElements();
29704     MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);
29705     MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);
29706 
29707     // Split 256-bit integers on XOP/pre-AVX2 targets.
29708     // Split 512-bit integers on non 512-bit BWI targets.
29709     if ((VT.is256BitVector() && ((Subtarget.hasXOP() && EltSizeInBits < 16) ||
29710                                  !Subtarget.hasAVX2())) ||
29711         (VT.is512BitVector() && !Subtarget.useBWIRegs() &&
29712          EltSizeInBits < 32)) {
29713       // Pre-mask the amount modulo using the wider vector.
29714       Op = DAG.getNode(Op.getOpcode(), DL, VT, Op0, Op1, AmtMod);
29715       return splitVectorOp(Op, DAG);
29716     }
29717 
29718     // Attempt to fold scalar shift as unpack(y,x) << zext(splat(z))
29719     if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, ShiftOpc)) {
29720       int ScalarAmtIdx = -1;
29721       if (SDValue ScalarAmt = DAG.getSplatSourceVector(AmtMod, ScalarAmtIdx)) {
29722         // Uniform vXi16 funnel shifts can be efficiently handled by default.
29723         if (EltSizeInBits == 16)
29724           return SDValue();
29725 
29726         SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
29727         SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
29728         Lo = getTargetVShiftNode(ShiftOpc, DL, ExtVT, Lo, ScalarAmt,
29729                                  ScalarAmtIdx, Subtarget, DAG);
29730         Hi = getTargetVShiftNode(ShiftOpc, DL, ExtVT, Hi, ScalarAmt,
29731                                  ScalarAmtIdx, Subtarget, DAG);
29732         return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR);
29733       }
29734     }
29735 
29736     MVT WideSVT = MVT::getIntegerVT(
29737         std::min<unsigned>(EltSizeInBits * 2, Subtarget.hasBWI() ? 16 : 32));
29738     MVT WideVT = MVT::getVectorVT(WideSVT, NumElts);
29739 
29740     // If per-element shifts are legal, fallback to generic expansion.
29741     if (supportedVectorVarShift(VT, Subtarget, ShiftOpc) || Subtarget.hasXOP())
29742       return SDValue();
29743 
29744     // Attempt to fold as:
29745     // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw.
29746     // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))).
29747     if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) &&
29748         supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) {
29749       Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Op0);
29750       Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op1);
29751       AmtMod = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
29752       Op0 = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, Op0,
29753                                        EltSizeInBits, DAG);
29754       SDValue Res = DAG.getNode(ISD::OR, DL, WideVT, Op0, Op1);
29755       Res = DAG.getNode(ShiftOpc, DL, WideVT, Res, AmtMod);
29756       if (!IsFSHR)
29757         Res = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, Res,
29758                                          EltSizeInBits, DAG);
29759       return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
29760     }
29761 
29762     // Attempt to fold per-element (ExtVT) shift as unpack(y,x) << zext(z)
29763     if (((IsCst || !Subtarget.hasAVX512()) && !IsFSHR && EltSizeInBits <= 16) ||
29764         supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) {
29765       SDValue Z = DAG.getConstant(0, DL, VT);
29766       SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
29767       SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
29768       SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
29769       SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
29770       SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
29771       SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
29772       return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR);
29773     }
29774 
29775     // Fallback to generic expansion.
29776     return SDValue();
29777   }
29778   assert(
29779       (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
29780       "Unexpected funnel shift type!");
29781 
29782   // Expand slow SHLD/SHRD cases if we are not optimizing for size.
29783   bool OptForSize = DAG.shouldOptForSize();
29784   bool ExpandFunnel = !OptForSize && Subtarget.isSHLDSlow();
29785 
29786   // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw.
29787   // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))).
29788   if ((VT == MVT::i8 || (ExpandFunnel && VT == MVT::i16)) &&
29789       !isa<ConstantSDNode>(Amt)) {
29790     SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
29791     SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
29792     Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32);
29793     Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32);
29794     Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask);
29795     SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
29796     Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1);
29797     if (IsFSHR) {
29798       Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt);
29799     } else {
29800       Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt);
29801       Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift);
29802     }
29803     return DAG.getZExtOrTrunc(Res, DL, VT);
29804   }
29805 
29806   if (VT == MVT::i8 || ExpandFunnel)
29807     return SDValue();
29808 
29809   // i16 needs to modulo the shift amount, but i32/i64 have implicit modulo.
29810   if (VT == MVT::i16) {
29811     Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt,
29812                       DAG.getConstant(15, DL, Amt.getValueType()));
29813     unsigned FSHOp = (IsFSHR ? X86ISD::FSHR : X86ISD::FSHL);
29814     return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt);
29815   }
29816 
29817   return Op;
29818 }
29819 
29820 static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
29821                            SelectionDAG &DAG) {
29822   MVT VT = Op.getSimpleValueType();
29823   assert(VT.isVector() && "Custom lowering only for vector rotates!");
29824 
29825   SDLoc DL(Op);
29826   SDValue R = Op.getOperand(0);
29827   SDValue Amt = Op.getOperand(1);
29828   unsigned Opcode = Op.getOpcode();
29829   unsigned EltSizeInBits = VT.getScalarSizeInBits();
29830   int NumElts = VT.getVectorNumElements();
29831   bool IsROTL = Opcode == ISD::ROTL;
29832 
29833   // Check for constant splat rotation amount.
29834   APInt CstSplatValue;
29835   bool IsCstSplat = X86::isConstantSplat(Amt, CstSplatValue);
29836 
29837   // Check for splat rotate by zero.
29838   if (IsCstSplat && CstSplatValue.urem(EltSizeInBits) == 0)
29839     return R;
29840 
29841   // AVX512 implicitly uses modulo rotation amounts.
29842   if (Subtarget.hasAVX512() && 32 <= EltSizeInBits) {
29843     // Attempt to rotate by immediate.
29844     if (IsCstSplat) {
29845       unsigned RotOpc = IsROTL ? X86ISD::VROTLI : X86ISD::VROTRI;
29846       uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
29847       return DAG.getNode(RotOpc, DL, VT, R,
29848                          DAG.getTargetConstant(RotAmt, DL, MVT::i8));
29849     }
29850 
29851     // Else, fall-back on VPROLV/VPRORV.
29852     return Op;
29853   }
29854 
29855   // AVX512 VBMI2 vXi16 - lower to funnel shifts.
29856   if (Subtarget.hasVBMI2() && 16 == EltSizeInBits) {
29857     unsigned FunnelOpc = IsROTL ? ISD::FSHL : ISD::FSHR;
29858     return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
29859   }
29860 
29861   SDValue Z = DAG.getConstant(0, DL, VT);
29862 
29863   if (!IsROTL) {
29864     // If the ISD::ROTR amount is constant, we're always better converting to
29865     // ISD::ROTL.
29866     if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt}))
29867       return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt);
29868 
29869     // XOP targets always prefers ISD::ROTL.
29870     if (Subtarget.hasXOP())
29871       return DAG.getNode(ISD::ROTL, DL, VT, R,
29872                          DAG.getNode(ISD::SUB, DL, VT, Z, Amt));
29873   }
29874 
29875   // Split 256-bit integers on XOP/pre-AVX2 targets.
29876   if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2()))
29877     return splitVectorIntBinary(Op, DAG);
29878 
29879   // XOP has 128-bit vector variable + immediate rotates.
29880   // +ve/-ve Amt = rotate left/right - just need to handle ISD::ROTL.
29881   // XOP implicitly uses modulo rotation amounts.
29882   if (Subtarget.hasXOP()) {
29883     assert(IsROTL && "Only ROTL expected");
29884     assert(VT.is128BitVector() && "Only rotate 128-bit vectors!");
29885 
29886     // Attempt to rotate by immediate.
29887     if (IsCstSplat) {
29888       uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
29889       return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
29890                          DAG.getTargetConstant(RotAmt, DL, MVT::i8));
29891     }
29892 
29893     // Use general rotate by variable (per-element).
29894     return Op;
29895   }
29896 
29897   // Rotate by an uniform constant - expand back to shifts.
29898   // TODO: Can't use generic expansion as UNDEF amt elements can be converted
29899   // to other values when folded to shift amounts, losing the splat.
29900   if (IsCstSplat) {
29901     uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
29902     uint64_t ShlAmt = IsROTL ? RotAmt : (EltSizeInBits - RotAmt);
29903     uint64_t SrlAmt = IsROTL ? (EltSizeInBits - RotAmt) : RotAmt;
29904     SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, R,
29905                               DAG.getShiftAmountConstant(ShlAmt, VT, DL));
29906     SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, R,
29907                               DAG.getShiftAmountConstant(SrlAmt, VT, DL));
29908     return DAG.getNode(ISD::OR, DL, VT, Shl, Srl);
29909   }
29910 
29911   // Split 512-bit integers on non 512-bit BWI targets.
29912   if (VT.is512BitVector() && !Subtarget.useBWIRegs())
29913     return splitVectorIntBinary(Op, DAG);
29914 
29915   assert(
29916       (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
29917        ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) &&
29918         Subtarget.hasAVX2()) ||
29919        ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) &&
29920       "Only vXi32/vXi16/vXi8 vector rotates supported");
29921 
29922   MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);
29923   MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);
29924 
29925   SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
29926   SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
29927 
29928   // Attempt to fold as unpack(x,x) << zext(splat(y)):
29929   // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw.
29930   // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))).
29931   if (EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) {
29932     int BaseRotAmtIdx = -1;
29933     if (SDValue BaseRotAmt = DAG.getSplatSourceVector(AmtMod, BaseRotAmtIdx)) {
29934       if (EltSizeInBits == 16 && Subtarget.hasSSE41()) {
29935         unsigned FunnelOpc = IsROTL ? ISD::FSHL : ISD::FSHR;
29936         return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
29937       }
29938       unsigned ShiftX86Opc = IsROTL ? X86ISD::VSHLI : X86ISD::VSRLI;
29939       SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
29940       SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
29941       Lo = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Lo, BaseRotAmt,
29942                                BaseRotAmtIdx, Subtarget, DAG);
29943       Hi = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Hi, BaseRotAmt,
29944                                BaseRotAmtIdx, Subtarget, DAG);
29945       return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
29946     }
29947   }
29948 
29949   bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
29950   unsigned ShiftOpc = IsROTL ? ISD::SHL : ISD::SRL;
29951 
29952   // Attempt to fold as unpack(x,x) << zext(y):
29953   // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw.
29954   // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))).
29955   // Const vXi16/vXi32 are excluded in favor of MUL-based lowering.
29956   if (!(ConstantAmt && EltSizeInBits != 8) &&
29957       !supportedVectorVarShift(VT, Subtarget, ShiftOpc) &&
29958       (ConstantAmt || supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc))) {
29959     SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
29960     SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
29961     SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
29962     SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
29963     SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
29964     SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
29965     return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
29966   }
29967 
29968   // v16i8/v32i8/v64i8: Split rotation into rot4/rot2/rot1 stages and select by
29969   // the amount bit.
29970   // TODO: We're doing nothing here that we couldn't do for funnel shifts.
29971   if (EltSizeInBits == 8) {
29972     MVT WideVT =
29973         MVT::getVectorVT(Subtarget.hasBWI() ? MVT::i16 : MVT::i32, NumElts);
29974 
29975     // Attempt to fold as:
29976     // rotl(x,y) -> (((aext(x) << bw) | zext(x)) << (y & (bw-1))) >> bw.
29977     // rotr(x,y) -> (((aext(x) << bw) | zext(x)) >> (y & (bw-1))).
29978     if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) &&
29979         supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) {
29980       // If we're rotating by constant, just use default promotion.
29981       if (ConstantAmt)
29982         return SDValue();
29983       // See if we can perform this by widening to vXi16 or vXi32.
29984       R = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, R);
29985       R = DAG.getNode(
29986           ISD::OR, DL, WideVT, R,
29987           getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, R, 8, DAG));
29988       Amt = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
29989       R = DAG.getNode(ShiftOpc, DL, WideVT, R, Amt);
29990       if (IsROTL)
29991         R = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, R, 8, DAG);
29992       return DAG.getNode(ISD::TRUNCATE, DL, VT, R);
29993     }
29994 
29995     // We don't need ModuloAmt here as we just peek at individual bits.
29996     auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
29997       if (Subtarget.hasSSE41()) {
29998         // On SSE41 targets we can use PBLENDVB which selects bytes based just
29999         // on the sign bit.
30000         V0 = DAG.getBitcast(VT, V0);
30001         V1 = DAG.getBitcast(VT, V1);
30002         Sel = DAG.getBitcast(VT, Sel);
30003         return DAG.getBitcast(SelVT,
30004                               DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1));
30005       }
30006       // On pre-SSE41 targets we test for the sign bit by comparing to
30007       // zero - a negative value will set all bits of the lanes to true
30008       // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
30009       SDValue Z = DAG.getConstant(0, DL, SelVT);
30010       SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
30011       return DAG.getSelect(DL, SelVT, C, V0, V1);
30012     };
30013 
30014     // ISD::ROTR is currently only profitable on AVX512 targets with VPTERNLOG.
30015     if (!IsROTL && !useVPTERNLOG(Subtarget, VT)) {
30016       Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
30017       IsROTL = true;
30018     }
30019 
30020     unsigned ShiftLHS = IsROTL ? ISD::SHL : ISD::SRL;
30021     unsigned ShiftRHS = IsROTL ? ISD::SRL : ISD::SHL;
30022 
30023     // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
30024     // We can safely do this using i16 shifts as we're only interested in
30025     // the 3 lower bits of each byte.
30026     Amt = DAG.getBitcast(ExtVT, Amt);
30027     Amt = DAG.getNode(ISD::SHL, DL, ExtVT, Amt, DAG.getConstant(5, DL, ExtVT));
30028     Amt = DAG.getBitcast(VT, Amt);
30029 
30030     // r = VSELECT(r, rot(r, 4), a);
30031     SDValue M;
30032     M = DAG.getNode(
30033         ISD::OR, DL, VT,
30034         DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(4, DL, VT)),
30035         DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(4, DL, VT)));
30036     R = SignBitSelect(VT, Amt, M, R);
30037 
30038     // a += a
30039     Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
30040 
30041     // r = VSELECT(r, rot(r, 2), a);
30042     M = DAG.getNode(
30043         ISD::OR, DL, VT,
30044         DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(2, DL, VT)),
30045         DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(6, DL, VT)));
30046     R = SignBitSelect(VT, Amt, M, R);
30047 
30048     // a += a
30049     Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
30050 
30051     // return VSELECT(r, rot(r, 1), a);
30052     M = DAG.getNode(
30053         ISD::OR, DL, VT,
30054         DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(1, DL, VT)),
30055         DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(7, DL, VT)));
30056     return SignBitSelect(VT, Amt, M, R);
30057   }
30058 
30059   bool IsSplatAmt = DAG.isSplatValue(Amt);
30060   bool LegalVarShifts = supportedVectorVarShift(VT, Subtarget, ISD::SHL) &&
30061                         supportedVectorVarShift(VT, Subtarget, ISD::SRL);
30062 
30063   // Fallback for splats + all supported variable shifts.
30064   // Fallback for non-constants AVX2 vXi16 as well.
30065   if (IsSplatAmt || LegalVarShifts || (Subtarget.hasAVX2() && !ConstantAmt)) {
30066     Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30067     SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
30068     AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt);
30069     SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt);
30070     SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR);
30071     return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);
30072   }
30073 
30074   // Everything below assumes ISD::ROTL.
30075   if (!IsROTL) {
30076     Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
30077     IsROTL = true;
30078   }
30079 
30080   // ISD::ROT* uses modulo rotate amounts.
30081   Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30082 
30083   assert(IsROTL && "Only ROTL supported");
30084 
30085   // As with shifts, attempt to convert the rotation amount to a multiplication
30086   // factor, fallback to general expansion.
30087   SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
30088   if (!Scale)
30089     return SDValue();
30090 
30091   // v8i16/v16i16: perform unsigned multiply hi/lo and OR the results.
30092   if (EltSizeInBits == 16) {
30093     SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
30094     SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
30095     return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
30096   }
30097 
30098   // v4i32: make use of the PMULUDQ instruction to multiply 2 lanes of v4i32
30099   // to v2i64 results at a time. The upper 32-bits contain the wrapped bits
30100   // that can then be OR'd with the lower 32-bits.
30101   assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected");
30102   static const int OddMask[] = {1, -1, 3, -1};
30103   SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
30104   SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
30105 
30106   SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
30107                               DAG.getBitcast(MVT::v2i64, R),
30108                               DAG.getBitcast(MVT::v2i64, Scale));
30109   SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
30110                               DAG.getBitcast(MVT::v2i64, R13),
30111                               DAG.getBitcast(MVT::v2i64, Scale13));
30112   Res02 = DAG.getBitcast(VT, Res02);
30113   Res13 = DAG.getBitcast(VT, Res13);
30114 
30115   return DAG.getNode(ISD::OR, DL, VT,
30116                      DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}),
30117                      DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7}));
30118 }
30119 
30120 /// Returns true if the operand type is exactly twice the native width, and
30121 /// the corresponding cmpxchg8b or cmpxchg16b instruction is available.
30122 /// Used to know whether to use cmpxchg8/16b when expanding atomic operations
30123 /// (otherwise we leave them alone to become __sync_fetch_and_... calls).
30124 bool X86TargetLowering::needsCmpXchgNb(Type *MemType) const {
30125   unsigned OpWidth = MemType->getPrimitiveSizeInBits();
30126 
30127   if (OpWidth == 64)
30128     return Subtarget.canUseCMPXCHG8B() && !Subtarget.is64Bit();
30129   if (OpWidth == 128)
30130     return Subtarget.canUseCMPXCHG16B();
30131 
30132   return false;
30133 }
30134 
30135 TargetLoweringBase::AtomicExpansionKind
30136 X86TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
30137   Type *MemType = SI->getValueOperand()->getType();
30138 
30139   bool NoImplicitFloatOps =
30140       SI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
30141   if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
30142       !Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
30143       (Subtarget.hasSSE1() || Subtarget.hasX87()))
30144     return AtomicExpansionKind::None;
30145 
30146   return needsCmpXchgNb(MemType) ? AtomicExpansionKind::Expand
30147                                  : AtomicExpansionKind::None;
30148 }
30149 
30150 // Note: this turns large loads into lock cmpxchg8b/16b.
30151 // TODO: In 32-bit mode, use MOVLPS when SSE1 is available?
30152 TargetLowering::AtomicExpansionKind
30153 X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
30154   Type *MemType = LI->getType();
30155 
30156   // If this a 64 bit atomic load on a 32-bit target and SSE2 is enabled, we
30157   // can use movq to do the load. If we have X87 we can load into an 80-bit
30158   // X87 register and store it to a stack temporary.
30159   bool NoImplicitFloatOps =
30160       LI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
30161   if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
30162       !Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
30163       (Subtarget.hasSSE1() || Subtarget.hasX87()))
30164     return AtomicExpansionKind::None;
30165 
30166   return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
30167                                  : AtomicExpansionKind::None;
30168 }
30169 
30170 enum BitTestKind : unsigned {
30171   UndefBit,
30172   ConstantBit,
30173   NotConstantBit,
30174   ShiftBit,
30175   NotShiftBit
30176 };
30177 
30178 static std::pair<Value *, BitTestKind> FindSingleBitChange(Value *V) {
30179   using namespace llvm::PatternMatch;
30180   BitTestKind BTK = UndefBit;
30181   auto *C = dyn_cast<ConstantInt>(V);
30182   if (C) {
30183     // Check if V is a power of 2 or NOT power of 2.
30184     if (isPowerOf2_64(C->getZExtValue()))
30185       BTK = ConstantBit;
30186     else if (isPowerOf2_64((~C->getValue()).getZExtValue()))
30187       BTK = NotConstantBit;
30188     return {V, BTK};
30189   }
30190 
30191   // Check if V is some power of 2 pattern known to be non-zero
30192   auto *I = dyn_cast<Instruction>(V);
30193   if (I) {
30194     bool Not = false;
30195     // Check if we have a NOT
30196     Value *PeekI;
30197     if (match(I, m_c_Xor(m_Value(PeekI), m_AllOnes())) ||
30198         match(I, m_Sub(m_AllOnes(), m_Value(PeekI)))) {
30199       Not = true;
30200       I = dyn_cast<Instruction>(PeekI);
30201 
30202       // If I is constant, it will fold and we can evaluate later. If its an
30203       // argument or something of that nature, we can't analyze.
30204       if (I == nullptr)
30205         return {nullptr, UndefBit};
30206     }
30207     // We can only use 1 << X without more sophisticated analysis. C << X where
30208     // C is a power of 2 but not 1 can result in zero which cannot be translated
30209     // to bittest. Likewise any C >> X (either arith or logical) can be zero.
30210     if (I->getOpcode() == Instruction::Shl) {
30211       // Todo(1): The cmpxchg case is pretty costly so matching `BLSI(X)`, `X &
30212       // -X` and some other provable power of 2 patterns that we can use CTZ on
30213       // may be profitable.
30214       // Todo(2): It may be possible in some cases to prove that Shl(C, X) is
30215       // non-zero even where C != 1. Likewise LShr(C, X) and AShr(C, X) may also
30216       // be provably a non-zero power of 2.
30217       // Todo(3): ROTL and ROTR patterns on a power of 2 C should also be
30218       // transformable to bittest.
30219       auto *ShiftVal = dyn_cast<ConstantInt>(I->getOperand(0));
30220       if (!ShiftVal)
30221         return {nullptr, UndefBit};
30222       if (ShiftVal->equalsInt(1))
30223         BTK = Not ? NotShiftBit : ShiftBit;
30224 
30225       if (BTK == UndefBit)
30226         return {nullptr, UndefBit};
30227 
30228       Value *BitV = I->getOperand(1);
30229 
30230       Value *AndOp;
30231       const APInt *AndC;
30232       if (match(BitV, m_c_And(m_Value(AndOp), m_APInt(AndC)))) {
30233         // Read past a shiftmask instruction to find count
30234         if (*AndC == (I->getType()->getPrimitiveSizeInBits() - 1))
30235           BitV = AndOp;
30236       }
30237       return {BitV, BTK};
30238     }
30239   }
30240   return {nullptr, UndefBit};
30241 }
30242 
30243 TargetLowering::AtomicExpansionKind
30244 X86TargetLowering::shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const {
30245   using namespace llvm::PatternMatch;
30246   // If the atomicrmw's result isn't actually used, we can just add a "lock"
30247   // prefix to a normal instruction for these operations.
30248   if (AI->use_empty())
30249     return AtomicExpansionKind::None;
30250 
30251   if (AI->getOperation() == AtomicRMWInst::Xor) {
30252     // A ^ SignBit -> A + SignBit. This allows us to use `xadd` which is
30253     // preferable to both `cmpxchg` and `btc`.
30254     if (match(AI->getOperand(1), m_SignMask()))
30255       return AtomicExpansionKind::None;
30256   }
30257 
30258   // If the atomicrmw's result is used by a single bit AND, we may use
30259   // bts/btr/btc instruction for these operations.
30260   // Note: InstCombinePass can cause a de-optimization here. It replaces the
30261   // SETCC(And(AtomicRMW(P, power_of_2), power_of_2)) with LShr and Xor
30262   // (depending on CC). This pattern can only use bts/btr/btc but we don't
30263   // detect it.
30264   Instruction *I = AI->user_back();
30265   auto BitChange = FindSingleBitChange(AI->getValOperand());
30266   if (BitChange.second == UndefBit || !AI->hasOneUse() ||
30267       I->getOpcode() != Instruction::And ||
30268       AI->getType()->getPrimitiveSizeInBits() == 8 ||
30269       AI->getParent() != I->getParent())
30270     return AtomicExpansionKind::CmpXChg;
30271 
30272   unsigned OtherIdx = I->getOperand(0) == AI ? 1 : 0;
30273 
30274   // This is a redundant AND, it should get cleaned up elsewhere.
30275   if (AI == I->getOperand(OtherIdx))
30276     return AtomicExpansionKind::CmpXChg;
30277 
30278   // The following instruction must be a AND single bit.
30279   if (BitChange.second == ConstantBit || BitChange.second == NotConstantBit) {
30280     auto *C1 = cast<ConstantInt>(AI->getValOperand());
30281     auto *C2 = dyn_cast<ConstantInt>(I->getOperand(OtherIdx));
30282     if (!C2 || !isPowerOf2_64(C2->getZExtValue())) {
30283       return AtomicExpansionKind::CmpXChg;
30284     }
30285     if (AI->getOperation() == AtomicRMWInst::And) {
30286       return ~C1->getValue() == C2->getValue()
30287                  ? AtomicExpansionKind::BitTestIntrinsic
30288                  : AtomicExpansionKind::CmpXChg;
30289     }
30290     return C1 == C2 ? AtomicExpansionKind::BitTestIntrinsic
30291                     : AtomicExpansionKind::CmpXChg;
30292   }
30293 
30294   assert(BitChange.second == ShiftBit || BitChange.second == NotShiftBit);
30295 
30296   auto BitTested = FindSingleBitChange(I->getOperand(OtherIdx));
30297   if (BitTested.second != ShiftBit && BitTested.second != NotShiftBit)
30298     return AtomicExpansionKind::CmpXChg;
30299 
30300   assert(BitChange.first != nullptr && BitTested.first != nullptr);
30301 
30302   // If shift amounts are not the same we can't use BitTestIntrinsic.
30303   if (BitChange.first != BitTested.first)
30304     return AtomicExpansionKind::CmpXChg;
30305 
30306   // If atomic AND need to be masking all be one bit and testing the one bit
30307   // unset in the mask.
30308   if (AI->getOperation() == AtomicRMWInst::And)
30309     return (BitChange.second == NotShiftBit && BitTested.second == ShiftBit)
30310                ? AtomicExpansionKind::BitTestIntrinsic
30311                : AtomicExpansionKind::CmpXChg;
30312 
30313   // If atomic XOR/OR need to be setting and testing the same bit.
30314   return (BitChange.second == ShiftBit && BitTested.second == ShiftBit)
30315              ? AtomicExpansionKind::BitTestIntrinsic
30316              : AtomicExpansionKind::CmpXChg;
30317 }
30318 
30319 void X86TargetLowering::emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const {
30320   IRBuilder<> Builder(AI);
30321   Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30322   Intrinsic::ID IID_C = Intrinsic::not_intrinsic;
30323   Intrinsic::ID IID_I = Intrinsic::not_intrinsic;
30324   switch (AI->getOperation()) {
30325   default:
30326     llvm_unreachable("Unknown atomic operation");
30327   case AtomicRMWInst::Or:
30328     IID_C = Intrinsic::x86_atomic_bts;
30329     IID_I = Intrinsic::x86_atomic_bts_rm;
30330     break;
30331   case AtomicRMWInst::Xor:
30332     IID_C = Intrinsic::x86_atomic_btc;
30333     IID_I = Intrinsic::x86_atomic_btc_rm;
30334     break;
30335   case AtomicRMWInst::And:
30336     IID_C = Intrinsic::x86_atomic_btr;
30337     IID_I = Intrinsic::x86_atomic_btr_rm;
30338     break;
30339   }
30340   Instruction *I = AI->user_back();
30341   LLVMContext &Ctx = AI->getContext();
30342   Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(),
30343                                           PointerType::getUnqual(Ctx));
30344   Function *BitTest = nullptr;
30345   Value *Result = nullptr;
30346   auto BitTested = FindSingleBitChange(AI->getValOperand());
30347   assert(BitTested.first != nullptr);
30348 
30349   if (BitTested.second == ConstantBit || BitTested.second == NotConstantBit) {
30350     auto *C = cast<ConstantInt>(I->getOperand(I->getOperand(0) == AI ? 1 : 0));
30351 
30352     BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_C, AI->getType());
30353 
30354     unsigned Imm = llvm::countr_zero(C->getZExtValue());
30355     Result = Builder.CreateCall(BitTest, {Addr, Builder.getInt8(Imm)});
30356   } else {
30357     BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_I, AI->getType());
30358 
30359     assert(BitTested.second == ShiftBit || BitTested.second == NotShiftBit);
30360 
30361     Value *SI = BitTested.first;
30362     assert(SI != nullptr);
30363 
30364     // BT{S|R|C} on memory operand don't modulo bit position so we need to
30365     // mask it.
30366     unsigned ShiftBits = SI->getType()->getPrimitiveSizeInBits();
30367     Value *BitPos =
30368         Builder.CreateAnd(SI, Builder.getIntN(ShiftBits, ShiftBits - 1));
30369     // Todo(1): In many cases it may be provable that SI is less than
30370     // ShiftBits in which case this mask is unnecessary
30371     // Todo(2): In the fairly idiomatic case of P[X / sizeof_bits(X)] OP 1
30372     // << (X % sizeof_bits(X)) we can drop the shift mask and AGEN in
30373     // favor of just a raw BT{S|R|C}.
30374 
30375     Result = Builder.CreateCall(BitTest, {Addr, BitPos});
30376     Result = Builder.CreateZExtOrTrunc(Result, AI->getType());
30377 
30378     // If the result is only used for zero/non-zero status then we don't need to
30379     // shift value back. Otherwise do so.
30380     for (auto It = I->user_begin(); It != I->user_end(); ++It) {
30381       if (auto *ICmp = dyn_cast<ICmpInst>(*It)) {
30382         if (ICmp->isEquality()) {
30383           auto *C0 = dyn_cast<ConstantInt>(ICmp->getOperand(0));
30384           auto *C1 = dyn_cast<ConstantInt>(ICmp->getOperand(1));
30385           if (C0 || C1) {
30386             assert(C0 == nullptr || C1 == nullptr);
30387             if ((C0 ? C0 : C1)->isZero())
30388               continue;
30389           }
30390         }
30391       }
30392       Result = Builder.CreateShl(Result, BitPos);
30393       break;
30394     }
30395   }
30396 
30397   I->replaceAllUsesWith(Result);
30398   I->eraseFromParent();
30399   AI->eraseFromParent();
30400 }
30401 
30402 static bool shouldExpandCmpArithRMWInIR(AtomicRMWInst *AI) {
30403   using namespace llvm::PatternMatch;
30404   if (!AI->hasOneUse())
30405     return false;
30406 
30407   Value *Op = AI->getOperand(1);
30408   ICmpInst::Predicate Pred;
30409   Instruction *I = AI->user_back();
30410   AtomicRMWInst::BinOp Opc = AI->getOperation();
30411   if (Opc == AtomicRMWInst::Add) {
30412     if (match(I, m_c_ICmp(Pred, m_Sub(m_ZeroInt(), m_Specific(Op)), m_Value())))
30413       return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30414     if (match(I, m_OneUse(m_c_Add(m_Specific(Op), m_Value())))) {
30415       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30416         return Pred == CmpInst::ICMP_SLT;
30417       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes())))
30418         return Pred == CmpInst::ICMP_SGT;
30419     }
30420     return false;
30421   }
30422   if (Opc == AtomicRMWInst::Sub) {
30423     if (match(I, m_c_ICmp(Pred, m_Specific(Op), m_Value())))
30424       return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30425     if (match(I, m_OneUse(m_Sub(m_Value(), m_Specific(Op))))) {
30426       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30427         return Pred == CmpInst::ICMP_SLT;
30428       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes())))
30429         return Pred == CmpInst::ICMP_SGT;
30430     }
30431     return false;
30432   }
30433   if ((Opc == AtomicRMWInst::Or &&
30434        match(I, m_OneUse(m_c_Or(m_Specific(Op), m_Value())))) ||
30435       (Opc == AtomicRMWInst::And &&
30436        match(I, m_OneUse(m_c_And(m_Specific(Op), m_Value()))))) {
30437     if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30438       return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE ||
30439              Pred == CmpInst::ICMP_SLT;
30440     if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes())))
30441       return Pred == CmpInst::ICMP_SGT;
30442     return false;
30443   }
30444   if (Opc == AtomicRMWInst::Xor) {
30445     if (match(I, m_c_ICmp(Pred, m_Specific(Op), m_Value())))
30446       return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30447     if (match(I, m_OneUse(m_c_Xor(m_Specific(Op), m_Value())))) {
30448       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30449         return Pred == CmpInst::ICMP_SLT;
30450       if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_AllOnes())))
30451         return Pred == CmpInst::ICMP_SGT;
30452     }
30453     return false;
30454   }
30455 
30456   return false;
30457 }
30458 
30459 void X86TargetLowering::emitCmpArithAtomicRMWIntrinsic(
30460     AtomicRMWInst *AI) const {
30461   IRBuilder<> Builder(AI);
30462   Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30463   Instruction *TempI = nullptr;
30464   LLVMContext &Ctx = AI->getContext();
30465   ICmpInst *ICI = dyn_cast<ICmpInst>(AI->user_back());
30466   if (!ICI) {
30467     TempI = AI->user_back();
30468     assert(TempI->hasOneUse() && "Must have one use");
30469     ICI = cast<ICmpInst>(TempI->user_back());
30470   }
30471   X86::CondCode CC = X86::COND_INVALID;
30472   ICmpInst::Predicate Pred = ICI->getPredicate();
30473   switch (Pred) {
30474   default:
30475     llvm_unreachable("Not supported Pred");
30476   case CmpInst::ICMP_EQ:
30477     CC = X86::COND_E;
30478     break;
30479   case CmpInst::ICMP_NE:
30480     CC = X86::COND_NE;
30481     break;
30482   case CmpInst::ICMP_SLT:
30483     CC = X86::COND_S;
30484     break;
30485   case CmpInst::ICMP_SGT:
30486     CC = X86::COND_NS;
30487     break;
30488   }
30489   Intrinsic::ID IID = Intrinsic::not_intrinsic;
30490   switch (AI->getOperation()) {
30491   default:
30492     llvm_unreachable("Unknown atomic operation");
30493   case AtomicRMWInst::Add:
30494     IID = Intrinsic::x86_atomic_add_cc;
30495     break;
30496   case AtomicRMWInst::Sub:
30497     IID = Intrinsic::x86_atomic_sub_cc;
30498     break;
30499   case AtomicRMWInst::Or:
30500     IID = Intrinsic::x86_atomic_or_cc;
30501     break;
30502   case AtomicRMWInst::And:
30503     IID = Intrinsic::x86_atomic_and_cc;
30504     break;
30505   case AtomicRMWInst::Xor:
30506     IID = Intrinsic::x86_atomic_xor_cc;
30507     break;
30508   }
30509   Function *CmpArith =
30510       Intrinsic::getDeclaration(AI->getModule(), IID, AI->getType());
30511   Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(),
30512                                           PointerType::getUnqual(Ctx));
30513   Value *Call = Builder.CreateCall(
30514       CmpArith, {Addr, AI->getValOperand(), Builder.getInt32((unsigned)CC)});
30515   Value *Result = Builder.CreateTrunc(Call, Type::getInt1Ty(Ctx));
30516   ICI->replaceAllUsesWith(Result);
30517   ICI->eraseFromParent();
30518   if (TempI)
30519     TempI->eraseFromParent();
30520   AI->eraseFromParent();
30521 }
30522 
30523 TargetLowering::AtomicExpansionKind
30524 X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
30525   unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
30526   Type *MemType = AI->getType();
30527 
30528   // If the operand is too big, we must see if cmpxchg8/16b is available
30529   // and default to library calls otherwise.
30530   if (MemType->getPrimitiveSizeInBits() > NativeWidth) {
30531     return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
30532                                    : AtomicExpansionKind::None;
30533   }
30534 
30535   AtomicRMWInst::BinOp Op = AI->getOperation();
30536   switch (Op) {
30537   case AtomicRMWInst::Xchg:
30538     return AtomicExpansionKind::None;
30539   case AtomicRMWInst::Add:
30540   case AtomicRMWInst::Sub:
30541     if (shouldExpandCmpArithRMWInIR(AI))
30542       return AtomicExpansionKind::CmpArithIntrinsic;
30543     // It's better to use xadd, xsub or xchg for these in other cases.
30544     return AtomicExpansionKind::None;
30545   case AtomicRMWInst::Or:
30546   case AtomicRMWInst::And:
30547   case AtomicRMWInst::Xor:
30548     if (shouldExpandCmpArithRMWInIR(AI))
30549       return AtomicExpansionKind::CmpArithIntrinsic;
30550     return shouldExpandLogicAtomicRMWInIR(AI);
30551   case AtomicRMWInst::Nand:
30552   case AtomicRMWInst::Max:
30553   case AtomicRMWInst::Min:
30554   case AtomicRMWInst::UMax:
30555   case AtomicRMWInst::UMin:
30556   case AtomicRMWInst::FAdd:
30557   case AtomicRMWInst::FSub:
30558   case AtomicRMWInst::FMax:
30559   case AtomicRMWInst::FMin:
30560   case AtomicRMWInst::UIncWrap:
30561   case AtomicRMWInst::UDecWrap:
30562   default:
30563     // These always require a non-trivial set of data operations on x86. We must
30564     // use a cmpxchg loop.
30565     return AtomicExpansionKind::CmpXChg;
30566   }
30567 }
30568 
30569 LoadInst *
30570 X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
30571   unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
30572   Type *MemType = AI->getType();
30573   // Accesses larger than the native width are turned into cmpxchg/libcalls, so
30574   // there is no benefit in turning such RMWs into loads, and it is actually
30575   // harmful as it introduces a mfence.
30576   if (MemType->getPrimitiveSizeInBits() > NativeWidth)
30577     return nullptr;
30578 
30579   // If this is a canonical idempotent atomicrmw w/no uses, we have a better
30580   // lowering available in lowerAtomicArith.
30581   // TODO: push more cases through this path.
30582   if (auto *C = dyn_cast<ConstantInt>(AI->getValOperand()))
30583     if (AI->getOperation() == AtomicRMWInst::Or && C->isZero() &&
30584         AI->use_empty())
30585       return nullptr;
30586 
30587   IRBuilder<> Builder(AI);
30588   Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30589   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
30590   auto SSID = AI->getSyncScopeID();
30591   // We must restrict the ordering to avoid generating loads with Release or
30592   // ReleaseAcquire orderings.
30593   auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
30594 
30595   // Before the load we need a fence. Here is an example lifted from
30596   // http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
30597   // is required:
30598   // Thread 0:
30599   //   x.store(1, relaxed);
30600   //   r1 = y.fetch_add(0, release);
30601   // Thread 1:
30602   //   y.fetch_add(42, acquire);
30603   //   r2 = x.load(relaxed);
30604   // r1 = r2 = 0 is impossible, but becomes possible if the idempotent rmw is
30605   // lowered to just a load without a fence. A mfence flushes the store buffer,
30606   // making the optimization clearly correct.
30607   // FIXME: it is required if isReleaseOrStronger(Order) but it is not clear
30608   // otherwise, we might be able to be more aggressive on relaxed idempotent
30609   // rmw. In practice, they do not look useful, so we don't try to be
30610   // especially clever.
30611   if (SSID == SyncScope::SingleThread)
30612     // FIXME: we could just insert an ISD::MEMBARRIER here, except we are at
30613     // the IR level, so we must wrap it in an intrinsic.
30614     return nullptr;
30615 
30616   if (!Subtarget.hasMFence())
30617     // FIXME: it might make sense to use a locked operation here but on a
30618     // different cache-line to prevent cache-line bouncing. In practice it
30619     // is probably a small win, and x86 processors without mfence are rare
30620     // enough that we do not bother.
30621     return nullptr;
30622 
30623   Function *MFence =
30624       llvm::Intrinsic::getDeclaration(M, Intrinsic::x86_sse2_mfence);
30625   Builder.CreateCall(MFence, {});
30626 
30627   // Finally we can emit the atomic load.
30628   LoadInst *Loaded = Builder.CreateAlignedLoad(
30629       AI->getType(), AI->getPointerOperand(), AI->getAlign());
30630   Loaded->setAtomic(Order, SSID);
30631   AI->replaceAllUsesWith(Loaded);
30632   AI->eraseFromParent();
30633   return Loaded;
30634 }
30635 
30636 /// Emit a locked operation on a stack location which does not change any
30637 /// memory location, but does involve a lock prefix.  Location is chosen to be
30638 /// a) very likely accessed only by a single thread to minimize cache traffic,
30639 /// and b) definitely dereferenceable.  Returns the new Chain result.
30640 static SDValue emitLockedStackOp(SelectionDAG &DAG,
30641                                  const X86Subtarget &Subtarget, SDValue Chain,
30642                                  const SDLoc &DL) {
30643   // Implementation notes:
30644   // 1) LOCK prefix creates a full read/write reordering barrier for memory
30645   // operations issued by the current processor.  As such, the location
30646   // referenced is not relevant for the ordering properties of the instruction.
30647   // See: Intel® 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
30648   // 8.2.3.9  Loads and Stores Are Not Reordered with Locked Instructions
30649   // 2) Using an immediate operand appears to be the best encoding choice
30650   // here since it doesn't require an extra register.
30651   // 3) OR appears to be very slightly faster than ADD. (Though, the difference
30652   // is small enough it might just be measurement noise.)
30653   // 4) When choosing offsets, there are several contributing factors:
30654   //   a) If there's no redzone, we default to TOS.  (We could allocate a cache
30655   //      line aligned stack object to improve this case.)
30656   //   b) To minimize our chances of introducing a false dependence, we prefer
30657   //      to offset the stack usage from TOS slightly.
30658   //   c) To minimize concerns about cross thread stack usage - in particular,
30659   //      the idiomatic MyThreadPool.run([&StackVars]() {...}) pattern which
30660   //      captures state in the TOS frame and accesses it from many threads -
30661   //      we want to use an offset such that the offset is in a distinct cache
30662   //      line from the TOS frame.
30663   //
30664   // For a general discussion of the tradeoffs and benchmark results, see:
30665   // https://shipilev.net/blog/2014/on-the-fence-with-dependencies/
30666 
30667   auto &MF = DAG.getMachineFunction();
30668   auto &TFL = *Subtarget.getFrameLowering();
30669   const unsigned SPOffset = TFL.has128ByteRedZone(MF) ? -64 : 0;
30670 
30671   if (Subtarget.is64Bit()) {
30672     SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
30673     SDValue Ops[] = {
30674       DAG.getRegister(X86::RSP, MVT::i64),                  // Base
30675       DAG.getTargetConstant(1, DL, MVT::i8),                // Scale
30676       DAG.getRegister(0, MVT::i64),                         // Index
30677       DAG.getTargetConstant(SPOffset, DL, MVT::i32),        // Disp
30678       DAG.getRegister(0, MVT::i16),                         // Segment.
30679       Zero,
30680       Chain};
30681     SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
30682                                      MVT::Other, Ops);
30683     return SDValue(Res, 1);
30684   }
30685 
30686   SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
30687   SDValue Ops[] = {
30688     DAG.getRegister(X86::ESP, MVT::i32),            // Base
30689     DAG.getTargetConstant(1, DL, MVT::i8),          // Scale
30690     DAG.getRegister(0, MVT::i32),                   // Index
30691     DAG.getTargetConstant(SPOffset, DL, MVT::i32),  // Disp
30692     DAG.getRegister(0, MVT::i16),                   // Segment.
30693     Zero,
30694     Chain
30695   };
30696   SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
30697                                    MVT::Other, Ops);
30698   return SDValue(Res, 1);
30699 }
30700 
30701 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
30702                                  SelectionDAG &DAG) {
30703   SDLoc dl(Op);
30704   AtomicOrdering FenceOrdering =
30705       static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
30706   SyncScope::ID FenceSSID =
30707       static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
30708 
30709   // The only fence that needs an instruction is a sequentially-consistent
30710   // cross-thread fence.
30711   if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
30712       FenceSSID == SyncScope::System) {
30713     if (Subtarget.hasMFence())
30714       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
30715 
30716     SDValue Chain = Op.getOperand(0);
30717     return emitLockedStackOp(DAG, Subtarget, Chain, dl);
30718   }
30719 
30720   // MEMBARRIER is a compiler barrier; it codegens to a no-op.
30721   return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
30722 }
30723 
30724 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
30725                              SelectionDAG &DAG) {
30726   MVT T = Op.getSimpleValueType();
30727   SDLoc DL(Op);
30728   unsigned Reg = 0;
30729   unsigned size = 0;
30730   switch(T.SimpleTy) {
30731   default: llvm_unreachable("Invalid value type!");
30732   case MVT::i8:  Reg = X86::AL;  size = 1; break;
30733   case MVT::i16: Reg = X86::AX;  size = 2; break;
30734   case MVT::i32: Reg = X86::EAX; size = 4; break;
30735   case MVT::i64:
30736     assert(Subtarget.is64Bit() && "Node not type legal!");
30737     Reg = X86::RAX; size = 8;
30738     break;
30739   }
30740   SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
30741                                   Op.getOperand(2), SDValue());
30742   SDValue Ops[] = { cpIn.getValue(0),
30743                     Op.getOperand(1),
30744                     Op.getOperand(3),
30745                     DAG.getTargetConstant(size, DL, MVT::i8),
30746                     cpIn.getValue(1) };
30747   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
30748   MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
30749   SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
30750                                            Ops, T, MMO);
30751 
30752   SDValue cpOut =
30753     DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
30754   SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
30755                                       MVT::i32, cpOut.getValue(2));
30756   SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
30757 
30758   return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
30759                      cpOut, Success, EFLAGS.getValue(1));
30760 }
30761 
30762 // Create MOVMSKB, taking into account whether we need to split for AVX1.
30763 static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
30764                            const X86Subtarget &Subtarget) {
30765   MVT InVT = V.getSimpleValueType();
30766 
30767   if (InVT == MVT::v64i8) {
30768     SDValue Lo, Hi;
30769     std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
30770     Lo = getPMOVMSKB(DL, Lo, DAG, Subtarget);
30771     Hi = getPMOVMSKB(DL, Hi, DAG, Subtarget);
30772     Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo);
30773     Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi);
30774     Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi,
30775                      DAG.getConstant(32, DL, MVT::i8));
30776     return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi);
30777   }
30778   if (InVT == MVT::v32i8 && !Subtarget.hasInt256()) {
30779     SDValue Lo, Hi;
30780     std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
30781     Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);
30782     Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);
30783     Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi,
30784                      DAG.getConstant(16, DL, MVT::i8));
30785     return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi);
30786   }
30787 
30788   return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
30789 }
30790 
30791 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
30792                             SelectionDAG &DAG) {
30793   SDValue Src = Op.getOperand(0);
30794   MVT SrcVT = Src.getSimpleValueType();
30795   MVT DstVT = Op.getSimpleValueType();
30796 
30797   // Legalize (v64i1 (bitcast i64 (X))) by splitting the i64, bitcasting each
30798   // half to v32i1 and concatenating the result.
30799   if (SrcVT == MVT::i64 && DstVT == MVT::v64i1) {
30800     assert(!Subtarget.is64Bit() && "Expected 32-bit mode");
30801     assert(Subtarget.hasBWI() && "Expected BWI target");
30802     SDLoc dl(Op);
30803     SDValue Lo, Hi;
30804     std::tie(Lo, Hi) = DAG.SplitScalar(Src, dl, MVT::i32, MVT::i32);
30805     Lo = DAG.getBitcast(MVT::v32i1, Lo);
30806     Hi = DAG.getBitcast(MVT::v32i1, Hi);
30807     return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
30808   }
30809 
30810   // Use MOVMSK for vector to scalar conversion to prevent scalarization.
30811   if ((SrcVT == MVT::v16i1 || SrcVT == MVT::v32i1) && DstVT.isScalarInteger()) {
30812     assert(!Subtarget.hasAVX512() && "Should use K-registers with AVX512");
30813     MVT SExtVT = SrcVT == MVT::v16i1 ? MVT::v16i8 : MVT::v32i8;
30814     SDLoc DL(Op);
30815     SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
30816     V = getPMOVMSKB(DL, V, DAG, Subtarget);
30817     return DAG.getZExtOrTrunc(V, DL, DstVT);
30818   }
30819 
30820   assert((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 ||
30821           SrcVT == MVT::i64) && "Unexpected VT!");
30822 
30823   assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
30824   if (!(DstVT == MVT::f64 && SrcVT == MVT::i64) &&
30825       !(DstVT == MVT::x86mmx && SrcVT.isVector()))
30826     // This conversion needs to be expanded.
30827     return SDValue();
30828 
30829   SDLoc dl(Op);
30830   if (SrcVT.isVector()) {
30831     // Widen the vector in input in the case of MVT::v2i32.
30832     // Example: from MVT::v2i32 to MVT::v4i32.
30833     MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(),
30834                                  SrcVT.getVectorNumElements() * 2);
30835     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src,
30836                       DAG.getUNDEF(SrcVT));
30837   } else {
30838     assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() &&
30839            "Unexpected source type in LowerBITCAST");
30840     Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
30841   }
30842 
30843   MVT V2X64VT = DstVT == MVT::f64 ? MVT::v2f64 : MVT::v2i64;
30844   Src = DAG.getNode(ISD::BITCAST, dl, V2X64VT, Src);
30845 
30846   if (DstVT == MVT::x86mmx)
30847     return DAG.getNode(X86ISD::MOVDQ2Q, dl, DstVT, Src);
30848 
30849   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, DstVT, Src,
30850                      DAG.getIntPtrConstant(0, dl));
30851 }
30852 
30853 /// Compute the horizontal sum of bytes in V for the elements of VT.
30854 ///
30855 /// Requires V to be a byte vector and VT to be an integer vector type with
30856 /// wider elements than V's type. The width of the elements of VT determines
30857 /// how many bytes of V are summed horizontally to produce each element of the
30858 /// result.
30859 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
30860                                       const X86Subtarget &Subtarget,
30861                                       SelectionDAG &DAG) {
30862   SDLoc DL(V);
30863   MVT ByteVecVT = V.getSimpleValueType();
30864   MVT EltVT = VT.getVectorElementType();
30865   assert(ByteVecVT.getVectorElementType() == MVT::i8 &&
30866          "Expected value to have byte element type.");
30867   assert(EltVT != MVT::i8 &&
30868          "Horizontal byte sum only makes sense for wider elements!");
30869   unsigned VecSize = VT.getSizeInBits();
30870   assert(ByteVecVT.getSizeInBits() == VecSize && "Cannot change vector size!");
30871 
30872   // PSADBW instruction horizontally add all bytes and leave the result in i64
30873   // chunks, thus directly computes the pop count for v2i64 and v4i64.
30874   if (EltVT == MVT::i64) {
30875     SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
30876     MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);
30877     V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros);
30878     return DAG.getBitcast(VT, V);
30879   }
30880 
30881   if (EltVT == MVT::i32) {
30882     // We unpack the low half and high half into i32s interleaved with zeros so
30883     // that we can use PSADBW to horizontally sum them. The most useful part of
30884     // this is that it lines up the results of two PSADBW instructions to be
30885     // two v2i64 vectors which concatenated are the 4 population counts. We can
30886     // then use PACKUSWB to shrink and concatenate them into a v4i32 again.
30887     SDValue Zeros = DAG.getConstant(0, DL, VT);
30888     SDValue V32 = DAG.getBitcast(VT, V);
30889     SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
30890     SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
30891 
30892     // Do the horizontal sums into two v2i64s.
30893     Zeros = DAG.getConstant(0, DL, ByteVecVT);
30894     MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);
30895     Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
30896                       DAG.getBitcast(ByteVecVT, Low), Zeros);
30897     High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
30898                        DAG.getBitcast(ByteVecVT, High), Zeros);
30899 
30900     // Merge them together.
30901     MVT ShortVecVT = MVT::getVectorVT(MVT::i16, VecSize / 16);
30902     V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
30903                     DAG.getBitcast(ShortVecVT, Low),
30904                     DAG.getBitcast(ShortVecVT, High));
30905 
30906     return DAG.getBitcast(VT, V);
30907   }
30908 
30909   // The only element type left is i16.
30910   assert(EltVT == MVT::i16 && "Unknown how to handle type");
30911 
30912   // To obtain pop count for each i16 element starting from the pop count for
30913   // i8 elements, shift the i16s left by 8, sum as i8s, and then shift as i16s
30914   // right by 8. It is important to shift as i16s as i8 vector shift isn't
30915   // directly supported.
30916   SDValue ShifterV = DAG.getConstant(8, DL, VT);
30917   SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
30918   V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl),
30919                   DAG.getBitcast(ByteVecVT, V));
30920   return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
30921 }
30922 
30923 static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
30924                                         const X86Subtarget &Subtarget,
30925                                         SelectionDAG &DAG) {
30926   MVT VT = Op.getSimpleValueType();
30927   MVT EltVT = VT.getVectorElementType();
30928   int NumElts = VT.getVectorNumElements();
30929   (void)EltVT;
30930   assert(EltVT == MVT::i8 && "Only vXi8 vector CTPOP lowering supported.");
30931 
30932   // Implement a lookup table in register by using an algorithm based on:
30933   // http://wm.ite.pl/articles/sse-popcount.html
30934   //
30935   // The general idea is that every lower byte nibble in the input vector is an
30936   // index into a in-register pre-computed pop count table. We then split up the
30937   // input vector in two new ones: (1) a vector with only the shifted-right
30938   // higher nibbles for each byte and (2) a vector with the lower nibbles (and
30939   // masked out higher ones) for each byte. PSHUFB is used separately with both
30940   // to index the in-register table. Next, both are added and the result is a
30941   // i8 vector where each element contains the pop count for input byte.
30942   const int LUT[16] = {/* 0 */ 0, /* 1 */ 1, /* 2 */ 1, /* 3 */ 2,
30943                        /* 4 */ 1, /* 5 */ 2, /* 6 */ 2, /* 7 */ 3,
30944                        /* 8 */ 1, /* 9 */ 2, /* a */ 2, /* b */ 3,
30945                        /* c */ 2, /* d */ 3, /* e */ 3, /* f */ 4};
30946 
30947   SmallVector<SDValue, 64> LUTVec;
30948   for (int i = 0; i < NumElts; ++i)
30949     LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
30950   SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
30951   SDValue M0F = DAG.getConstant(0x0F, DL, VT);
30952 
30953   // High nibbles
30954   SDValue FourV = DAG.getConstant(4, DL, VT);
30955   SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
30956 
30957   // Low nibbles
30958   SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
30959 
30960   // The input vector is used as the shuffle mask that index elements into the
30961   // LUT. After counting low and high nibbles, add the vector to obtain the
30962   // final pop count per i8 element.
30963   SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
30964   SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
30965   return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt);
30966 }
30967 
30968 // Please ensure that any codegen change from LowerVectorCTPOP is reflected in
30969 // updated cost models in X86TTIImpl::getIntrinsicInstrCost.
30970 static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget,
30971                                 SelectionDAG &DAG) {
30972   MVT VT = Op.getSimpleValueType();
30973   assert((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) &&
30974          "Unknown CTPOP type to handle");
30975   SDLoc DL(Op.getNode());
30976   SDValue Op0 = Op.getOperand(0);
30977 
30978   // TRUNC(CTPOP(ZEXT(X))) to make use of vXi32/vXi64 VPOPCNT instructions.
30979   if (Subtarget.hasVPOPCNTDQ()) {
30980     unsigned NumElems = VT.getVectorNumElements();
30981     assert((VT.getVectorElementType() == MVT::i8 ||
30982             VT.getVectorElementType() == MVT::i16) && "Unexpected type");
30983     if (NumElems < 16 || (NumElems == 16 && Subtarget.canExtendTo512DQ())) {
30984       MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
30985       Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0);
30986       Op = DAG.getNode(ISD::CTPOP, DL, NewVT, Op);
30987       return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
30988     }
30989   }
30990 
30991   // Decompose 256-bit ops into smaller 128-bit ops.
30992   if (VT.is256BitVector() && !Subtarget.hasInt256())
30993     return splitVectorIntUnary(Op, DAG);
30994 
30995   // Decompose 512-bit ops into smaller 256-bit ops.
30996   if (VT.is512BitVector() && !Subtarget.hasBWI())
30997     return splitVectorIntUnary(Op, DAG);
30998 
30999   // For element types greater than i8, do vXi8 pop counts and a bytesum.
31000   if (VT.getScalarType() != MVT::i8) {
31001     MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
31002     SDValue ByteOp = DAG.getBitcast(ByteVT, Op0);
31003     SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
31004     return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG);
31005   }
31006 
31007   // We can't use the fast LUT approach, so fall back on LegalizeDAG.
31008   if (!Subtarget.hasSSSE3())
31009     return SDValue();
31010 
31011   return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG);
31012 }
31013 
31014 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget &Subtarget,
31015                           SelectionDAG &DAG) {
31016   assert(Op.getSimpleValueType().isVector() &&
31017          "We only do custom lowering for vector population count.");
31018   return LowerVectorCTPOP(Op, Subtarget, DAG);
31019 }
31020 
31021 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
31022   MVT VT = Op.getSimpleValueType();
31023   SDValue In = Op.getOperand(0);
31024   SDLoc DL(Op);
31025 
31026   // For scalars, its still beneficial to transfer to/from the SIMD unit to
31027   // perform the BITREVERSE.
31028   if (!VT.isVector()) {
31029     MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits());
31030     SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
31031     Res = DAG.getNode(ISD::BITREVERSE, DL, VecVT, Res);
31032     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res,
31033                        DAG.getIntPtrConstant(0, DL));
31034   }
31035 
31036   int NumElts = VT.getVectorNumElements();
31037   int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
31038 
31039   // Decompose 256-bit ops into smaller 128-bit ops.
31040   if (VT.is256BitVector())
31041     return splitVectorIntUnary(Op, DAG);
31042 
31043   assert(VT.is128BitVector() &&
31044          "Only 128-bit vector bitreverse lowering supported.");
31045 
31046   // VPPERM reverses the bits of a byte with the permute Op (2 << 5), and we
31047   // perform the BSWAP in the shuffle.
31048   // Its best to shuffle using the second operand as this will implicitly allow
31049   // memory folding for multiple vectors.
31050   SmallVector<SDValue, 16> MaskElts;
31051   for (int i = 0; i != NumElts; ++i) {
31052     for (int j = ScalarSizeInBytes - 1; j >= 0; --j) {
31053       int SourceByte = 16 + (i * ScalarSizeInBytes) + j;
31054       int PermuteByte = SourceByte | (2 << 5);
31055       MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8));
31056     }
31057   }
31058 
31059   SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
31060   SDValue Res = DAG.getBitcast(MVT::v16i8, In);
31061   Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),
31062                     Res, Mask);
31063   return DAG.getBitcast(VT, Res);
31064 }
31065 
31066 static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
31067                                SelectionDAG &DAG) {
31068   MVT VT = Op.getSimpleValueType();
31069 
31070   if (Subtarget.hasXOP() && !VT.is512BitVector())
31071     return LowerBITREVERSE_XOP(Op, DAG);
31072 
31073   assert(Subtarget.hasSSSE3() && "SSSE3 required for BITREVERSE");
31074 
31075   SDValue In = Op.getOperand(0);
31076   SDLoc DL(Op);
31077 
31078   assert(VT.getScalarType() == MVT::i8 &&
31079          "Only byte vector BITREVERSE supported");
31080 
31081   // Split v64i8 without BWI so that we can still use the PSHUFB lowering.
31082   if (VT == MVT::v64i8 && !Subtarget.hasBWI())
31083     return splitVectorIntUnary(Op, DAG);
31084 
31085   // Decompose 256-bit ops into smaller 128-bit ops on pre-AVX2.
31086   if (VT == MVT::v32i8 && !Subtarget.hasInt256())
31087     return splitVectorIntUnary(Op, DAG);
31088 
31089   unsigned NumElts = VT.getVectorNumElements();
31090 
31091   // If we have GFNI, we can use GF2P8AFFINEQB to reverse the bits.
31092   if (Subtarget.hasGFNI()) {
31093     MVT MatrixVT = MVT::getVectorVT(MVT::i64, NumElts / 8);
31094     SDValue Matrix = DAG.getConstant(0x8040201008040201ULL, DL, MatrixVT);
31095     Matrix = DAG.getBitcast(VT, Matrix);
31096     return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix,
31097                        DAG.getTargetConstant(0, DL, MVT::i8));
31098   }
31099 
31100   // Perform BITREVERSE using PSHUFB lookups. Each byte is split into
31101   // two nibbles and a PSHUFB lookup to find the bitreverse of each
31102   // 0-15 value (moved to the other nibble).
31103   SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
31104   SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
31105   SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
31106 
31107   const int LoLUT[16] = {
31108       /* 0 */ 0x00, /* 1 */ 0x80, /* 2 */ 0x40, /* 3 */ 0xC0,
31109       /* 4 */ 0x20, /* 5 */ 0xA0, /* 6 */ 0x60, /* 7 */ 0xE0,
31110       /* 8 */ 0x10, /* 9 */ 0x90, /* a */ 0x50, /* b */ 0xD0,
31111       /* c */ 0x30, /* d */ 0xB0, /* e */ 0x70, /* f */ 0xF0};
31112   const int HiLUT[16] = {
31113       /* 0 */ 0x00, /* 1 */ 0x08, /* 2 */ 0x04, /* 3 */ 0x0C,
31114       /* 4 */ 0x02, /* 5 */ 0x0A, /* 6 */ 0x06, /* 7 */ 0x0E,
31115       /* 8 */ 0x01, /* 9 */ 0x09, /* a */ 0x05, /* b */ 0x0D,
31116       /* c */ 0x03, /* d */ 0x0B, /* e */ 0x07, /* f */ 0x0F};
31117 
31118   SmallVector<SDValue, 16> LoMaskElts, HiMaskElts;
31119   for (unsigned i = 0; i < NumElts; ++i) {
31120     LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8));
31121     HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8));
31122   }
31123 
31124   SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
31125   SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
31126   Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);
31127   Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);
31128   return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
31129 }
31130 
31131 static SDValue LowerPARITY(SDValue Op, const X86Subtarget &Subtarget,
31132                            SelectionDAG &DAG) {
31133   SDLoc DL(Op);
31134   SDValue X = Op.getOperand(0);
31135   MVT VT = Op.getSimpleValueType();
31136 
31137   // Special case. If the input fits in 8-bits we can use a single 8-bit TEST.
31138   if (VT == MVT::i8 ||
31139       DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) {
31140     X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
31141     SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X,
31142                                 DAG.getConstant(0, DL, MVT::i8));
31143     // Copy the inverse of the parity flag into a register with setcc.
31144     SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
31145     // Extend to the original type.
31146     return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp);
31147   }
31148 
31149   // If we have POPCNT, use the default expansion.
31150   if (Subtarget.hasPOPCNT())
31151     return SDValue();
31152 
31153   if (VT == MVT::i64) {
31154     // Xor the high and low 16-bits together using a 32-bit operation.
31155     SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
31156                              DAG.getNode(ISD::SRL, DL, MVT::i64, X,
31157                                          DAG.getConstant(32, DL, MVT::i8)));
31158     SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
31159     X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi);
31160   }
31161 
31162   if (VT != MVT::i16) {
31163     // Xor the high and low 16-bits together using a 32-bit operation.
31164     SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X,
31165                                DAG.getConstant(16, DL, MVT::i8));
31166     X = DAG.getNode(ISD::XOR, DL, MVT::i32, X, Hi16);
31167   } else {
31168     // If the input is 16-bits, we need to extend to use an i32 shift below.
31169     X = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, X);
31170   }
31171 
31172   // Finally xor the low 2 bytes together and use a 8-bit flag setting xor.
31173   // This should allow an h-reg to be used to save a shift.
31174   SDValue Hi = DAG.getNode(
31175       ISD::TRUNCATE, DL, MVT::i8,
31176       DAG.getNode(ISD::SRL, DL, MVT::i32, X, DAG.getConstant(8, DL, MVT::i8)));
31177   SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
31178   SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32);
31179   SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
31180 
31181   // Copy the inverse of the parity flag into a register with setcc.
31182   SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
31183   // Extend to the original type.
31184   return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp);
31185 }
31186 
31187 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
31188                                         const X86Subtarget &Subtarget) {
31189   unsigned NewOpc = 0;
31190   switch (N->getOpcode()) {
31191   case ISD::ATOMIC_LOAD_ADD:
31192     NewOpc = X86ISD::LADD;
31193     break;
31194   case ISD::ATOMIC_LOAD_SUB:
31195     NewOpc = X86ISD::LSUB;
31196     break;
31197   case ISD::ATOMIC_LOAD_OR:
31198     NewOpc = X86ISD::LOR;
31199     break;
31200   case ISD::ATOMIC_LOAD_XOR:
31201     NewOpc = X86ISD::LXOR;
31202     break;
31203   case ISD::ATOMIC_LOAD_AND:
31204     NewOpc = X86ISD::LAND;
31205     break;
31206   default:
31207     llvm_unreachable("Unknown ATOMIC_LOAD_ opcode");
31208   }
31209 
31210   MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
31211 
31212   return DAG.getMemIntrinsicNode(
31213       NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other),
31214       {N->getOperand(0), N->getOperand(1), N->getOperand(2)},
31215       /*MemVT=*/N->getSimpleValueType(0), MMO);
31216 }
31217 
31218 /// Lower atomic_load_ops into LOCK-prefixed operations.
31219 static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
31220                                 const X86Subtarget &Subtarget) {
31221   AtomicSDNode *AN = cast<AtomicSDNode>(N.getNode());
31222   SDValue Chain = N->getOperand(0);
31223   SDValue LHS = N->getOperand(1);
31224   SDValue RHS = N->getOperand(2);
31225   unsigned Opc = N->getOpcode();
31226   MVT VT = N->getSimpleValueType(0);
31227   SDLoc DL(N);
31228 
31229   // We can lower atomic_load_add into LXADD. However, any other atomicrmw op
31230   // can only be lowered when the result is unused.  They should have already
31231   // been transformed into a cmpxchg loop in AtomicExpand.
31232   if (N->hasAnyUseOfValue(0)) {
31233     // Handle (atomic_load_sub p, v) as (atomic_load_add p, -v), to be able to
31234     // select LXADD if LOCK_SUB can't be selected.
31235     // Handle (atomic_load_xor p, SignBit) as (atomic_load_add p, SignBit) so we
31236     // can use LXADD as opposed to cmpxchg.
31237     if (Opc == ISD::ATOMIC_LOAD_SUB ||
31238         (Opc == ISD::ATOMIC_LOAD_XOR && isMinSignedConstant(RHS))) {
31239       RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
31240       return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS, RHS,
31241                            AN->getMemOperand());
31242     }
31243     assert(Opc == ISD::ATOMIC_LOAD_ADD &&
31244            "Used AtomicRMW ops other than Add should have been expanded!");
31245     return N;
31246   }
31247 
31248   // Specialized lowering for the canonical form of an idemptotent atomicrmw.
31249   // The core idea here is that since the memory location isn't actually
31250   // changing, all we need is a lowering for the *ordering* impacts of the
31251   // atomicrmw.  As such, we can chose a different operation and memory
31252   // location to minimize impact on other code.
31253   // The above holds unless the node is marked volatile in which
31254   // case it needs to be preserved according to the langref.
31255   if (Opc == ISD::ATOMIC_LOAD_OR && isNullConstant(RHS) && !AN->isVolatile()) {
31256     // On X86, the only ordering which actually requires an instruction is
31257     // seq_cst which isn't SingleThread, everything just needs to be preserved
31258     // during codegen and then dropped. Note that we expect (but don't assume),
31259     // that orderings other than seq_cst and acq_rel have been canonicalized to
31260     // a store or load.
31261     if (AN->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent &&
31262         AN->getSyncScopeID() == SyncScope::System) {
31263       // Prefer a locked operation against a stack location to minimize cache
31264       // traffic.  This assumes that stack locations are very likely to be
31265       // accessed only by the owning thread.
31266       SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
31267       assert(!N->hasAnyUseOfValue(0));
31268       // NOTE: The getUNDEF is needed to give something for the unused result 0.
31269       return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31270                          DAG.getUNDEF(VT), NewChain);
31271     }
31272     // MEMBARRIER is a compiler barrier; it codegens to a no-op.
31273     SDValue NewChain = DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Chain);
31274     assert(!N->hasAnyUseOfValue(0));
31275     // NOTE: The getUNDEF is needed to give something for the unused result 0.
31276     return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31277                        DAG.getUNDEF(VT), NewChain);
31278   }
31279 
31280   SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
31281   // RAUW the chain, but don't worry about the result, as it's unused.
31282   assert(!N->hasAnyUseOfValue(0));
31283   // NOTE: The getUNDEF is needed to give something for the unused result 0.
31284   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31285                      DAG.getUNDEF(VT), LockOp.getValue(1));
31286 }
31287 
31288 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
31289                                  const X86Subtarget &Subtarget) {
31290   auto *Node = cast<AtomicSDNode>(Op.getNode());
31291   SDLoc dl(Node);
31292   EVT VT = Node->getMemoryVT();
31293 
31294   bool IsSeqCst =
31295       Node->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent;
31296   bool IsTypeLegal = DAG.getTargetLoweringInfo().isTypeLegal(VT);
31297 
31298   // If this store is not sequentially consistent and the type is legal
31299   // we can just keep it.
31300   if (!IsSeqCst && IsTypeLegal)
31301     return Op;
31302 
31303   if (VT == MVT::i64 && !IsTypeLegal) {
31304     // For illegal i64 atomic_stores, we can try to use MOVQ or MOVLPS if SSE
31305     // is enabled.
31306     bool NoImplicitFloatOps =
31307         DAG.getMachineFunction().getFunction().hasFnAttribute(
31308             Attribute::NoImplicitFloat);
31309     if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
31310       SDValue Chain;
31311       if (Subtarget.hasSSE1()) {
31312         SDValue SclToVec =
31313             DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal());
31314         MVT StVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
31315         SclToVec = DAG.getBitcast(StVT, SclToVec);
31316         SDVTList Tys = DAG.getVTList(MVT::Other);
31317         SDValue Ops[] = {Node->getChain(), SclToVec, Node->getBasePtr()};
31318         Chain = DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops,
31319                                         MVT::i64, Node->getMemOperand());
31320       } else if (Subtarget.hasX87()) {
31321         // First load this into an 80-bit X87 register using a stack temporary.
31322         // This will put the whole integer into the significand.
31323         SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
31324         int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
31325         MachinePointerInfo MPI =
31326             MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
31327         Chain = DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr,
31328                              MPI, MaybeAlign(), MachineMemOperand::MOStore);
31329         SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
31330         SDValue LdOps[] = {Chain, StackPtr};
31331         SDValue Value = DAG.getMemIntrinsicNode(
31332             X86ISD::FILD, dl, Tys, LdOps, MVT::i64, MPI,
31333             /*Align*/ std::nullopt, MachineMemOperand::MOLoad);
31334         Chain = Value.getValue(1);
31335 
31336         // Now use an FIST to do the atomic store.
31337         SDValue StoreOps[] = {Chain, Value, Node->getBasePtr()};
31338         Chain =
31339             DAG.getMemIntrinsicNode(X86ISD::FIST, dl, DAG.getVTList(MVT::Other),
31340                                     StoreOps, MVT::i64, Node->getMemOperand());
31341       }
31342 
31343       if (Chain) {
31344         // If this is a sequentially consistent store, also emit an appropriate
31345         // barrier.
31346         if (IsSeqCst)
31347           Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl);
31348 
31349         return Chain;
31350       }
31351     }
31352   }
31353 
31354   // Convert seq_cst store -> xchg
31355   // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
31356   // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
31357   SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(),
31358                                Node->getOperand(0), Node->getOperand(2),
31359                                Node->getOperand(1), Node->getMemOperand());
31360   return Swap.getValue(1);
31361 }
31362 
31363 static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
31364   SDNode *N = Op.getNode();
31365   MVT VT = N->getSimpleValueType(0);
31366   unsigned Opc = Op.getOpcode();
31367 
31368   // Let legalize expand this if it isn't a legal type yet.
31369   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
31370     return SDValue();
31371 
31372   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
31373   SDLoc DL(N);
31374 
31375   // Set the carry flag.
31376   SDValue Carry = Op.getOperand(2);
31377   EVT CarryVT = Carry.getValueType();
31378   Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
31379                       Carry, DAG.getAllOnesConstant(DL, CarryVT));
31380 
31381   bool IsAdd = Opc == ISD::UADDO_CARRY || Opc == ISD::SADDO_CARRY;
31382   SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs,
31383                             Op.getOperand(0), Op.getOperand(1),
31384                             Carry.getValue(1));
31385 
31386   bool IsSigned = Opc == ISD::SADDO_CARRY || Opc == ISD::SSUBO_CARRY;
31387   SDValue SetCC = getSETCC(IsSigned ? X86::COND_O : X86::COND_B,
31388                            Sum.getValue(1), DL, DAG);
31389   if (N->getValueType(1) == MVT::i1)
31390     SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
31391 
31392   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
31393 }
31394 
31395 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
31396                             SelectionDAG &DAG) {
31397   assert(Subtarget.isTargetDarwin() && Subtarget.is64Bit());
31398 
31399   // For MacOSX, we want to call an alternative entry point: __sincos_stret,
31400   // which returns the values as { float, float } (in XMM0) or
31401   // { double, double } (which is returned in XMM0, XMM1).
31402   SDLoc dl(Op);
31403   SDValue Arg = Op.getOperand(0);
31404   EVT ArgVT = Arg.getValueType();
31405   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
31406 
31407   TargetLowering::ArgListTy Args;
31408   TargetLowering::ArgListEntry Entry;
31409 
31410   Entry.Node = Arg;
31411   Entry.Ty = ArgTy;
31412   Entry.IsSExt = false;
31413   Entry.IsZExt = false;
31414   Args.push_back(Entry);
31415 
31416   bool isF64 = ArgVT == MVT::f64;
31417   // Only optimize x86_64 for now. i386 is a bit messy. For f32,
31418   // the small struct {f32, f32} is returned in (eax, edx). For f64,
31419   // the results are returned via SRet in memory.
31420   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
31421   RTLIB::Libcall LC = isF64 ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
31422   const char *LibcallName = TLI.getLibcallName(LC);
31423   SDValue Callee =
31424       DAG.getExternalSymbol(LibcallName, TLI.getPointerTy(DAG.getDataLayout()));
31425 
31426   Type *RetTy = isF64 ? (Type *)StructType::get(ArgTy, ArgTy)
31427                       : (Type *)FixedVectorType::get(ArgTy, 4);
31428 
31429   TargetLowering::CallLoweringInfo CLI(DAG);
31430   CLI.setDebugLoc(dl)
31431       .setChain(DAG.getEntryNode())
31432       .setLibCallee(CallingConv::C, RetTy, Callee, std::move(Args));
31433 
31434   std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
31435 
31436   if (isF64)
31437     // Returned in xmm0 and xmm1.
31438     return CallResult.first;
31439 
31440   // Returned in bits 0:31 and 32:64 xmm0.
31441   SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
31442                                CallResult.first, DAG.getIntPtrConstant(0, dl));
31443   SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
31444                                CallResult.first, DAG.getIntPtrConstant(1, dl));
31445   SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
31446   return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
31447 }
31448 
31449 /// Widen a vector input to a vector of NVT.  The
31450 /// input vector must have the same element type as NVT.
31451 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
31452                             bool FillWithZeroes = false) {
31453   // Check if InOp already has the right width.
31454   MVT InVT = InOp.getSimpleValueType();
31455   if (InVT == NVT)
31456     return InOp;
31457 
31458   if (InOp.isUndef())
31459     return DAG.getUNDEF(NVT);
31460 
31461   assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
31462          "input and widen element type must match");
31463 
31464   unsigned InNumElts = InVT.getVectorNumElements();
31465   unsigned WidenNumElts = NVT.getVectorNumElements();
31466   assert(WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0 &&
31467          "Unexpected request for vector widening");
31468 
31469   SDLoc dl(InOp);
31470   if (InOp.getOpcode() == ISD::CONCAT_VECTORS &&
31471       InOp.getNumOperands() == 2) {
31472     SDValue N1 = InOp.getOperand(1);
31473     if ((ISD::isBuildVectorAllZeros(N1.getNode()) && FillWithZeroes) ||
31474         N1.isUndef()) {
31475       InOp = InOp.getOperand(0);
31476       InVT = InOp.getSimpleValueType();
31477       InNumElts = InVT.getVectorNumElements();
31478     }
31479   }
31480   if (ISD::isBuildVectorOfConstantSDNodes(InOp.getNode()) ||
31481       ISD::isBuildVectorOfConstantFPSDNodes(InOp.getNode())) {
31482     SmallVector<SDValue, 16> Ops;
31483     for (unsigned i = 0; i < InNumElts; ++i)
31484       Ops.push_back(InOp.getOperand(i));
31485 
31486     EVT EltVT = InOp.getOperand(0).getValueType();
31487 
31488     SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
31489       DAG.getUNDEF(EltVT);
31490     for (unsigned i = 0; i < WidenNumElts - InNumElts; ++i)
31491       Ops.push_back(FillVal);
31492     return DAG.getBuildVector(NVT, dl, Ops);
31493   }
31494   SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, NVT) :
31495     DAG.getUNDEF(NVT);
31496   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal,
31497                      InOp, DAG.getIntPtrConstant(0, dl));
31498 }
31499 
31500 static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
31501                              SelectionDAG &DAG) {
31502   assert(Subtarget.hasAVX512() &&
31503          "MGATHER/MSCATTER are supported on AVX-512 arch only");
31504 
31505   MaskedScatterSDNode *N = cast<MaskedScatterSDNode>(Op.getNode());
31506   SDValue Src = N->getValue();
31507   MVT VT = Src.getSimpleValueType();
31508   assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op");
31509   SDLoc dl(Op);
31510 
31511   SDValue Scale = N->getScale();
31512   SDValue Index = N->getIndex();
31513   SDValue Mask = N->getMask();
31514   SDValue Chain = N->getChain();
31515   SDValue BasePtr = N->getBasePtr();
31516 
31517   if (VT == MVT::v2f32 || VT == MVT::v2i32) {
31518     assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
31519     // If the index is v2i64 and we have VLX we can use xmm for data and index.
31520     if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) {
31521       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
31522       EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
31523       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT));
31524       SDVTList VTs = DAG.getVTList(MVT::Other);
31525       SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
31526       return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
31527                                      N->getMemoryVT(), N->getMemOperand());
31528     }
31529     return SDValue();
31530   }
31531 
31532   MVT IndexVT = Index.getSimpleValueType();
31533 
31534   // If the index is v2i32, we're being called by type legalization and we
31535   // should just let the default handling take care of it.
31536   if (IndexVT == MVT::v2i32)
31537     return SDValue();
31538 
31539   // If we don't have VLX and neither the passthru or index is 512-bits, we
31540   // need to widen until one is.
31541   if (!Subtarget.hasVLX() && !VT.is512BitVector() &&
31542       !Index.getSimpleValueType().is512BitVector()) {
31543     // Determine how much we need to widen by to get a 512-bit type.
31544     unsigned Factor = std::min(512/VT.getSizeInBits(),
31545                                512/IndexVT.getSizeInBits());
31546     unsigned NumElts = VT.getVectorNumElements() * Factor;
31547 
31548     VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
31549     IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);
31550     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
31551 
31552     Src = ExtendToType(Src, VT, DAG);
31553     Index = ExtendToType(Index, IndexVT, DAG);
31554     Mask = ExtendToType(Mask, MaskVT, DAG, true);
31555   }
31556 
31557   SDVTList VTs = DAG.getVTList(MVT::Other);
31558   SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
31559   return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
31560                                  N->getMemoryVT(), N->getMemOperand());
31561 }
31562 
31563 static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
31564                           SelectionDAG &DAG) {
31565 
31566   MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
31567   MVT VT = Op.getSimpleValueType();
31568   MVT ScalarVT = VT.getScalarType();
31569   SDValue Mask = N->getMask();
31570   MVT MaskVT = Mask.getSimpleValueType();
31571   SDValue PassThru = N->getPassThru();
31572   SDLoc dl(Op);
31573 
31574   // Handle AVX masked loads which don't support passthru other than 0.
31575   if (MaskVT.getVectorElementType() != MVT::i1) {
31576     // We also allow undef in the isel pattern.
31577     if (PassThru.isUndef() || ISD::isBuildVectorAllZeros(PassThru.getNode()))
31578       return Op;
31579 
31580     SDValue NewLoad = DAG.getMaskedLoad(
31581         VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
31582         getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(),
31583         N->getMemOperand(), N->getAddressingMode(), N->getExtensionType(),
31584         N->isExpandingLoad());
31585     // Emit a blend.
31586     SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
31587     return DAG.getMergeValues({ Select, NewLoad.getValue(1) }, dl);
31588   }
31589 
31590   assert((!N->isExpandingLoad() || Subtarget.hasAVX512()) &&
31591          "Expanding masked load is supported on AVX-512 target only!");
31592 
31593   assert((!N->isExpandingLoad() || ScalarVT.getSizeInBits() >= 32) &&
31594          "Expanding masked load is supported for 32 and 64-bit types only!");
31595 
31596   assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
31597          "Cannot lower masked load op.");
31598 
31599   assert((ScalarVT.getSizeInBits() >= 32 ||
31600           (Subtarget.hasBWI() &&
31601               (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&
31602          "Unsupported masked load op.");
31603 
31604   // This operation is legal for targets with VLX, but without
31605   // VLX the vector should be widened to 512 bit
31606   unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits();
31607   MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
31608   PassThru = ExtendToType(PassThru, WideDataVT, DAG);
31609 
31610   // Mask element has to be i1.
31611   assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
31612          "Unexpected mask type");
31613 
31614   MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);
31615 
31616   Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
31617   SDValue NewLoad = DAG.getMaskedLoad(
31618       WideDataVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
31619       PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
31620       N->getExtensionType(), N->isExpandingLoad());
31621 
31622   SDValue Extract =
31623       DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, NewLoad.getValue(0),
31624                   DAG.getIntPtrConstant(0, dl));
31625   SDValue RetOps[] = {Extract, NewLoad.getValue(1)};
31626   return DAG.getMergeValues(RetOps, dl);
31627 }
31628 
31629 static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
31630                            SelectionDAG &DAG) {
31631   MaskedStoreSDNode *N = cast<MaskedStoreSDNode>(Op.getNode());
31632   SDValue DataToStore = N->getValue();
31633   MVT VT = DataToStore.getSimpleValueType();
31634   MVT ScalarVT = VT.getScalarType();
31635   SDValue Mask = N->getMask();
31636   SDLoc dl(Op);
31637 
31638   assert((!N->isCompressingStore() || Subtarget.hasAVX512()) &&
31639          "Expanding masked load is supported on AVX-512 target only!");
31640 
31641   assert((!N->isCompressingStore() || ScalarVT.getSizeInBits() >= 32) &&
31642          "Expanding masked load is supported for 32 and 64-bit types only!");
31643 
31644   assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
31645          "Cannot lower masked store op.");
31646 
31647   assert((ScalarVT.getSizeInBits() >= 32 ||
31648           (Subtarget.hasBWI() &&
31649               (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&
31650           "Unsupported masked store op.");
31651 
31652   // This operation is legal for targets with VLX, but without
31653   // VLX the vector should be widened to 512 bit
31654   unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits();
31655   MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
31656 
31657   // Mask element has to be i1.
31658   assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
31659          "Unexpected mask type");
31660 
31661   MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);
31662 
31663   DataToStore = ExtendToType(DataToStore, WideDataVT, DAG);
31664   Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
31665   return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(),
31666                             N->getOffset(), Mask, N->getMemoryVT(),
31667                             N->getMemOperand(), N->getAddressingMode(),
31668                             N->isTruncatingStore(), N->isCompressingStore());
31669 }
31670 
31671 static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
31672                             SelectionDAG &DAG) {
31673   assert(Subtarget.hasAVX2() &&
31674          "MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only");
31675 
31676   MaskedGatherSDNode *N = cast<MaskedGatherSDNode>(Op.getNode());
31677   SDLoc dl(Op);
31678   MVT VT = Op.getSimpleValueType();
31679   SDValue Index = N->getIndex();
31680   SDValue Mask = N->getMask();
31681   SDValue PassThru = N->getPassThru();
31682   MVT IndexVT = Index.getSimpleValueType();
31683 
31684   assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op");
31685 
31686   // If the index is v2i32, we're being called by type legalization.
31687   if (IndexVT == MVT::v2i32)
31688     return SDValue();
31689 
31690   // If we don't have VLX and neither the passthru or index is 512-bits, we
31691   // need to widen until one is.
31692   MVT OrigVT = VT;
31693   if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
31694       !IndexVT.is512BitVector()) {
31695     // Determine how much we need to widen by to get a 512-bit type.
31696     unsigned Factor = std::min(512/VT.getSizeInBits(),
31697                                512/IndexVT.getSizeInBits());
31698 
31699     unsigned NumElts = VT.getVectorNumElements() * Factor;
31700 
31701     VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
31702     IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);
31703     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
31704 
31705     PassThru = ExtendToType(PassThru, VT, DAG);
31706     Index = ExtendToType(Index, IndexVT, DAG);
31707     Mask = ExtendToType(Mask, MaskVT, DAG, true);
31708   }
31709 
31710   // Break dependency on the data register.
31711   if (PassThru.isUndef())
31712     PassThru = getZeroVector(VT, Subtarget, DAG, dl);
31713 
31714   SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
31715                     N->getScale() };
31716   SDValue NewGather = DAG.getMemIntrinsicNode(
31717       X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(),
31718       N->getMemOperand());
31719   SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT,
31720                                 NewGather, DAG.getIntPtrConstant(0, dl));
31721   return DAG.getMergeValues({Extract, NewGather.getValue(1)}, dl);
31722 }
31723 
31724 static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
31725   SDLoc dl(Op);
31726   SDValue Src = Op.getOperand(0);
31727   MVT DstVT = Op.getSimpleValueType();
31728 
31729   AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Op.getNode());
31730   unsigned SrcAS = N->getSrcAddressSpace();
31731 
31732   assert(SrcAS != N->getDestAddressSpace() &&
31733          "addrspacecast must be between different address spaces");
31734 
31735   if (SrcAS == X86AS::PTR32_UPTR && DstVT == MVT::i64) {
31736     Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Src);
31737   } else if (DstVT == MVT::i64) {
31738     Op = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Src);
31739   } else if (DstVT == MVT::i32) {
31740     Op = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src);
31741   } else {
31742     report_fatal_error("Bad address space in addrspacecast");
31743   }
31744   return Op;
31745 }
31746 
31747 SDValue X86TargetLowering::LowerGC_TRANSITION(SDValue Op,
31748                                               SelectionDAG &DAG) const {
31749   // TODO: Eventually, the lowering of these nodes should be informed by or
31750   // deferred to the GC strategy for the function in which they appear. For
31751   // now, however, they must be lowered to something. Since they are logically
31752   // no-ops in the case of a null GC strategy (or a GC strategy which does not
31753   // require special handling for these nodes), lower them as literal NOOPs for
31754   // the time being.
31755   SmallVector<SDValue, 2> Ops;
31756   Ops.push_back(Op.getOperand(0));
31757   if (Op->getGluedNode())
31758     Ops.push_back(Op->getOperand(Op->getNumOperands() - 1));
31759 
31760   SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
31761   return SDValue(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
31762 }
31763 
31764 // Custom split CVTPS2PH with wide types.
31765 static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) {
31766   SDLoc dl(Op);
31767   EVT VT = Op.getValueType();
31768   SDValue Lo, Hi;
31769   std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
31770   EVT LoVT, HiVT;
31771   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
31772   SDValue RC = Op.getOperand(1);
31773   Lo = DAG.getNode(X86ISD::CVTPS2PH, dl, LoVT, Lo, RC);
31774   Hi = DAG.getNode(X86ISD::CVTPS2PH, dl, HiVT, Hi, RC);
31775   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
31776 }
31777 
31778 static SDValue LowerPREFETCH(SDValue Op, const X86Subtarget &Subtarget,
31779                              SelectionDAG &DAG) {
31780   unsigned IsData = Op.getConstantOperandVal(4);
31781 
31782   // We don't support non-data prefetch without PREFETCHI.
31783   // Just preserve the chain.
31784   if (!IsData && !Subtarget.hasPREFETCHI())
31785     return Op.getOperand(0);
31786 
31787   return Op;
31788 }
31789 
31790 static StringRef getInstrStrFromOpNo(const SmallVectorImpl<StringRef> &AsmStrs,
31791                                      unsigned OpNo) {
31792   const APInt Operand(32, OpNo);
31793   std::string OpNoStr = llvm::toString(Operand, 10, false);
31794   std::string Str(" $");
31795 
31796   std::string OpNoStr1(Str + OpNoStr);             // e.g. " $1" (OpNo=1)
31797   std::string OpNoStr2(Str + "{" + OpNoStr + ":"); // With modifier, e.g. ${1:P}
31798 
31799   auto I = StringRef::npos;
31800   for (auto &AsmStr : AsmStrs) {
31801     // Match the OpNo string. We should match exactly to exclude match
31802     // sub-string, e.g. "$12" contain "$1"
31803     if (AsmStr.ends_with(OpNoStr1))
31804       I = AsmStr.size() - OpNoStr1.size();
31805 
31806     // Get the index of operand in AsmStr.
31807     if (I == StringRef::npos)
31808       I = AsmStr.find(OpNoStr1 + ",");
31809     if (I == StringRef::npos)
31810       I = AsmStr.find(OpNoStr2);
31811 
31812     if (I == StringRef::npos)
31813       continue;
31814 
31815     assert(I > 0 && "Unexpected inline asm string!");
31816     // Remove the operand string and label (if exsit).
31817     // For example:
31818     // ".L__MSASMLABEL_.${:uid}__l:call dword ptr ${0:P}"
31819     // ==>
31820     // ".L__MSASMLABEL_.${:uid}__l:call dword ptr "
31821     // ==>
31822     // "call dword ptr "
31823     auto TmpStr = AsmStr.substr(0, I);
31824     I = TmpStr.rfind(':');
31825     if (I != StringRef::npos)
31826       TmpStr = TmpStr.substr(I + 1);
31827     return TmpStr.take_while(llvm::isAlpha);
31828   }
31829 
31830   return StringRef();
31831 }
31832 
31833 bool X86TargetLowering::isInlineAsmTargetBranch(
31834     const SmallVectorImpl<StringRef> &AsmStrs, unsigned OpNo) const {
31835   // In a __asm block, __asm inst foo where inst is CALL or JMP should be
31836   // changed from indirect TargetLowering::C_Memory to direct
31837   // TargetLowering::C_Address.
31838   // We don't need to special case LOOP* and Jcc, which cannot target a memory
31839   // location.
31840   StringRef Inst = getInstrStrFromOpNo(AsmStrs, OpNo);
31841   return Inst.equals_insensitive("call") || Inst.equals_insensitive("jmp");
31842 }
31843 
31844 /// Provide custom lowering hooks for some operations.
31845 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
31846   switch (Op.getOpcode()) {
31847   default: llvm_unreachable("Should not custom lower this!");
31848   case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op, Subtarget, DAG);
31849   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
31850     return LowerCMP_SWAP(Op, Subtarget, DAG);
31851   case ISD::CTPOP:              return LowerCTPOP(Op, Subtarget, DAG);
31852   case ISD::ATOMIC_LOAD_ADD:
31853   case ISD::ATOMIC_LOAD_SUB:
31854   case ISD::ATOMIC_LOAD_OR:
31855   case ISD::ATOMIC_LOAD_XOR:
31856   case ISD::ATOMIC_LOAD_AND:    return lowerAtomicArith(Op, DAG, Subtarget);
31857   case ISD::ATOMIC_STORE:       return LowerATOMIC_STORE(Op, DAG, Subtarget);
31858   case ISD::BITREVERSE:         return LowerBITREVERSE(Op, Subtarget, DAG);
31859   case ISD::PARITY:             return LowerPARITY(Op, Subtarget, DAG);
31860   case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
31861   case ISD::CONCAT_VECTORS:     return LowerCONCAT_VECTORS(Op, Subtarget, DAG);
31862   case ISD::VECTOR_SHUFFLE:     return lowerVECTOR_SHUFFLE(Op, Subtarget, DAG);
31863   case ISD::VSELECT:            return LowerVSELECT(Op, DAG);
31864   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
31865   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
31866   case ISD::INSERT_SUBVECTOR:   return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
31867   case ISD::EXTRACT_SUBVECTOR:  return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
31868   case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, Subtarget,DAG);
31869   case ISD::ConstantPool:       return LowerConstantPool(Op, DAG);
31870   case ISD::GlobalAddress:      return LowerGlobalAddress(Op, DAG);
31871   case ISD::GlobalTLSAddress:   return LowerGlobalTLSAddress(Op, DAG);
31872   case ISD::ExternalSymbol:     return LowerExternalSymbol(Op, DAG);
31873   case ISD::BlockAddress:       return LowerBlockAddress(Op, DAG);
31874   case ISD::SHL_PARTS:
31875   case ISD::SRA_PARTS:
31876   case ISD::SRL_PARTS:          return LowerShiftParts(Op, DAG);
31877   case ISD::FSHL:
31878   case ISD::FSHR:               return LowerFunnelShift(Op, Subtarget, DAG);
31879   case ISD::STRICT_SINT_TO_FP:
31880   case ISD::SINT_TO_FP:         return LowerSINT_TO_FP(Op, DAG);
31881   case ISD::STRICT_UINT_TO_FP:
31882   case ISD::UINT_TO_FP:         return LowerUINT_TO_FP(Op, DAG);
31883   case ISD::TRUNCATE:           return LowerTRUNCATE(Op, DAG);
31884   case ISD::ZERO_EXTEND:        return LowerZERO_EXTEND(Op, Subtarget, DAG);
31885   case ISD::SIGN_EXTEND:        return LowerSIGN_EXTEND(Op, Subtarget, DAG);
31886   case ISD::ANY_EXTEND:         return LowerANY_EXTEND(Op, Subtarget, DAG);
31887   case ISD::ZERO_EXTEND_VECTOR_INREG:
31888   case ISD::SIGN_EXTEND_VECTOR_INREG:
31889     return LowerEXTEND_VECTOR_INREG(Op, Subtarget, DAG);
31890   case ISD::FP_TO_SINT:
31891   case ISD::STRICT_FP_TO_SINT:
31892   case ISD::FP_TO_UINT:
31893   case ISD::STRICT_FP_TO_UINT:  return LowerFP_TO_INT(Op, DAG);
31894   case ISD::FP_TO_SINT_SAT:
31895   case ISD::FP_TO_UINT_SAT:     return LowerFP_TO_INT_SAT(Op, DAG);
31896   case ISD::FP_EXTEND:
31897   case ISD::STRICT_FP_EXTEND:   return LowerFP_EXTEND(Op, DAG);
31898   case ISD::FP_ROUND:
31899   case ISD::STRICT_FP_ROUND:    return LowerFP_ROUND(Op, DAG);
31900   case ISD::FP16_TO_FP:
31901   case ISD::STRICT_FP16_TO_FP:  return LowerFP16_TO_FP(Op, DAG);
31902   case ISD::FP_TO_FP16:
31903   case ISD::STRICT_FP_TO_FP16:  return LowerFP_TO_FP16(Op, DAG);
31904   case ISD::FP_TO_BF16:         return LowerFP_TO_BF16(Op, DAG);
31905   case ISD::LOAD:               return LowerLoad(Op, Subtarget, DAG);
31906   case ISD::STORE:              return LowerStore(Op, Subtarget, DAG);
31907   case ISD::FADD:
31908   case ISD::FSUB:               return lowerFaddFsub(Op, DAG);
31909   case ISD::FROUND:             return LowerFROUND(Op, DAG);
31910   case ISD::FABS:
31911   case ISD::FNEG:               return LowerFABSorFNEG(Op, DAG);
31912   case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
31913   case ISD::FGETSIGN:           return LowerFGETSIGN(Op, DAG);
31914   case ISD::LRINT:
31915   case ISD::LLRINT:             return LowerLRINT_LLRINT(Op, DAG);
31916   case ISD::SETCC:
31917   case ISD::STRICT_FSETCC:
31918   case ISD::STRICT_FSETCCS:     return LowerSETCC(Op, DAG);
31919   case ISD::SETCCCARRY:         return LowerSETCCCARRY(Op, DAG);
31920   case ISD::SELECT:             return LowerSELECT(Op, DAG);
31921   case ISD::BRCOND:             return LowerBRCOND(Op, DAG);
31922   case ISD::JumpTable:          return LowerJumpTable(Op, DAG);
31923   case ISD::VASTART:            return LowerVASTART(Op, DAG);
31924   case ISD::VAARG:              return LowerVAARG(Op, DAG);
31925   case ISD::VACOPY:             return LowerVACOPY(Op, Subtarget, DAG);
31926   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
31927   case ISD::INTRINSIC_VOID:
31928   case ISD::INTRINSIC_W_CHAIN:  return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG);
31929   case ISD::RETURNADDR:         return LowerRETURNADDR(Op, DAG);
31930   case ISD::ADDROFRETURNADDR:   return LowerADDROFRETURNADDR(Op, DAG);
31931   case ISD::FRAMEADDR:          return LowerFRAMEADDR(Op, DAG);
31932   case ISD::FRAME_TO_ARGS_OFFSET:
31933                                 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
31934   case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
31935   case ISD::EH_RETURN:          return LowerEH_RETURN(Op, DAG);
31936   case ISD::EH_SJLJ_SETJMP:     return lowerEH_SJLJ_SETJMP(Op, DAG);
31937   case ISD::EH_SJLJ_LONGJMP:    return lowerEH_SJLJ_LONGJMP(Op, DAG);
31938   case ISD::EH_SJLJ_SETUP_DISPATCH:
31939     return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
31940   case ISD::INIT_TRAMPOLINE:    return LowerINIT_TRAMPOLINE(Op, DAG);
31941   case ISD::ADJUST_TRAMPOLINE:  return LowerADJUST_TRAMPOLINE(Op, DAG);
31942   case ISD::GET_ROUNDING:       return LowerGET_ROUNDING(Op, DAG);
31943   case ISD::SET_ROUNDING:       return LowerSET_ROUNDING(Op, DAG);
31944   case ISD::GET_FPENV_MEM:      return LowerGET_FPENV_MEM(Op, DAG);
31945   case ISD::SET_FPENV_MEM:      return LowerSET_FPENV_MEM(Op, DAG);
31946   case ISD::RESET_FPENV:        return LowerRESET_FPENV(Op, DAG);
31947   case ISD::CTLZ:
31948   case ISD::CTLZ_ZERO_UNDEF:    return LowerCTLZ(Op, Subtarget, DAG);
31949   case ISD::CTTZ:
31950   case ISD::CTTZ_ZERO_UNDEF:    return LowerCTTZ(Op, Subtarget, DAG);
31951   case ISD::MUL:                return LowerMUL(Op, Subtarget, DAG);
31952   case ISD::MULHS:
31953   case ISD::MULHU:              return LowerMULH(Op, Subtarget, DAG);
31954   case ISD::ROTL:
31955   case ISD::ROTR:               return LowerRotate(Op, Subtarget, DAG);
31956   case ISD::SRA:
31957   case ISD::SRL:
31958   case ISD::SHL:                return LowerShift(Op, Subtarget, DAG);
31959   case ISD::SADDO:
31960   case ISD::UADDO:
31961   case ISD::SSUBO:
31962   case ISD::USUBO:              return LowerXALUO(Op, DAG);
31963   case ISD::SMULO:
31964   case ISD::UMULO:              return LowerMULO(Op, Subtarget, DAG);
31965   case ISD::READCYCLECOUNTER:   return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
31966   case ISD::BITCAST:            return LowerBITCAST(Op, Subtarget, DAG);
31967   case ISD::SADDO_CARRY:
31968   case ISD::SSUBO_CARRY:
31969   case ISD::UADDO_CARRY:
31970   case ISD::USUBO_CARRY:        return LowerADDSUBO_CARRY(Op, DAG);
31971   case ISD::ADD:
31972   case ISD::SUB:                return lowerAddSub(Op, DAG, Subtarget);
31973   case ISD::UADDSAT:
31974   case ISD::SADDSAT:
31975   case ISD::USUBSAT:
31976   case ISD::SSUBSAT:            return LowerADDSAT_SUBSAT(Op, DAG, Subtarget);
31977   case ISD::SMAX:
31978   case ISD::SMIN:
31979   case ISD::UMAX:
31980   case ISD::UMIN:               return LowerMINMAX(Op, Subtarget, DAG);
31981   case ISD::FMINIMUM:
31982   case ISD::FMAXIMUM:
31983     return LowerFMINIMUM_FMAXIMUM(Op, Subtarget, DAG);
31984   case ISD::ABS:                return LowerABS(Op, Subtarget, DAG);
31985   case ISD::ABDS:
31986   case ISD::ABDU:               return LowerABD(Op, Subtarget, DAG);
31987   case ISD::AVGCEILU:           return LowerAVG(Op, Subtarget, DAG);
31988   case ISD::FSINCOS:            return LowerFSINCOS(Op, Subtarget, DAG);
31989   case ISD::MLOAD:              return LowerMLOAD(Op, Subtarget, DAG);
31990   case ISD::MSTORE:             return LowerMSTORE(Op, Subtarget, DAG);
31991   case ISD::MGATHER:            return LowerMGATHER(Op, Subtarget, DAG);
31992   case ISD::MSCATTER:           return LowerMSCATTER(Op, Subtarget, DAG);
31993   case ISD::GC_TRANSITION_START:
31994   case ISD::GC_TRANSITION_END:  return LowerGC_TRANSITION(Op, DAG);
31995   case ISD::ADDRSPACECAST:      return LowerADDRSPACECAST(Op, DAG);
31996   case X86ISD::CVTPS2PH:        return LowerCVTPS2PH(Op, DAG);
31997   case ISD::PREFETCH:           return LowerPREFETCH(Op, Subtarget, DAG);
31998   }
31999 }
32000 
32001 /// Replace a node with an illegal result type with a new node built out of
32002 /// custom code.
32003 void X86TargetLowering::ReplaceNodeResults(SDNode *N,
32004                                            SmallVectorImpl<SDValue>&Results,
32005                                            SelectionDAG &DAG) const {
32006   SDLoc dl(N);
32007   switch (N->getOpcode()) {
32008   default:
32009 #ifndef NDEBUG
32010     dbgs() << "ReplaceNodeResults: ";
32011     N->dump(&DAG);
32012 #endif
32013     llvm_unreachable("Do not know how to custom type legalize this operation!");
32014   case X86ISD::CVTPH2PS: {
32015     EVT VT = N->getValueType(0);
32016     SDValue Lo, Hi;
32017     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
32018     EVT LoVT, HiVT;
32019     std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
32020     Lo = DAG.getNode(X86ISD::CVTPH2PS, dl, LoVT, Lo);
32021     Hi = DAG.getNode(X86ISD::CVTPH2PS, dl, HiVT, Hi);
32022     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32023     Results.push_back(Res);
32024     return;
32025   }
32026   case X86ISD::STRICT_CVTPH2PS: {
32027     EVT VT = N->getValueType(0);
32028     SDValue Lo, Hi;
32029     std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 1);
32030     EVT LoVT, HiVT;
32031     std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
32032     Lo = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {LoVT, MVT::Other},
32033                      {N->getOperand(0), Lo});
32034     Hi = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {HiVT, MVT::Other},
32035                      {N->getOperand(0), Hi});
32036     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
32037                                 Lo.getValue(1), Hi.getValue(1));
32038     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32039     Results.push_back(Res);
32040     Results.push_back(Chain);
32041     return;
32042   }
32043   case X86ISD::CVTPS2PH:
32044     Results.push_back(LowerCVTPS2PH(SDValue(N, 0), DAG));
32045     return;
32046   case ISD::CTPOP: {
32047     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
32048     // Use a v2i64 if possible.
32049     bool NoImplicitFloatOps =
32050         DAG.getMachineFunction().getFunction().hasFnAttribute(
32051             Attribute::NoImplicitFloat);
32052     if (isTypeLegal(MVT::v2i64) && !NoImplicitFloatOps) {
32053       SDValue Wide =
32054           DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0));
32055       Wide = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, Wide);
32056       // Bit count should fit in 32-bits, extract it as that and then zero
32057       // extend to i64. Otherwise we end up extracting bits 63:32 separately.
32058       Wide = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Wide);
32059       Wide = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Wide,
32060                          DAG.getIntPtrConstant(0, dl));
32061       Wide = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Wide);
32062       Results.push_back(Wide);
32063     }
32064     return;
32065   }
32066   case ISD::MUL: {
32067     EVT VT = N->getValueType(0);
32068     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32069            VT.getVectorElementType() == MVT::i8 && "Unexpected VT!");
32070     // Pre-promote these to vXi16 to avoid op legalization thinking all 16
32071     // elements are needed.
32072     MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
32073     SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
32074     SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
32075     SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
32076     Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
32077     unsigned NumConcats = 16 / VT.getVectorNumElements();
32078     SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
32079     ConcatOps[0] = Res;
32080     Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps);
32081     Results.push_back(Res);
32082     return;
32083   }
32084   case ISD::SMULO:
32085   case ISD::UMULO: {
32086     EVT VT = N->getValueType(0);
32087     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32088            VT == MVT::v2i32 && "Unexpected VT!");
32089     bool IsSigned = N->getOpcode() == ISD::SMULO;
32090     unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
32091     SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0));
32092     SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1));
32093     SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1);
32094     // Extract the high 32 bits from each result using PSHUFD.
32095     // TODO: Could use SRL+TRUNCATE but that doesn't become a PSHUFD.
32096     SDValue Hi = DAG.getBitcast(MVT::v4i32, Res);
32097     Hi = DAG.getVectorShuffle(MVT::v4i32, dl, Hi, Hi, {1, 3, -1, -1});
32098     Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Hi,
32099                      DAG.getIntPtrConstant(0, dl));
32100 
32101     // Truncate the low bits of the result. This will become PSHUFD.
32102     Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
32103 
32104     SDValue HiCmp;
32105     if (IsSigned) {
32106       // SMULO overflows if the high bits don't match the sign of the low.
32107       HiCmp = DAG.getNode(ISD::SRA, dl, VT, Res, DAG.getConstant(31, dl, VT));
32108     } else {
32109       // UMULO overflows if the high bits are non-zero.
32110       HiCmp = DAG.getConstant(0, dl, VT);
32111     }
32112     SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE);
32113 
32114     // Widen the result with by padding with undef.
32115     Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
32116                       DAG.getUNDEF(VT));
32117     Results.push_back(Res);
32118     Results.push_back(Ovf);
32119     return;
32120   }
32121   case X86ISD::VPMADDWD: {
32122     // Legalize types for X86ISD::VPMADDWD by widening.
32123     assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32124 
32125     EVT VT = N->getValueType(0);
32126     EVT InVT = N->getOperand(0).getValueType();
32127     assert(VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 &&
32128            "Expected a VT that divides into 128 bits.");
32129     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32130            "Unexpected type action!");
32131     unsigned NumConcat = 128 / InVT.getSizeInBits();
32132 
32133     EVT InWideVT = EVT::getVectorVT(*DAG.getContext(),
32134                                     InVT.getVectorElementType(),
32135                                     NumConcat * InVT.getVectorNumElements());
32136     EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
32137                                   VT.getVectorElementType(),
32138                                   NumConcat * VT.getVectorNumElements());
32139 
32140     SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
32141     Ops[0] = N->getOperand(0);
32142     SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
32143     Ops[0] = N->getOperand(1);
32144     SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
32145 
32146     SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1);
32147     Results.push_back(Res);
32148     return;
32149   }
32150   // We might have generated v2f32 FMIN/FMAX operations. Widen them to v4f32.
32151   case X86ISD::FMINC:
32152   case X86ISD::FMIN:
32153   case X86ISD::FMAXC:
32154   case X86ISD::FMAX: {
32155     EVT VT = N->getValueType(0);
32156     assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX.");
32157     SDValue UNDEF = DAG.getUNDEF(VT);
32158     SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
32159                               N->getOperand(0), UNDEF);
32160     SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
32161                               N->getOperand(1), UNDEF);
32162     Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS));
32163     return;
32164   }
32165   case ISD::SDIV:
32166   case ISD::UDIV:
32167   case ISD::SREM:
32168   case ISD::UREM: {
32169     EVT VT = N->getValueType(0);
32170     if (VT.isVector()) {
32171       assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32172              "Unexpected type action!");
32173       // If this RHS is a constant splat vector we can widen this and let
32174       // division/remainder by constant optimize it.
32175       // TODO: Can we do something for non-splat?
32176       APInt SplatVal;
32177       if (ISD::isConstantSplatVector(N->getOperand(1).getNode(), SplatVal)) {
32178         unsigned NumConcats = 128 / VT.getSizeInBits();
32179         SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
32180         Ops0[0] = N->getOperand(0);
32181         EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT);
32182         SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
32183         SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
32184         SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
32185         Results.push_back(Res);
32186       }
32187       return;
32188     }
32189 
32190     SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
32191     Results.push_back(V);
32192     return;
32193   }
32194   case ISD::TRUNCATE: {
32195     MVT VT = N->getSimpleValueType(0);
32196     if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
32197       return;
32198 
32199     // The generic legalizer will try to widen the input type to the same
32200     // number of elements as the widened result type. But this isn't always
32201     // the best thing so do some custom legalization to avoid some cases.
32202     MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();
32203     SDValue In = N->getOperand(0);
32204     EVT InVT = In.getValueType();
32205     EVT InEltVT = InVT.getVectorElementType();
32206     EVT EltVT = VT.getVectorElementType();
32207     unsigned MinElts = VT.getVectorNumElements();
32208     unsigned WidenNumElts = WidenVT.getVectorNumElements();
32209     unsigned InBits = InVT.getSizeInBits();
32210 
32211     // See if there are sufficient leading bits to perform a PACKUS/PACKSS.
32212     unsigned PackOpcode;
32213     if (SDValue Src =
32214             matchTruncateWithPACK(PackOpcode, VT, In, dl, DAG, Subtarget)) {
32215       if (SDValue Res = truncateVectorWithPACK(PackOpcode, VT, Src,
32216                                                dl, DAG, Subtarget)) {
32217         Res = widenSubVector(WidenVT, Res, false, Subtarget, DAG, dl);
32218         Results.push_back(Res);
32219         return;
32220       }
32221     }
32222 
32223     if (128 % InBits == 0) {
32224       // 128 bit and smaller inputs should avoid truncate all together and
32225       // just use a build_vector that will become a shuffle.
32226       // TODO: Widen and use a shuffle directly?
32227       SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
32228       // Use the original element count so we don't do more scalar opts than
32229       // necessary.
32230       for (unsigned i=0; i < MinElts; ++i) {
32231         SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, In,
32232                                   DAG.getIntPtrConstant(i, dl));
32233         Ops[i] = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Val);
32234       }
32235       Results.push_back(DAG.getBuildVector(WidenVT, dl, Ops));
32236       return;
32237     }
32238 
32239     // With AVX512 there are some cases that can use a target specific
32240     // truncate node to go from 256/512 to less than 128 with zeros in the
32241     // upper elements of the 128 bit result.
32242     if (Subtarget.hasAVX512() && isTypeLegal(InVT)) {
32243       // We can use VTRUNC directly if for 256 bits with VLX or for any 512.
32244       if ((InBits == 256 && Subtarget.hasVLX()) || InBits == 512) {
32245         Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
32246         return;
32247       }
32248       // There's one case we can widen to 512 bits and use VTRUNC.
32249       if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) {
32250         In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In,
32251                          DAG.getUNDEF(MVT::v4i64));
32252         Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
32253         return;
32254       }
32255     }
32256     if (Subtarget.hasVLX() && InVT == MVT::v8i64 && VT == MVT::v8i8 &&
32257         getTypeAction(*DAG.getContext(), InVT) == TypeSplitVector &&
32258         isTypeLegal(MVT::v4i64)) {
32259       // Input needs to be split and output needs to widened. Let's use two
32260       // VTRUNCs, and shuffle their results together into the wider type.
32261       SDValue Lo, Hi;
32262       std::tie(Lo, Hi) = DAG.SplitVector(In, dl);
32263 
32264       Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo);
32265       Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi);
32266       SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
32267                                          { 0,  1,  2,  3, 16, 17, 18, 19,
32268                                           -1, -1, -1, -1, -1, -1, -1, -1 });
32269       Results.push_back(Res);
32270       return;
32271     }
32272 
32273     // Attempt to widen the truncation input vector to let LowerTRUNCATE handle
32274     // this via type legalization.
32275     if ((InEltVT == MVT::i16 || InEltVT == MVT::i32 || InEltVT == MVT::i64) &&
32276         (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32) &&
32277         (!Subtarget.hasSSSE3() ||
32278          (!isTypeLegal(InVT) &&
32279           !(MinElts <= 4 && InEltVT == MVT::i64 && EltVT == MVT::i8)))) {
32280       SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl,
32281                                        InEltVT.getSizeInBits() * WidenNumElts);
32282       Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, WidenVT, WidenIn));
32283       return;
32284     }
32285 
32286     return;
32287   }
32288   case ISD::ANY_EXTEND:
32289     // Right now, only MVT::v8i8 has Custom action for an illegal type.
32290     // It's intended to custom handle the input type.
32291     assert(N->getValueType(0) == MVT::v8i8 &&
32292            "Do not know how to legalize this Node");
32293     return;
32294   case ISD::SIGN_EXTEND:
32295   case ISD::ZERO_EXTEND: {
32296     EVT VT = N->getValueType(0);
32297     SDValue In = N->getOperand(0);
32298     EVT InVT = In.getValueType();
32299     if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
32300         (InVT == MVT::v4i16 || InVT == MVT::v4i8)){
32301       assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
32302              "Unexpected type action!");
32303       assert(N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode");
32304       // Custom split this so we can extend i8/i16->i32 invec. This is better
32305       // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
32306       // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting
32307       // we allow the sra from the extend to i32 to be shared by the split.
32308       In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In);
32309 
32310       // Fill a vector with sign bits for each element.
32311       SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
32312       SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);
32313 
32314       // Create an unpackl and unpackh to interleave the sign bits then bitcast
32315       // to v2i64.
32316       SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
32317                                         {0, 4, 1, 5});
32318       Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo);
32319       SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
32320                                         {2, 6, 3, 7});
32321       Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi);
32322 
32323       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32324       Results.push_back(Res);
32325       return;
32326     }
32327 
32328     if (VT == MVT::v16i32 || VT == MVT::v8i64) {
32329       if (!InVT.is128BitVector()) {
32330         // Not a 128 bit vector, but maybe type legalization will promote
32331         // it to 128 bits.
32332         if (getTypeAction(*DAG.getContext(), InVT) != TypePromoteInteger)
32333           return;
32334         InVT = getTypeToTransformTo(*DAG.getContext(), InVT);
32335         if (!InVT.is128BitVector())
32336           return;
32337 
32338         // Promote the input to 128 bits. Type legalization will turn this into
32339         // zext_inreg/sext_inreg.
32340         In = DAG.getNode(N->getOpcode(), dl, InVT, In);
32341       }
32342 
32343       // Perform custom splitting instead of the two stage extend we would get
32344       // by default.
32345       EVT LoVT, HiVT;
32346       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
32347       assert(isTypeLegal(LoVT) && "Split VT not legal?");
32348 
32349       SDValue Lo = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, LoVT, In, DAG);
32350 
32351       // We need to shift the input over by half the number of elements.
32352       unsigned NumElts = InVT.getVectorNumElements();
32353       unsigned HalfNumElts = NumElts / 2;
32354       SmallVector<int, 16> ShufMask(NumElts, SM_SentinelUndef);
32355       for (unsigned i = 0; i != HalfNumElts; ++i)
32356         ShufMask[i] = i + HalfNumElts;
32357 
32358       SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
32359       Hi = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, HiVT, Hi, DAG);
32360 
32361       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32362       Results.push_back(Res);
32363     }
32364     return;
32365   }
32366   case ISD::FP_TO_SINT:
32367   case ISD::STRICT_FP_TO_SINT:
32368   case ISD::FP_TO_UINT:
32369   case ISD::STRICT_FP_TO_UINT: {
32370     bool IsStrict = N->isStrictFPOpcode();
32371     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
32372                     N->getOpcode() == ISD::STRICT_FP_TO_SINT;
32373     EVT VT = N->getValueType(0);
32374     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32375     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
32376     EVT SrcVT = Src.getValueType();
32377 
32378     SDValue Res;
32379     if (isSoftF16(SrcVT, Subtarget)) {
32380       EVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
32381       if (IsStrict) {
32382         Res =
32383             DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},
32384                         {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
32385                                             {NVT, MVT::Other}, {Chain, Src})});
32386         Chain = Res.getValue(1);
32387       } else {
32388         Res = DAG.getNode(N->getOpcode(), dl, VT,
32389                           DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
32390       }
32391       Results.push_back(Res);
32392       if (IsStrict)
32393         Results.push_back(Chain);
32394 
32395       return;
32396     }
32397 
32398     if (VT.isVector() && Subtarget.hasFP16() &&
32399         SrcVT.getVectorElementType() == MVT::f16) {
32400       EVT EleVT = VT.getVectorElementType();
32401       EVT ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
32402 
32403       if (SrcVT != MVT::v8f16) {
32404         SDValue Tmp =
32405             IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
32406         SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
32407         Ops[0] = Src;
32408         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
32409       }
32410 
32411       if (IsStrict) {
32412         unsigned Opc =
32413             IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
32414         Res =
32415             DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});
32416         Chain = Res.getValue(1);
32417       } else {
32418         unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
32419         Res = DAG.getNode(Opc, dl, ResVT, Src);
32420       }
32421 
32422       // TODO: Need to add exception check code for strict FP.
32423       if (EleVT.getSizeInBits() < 16) {
32424         MVT TmpVT = MVT::getVectorVT(EleVT.getSimpleVT(), 8);
32425         Res = DAG.getNode(ISD::TRUNCATE, dl, TmpVT, Res);
32426 
32427         // Now widen to 128 bits.
32428         unsigned NumConcats = 128 / TmpVT.getSizeInBits();
32429         MVT ConcatVT = MVT::getVectorVT(EleVT.getSimpleVT(), 8 * NumConcats);
32430         SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(TmpVT));
32431         ConcatOps[0] = Res;
32432         Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
32433       }
32434 
32435       Results.push_back(Res);
32436       if (IsStrict)
32437         Results.push_back(Chain);
32438 
32439       return;
32440     }
32441 
32442     if (VT.isVector() && VT.getScalarSizeInBits() < 32) {
32443       assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32444              "Unexpected type action!");
32445 
32446       // Try to create a 128 bit vector, but don't exceed a 32 bit element.
32447       unsigned NewEltWidth = std::min(128 / VT.getVectorNumElements(), 32U);
32448       MVT PromoteVT = MVT::getVectorVT(MVT::getIntegerVT(NewEltWidth),
32449                                        VT.getVectorNumElements());
32450       SDValue Res;
32451       SDValue Chain;
32452       if (IsStrict) {
32453         Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {PromoteVT, MVT::Other},
32454                           {N->getOperand(0), Src});
32455         Chain = Res.getValue(1);
32456       } else
32457         Res = DAG.getNode(ISD::FP_TO_SINT, dl, PromoteVT, Src);
32458 
32459       // Preserve what we know about the size of the original result. If the
32460       // result is v2i32, we have to manually widen the assert.
32461       if (PromoteVT == MVT::v2i32)
32462         Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
32463                           DAG.getUNDEF(MVT::v2i32));
32464 
32465       Res = DAG.getNode(!IsSigned ? ISD::AssertZext : ISD::AssertSext, dl,
32466                         Res.getValueType(), Res,
32467                         DAG.getValueType(VT.getVectorElementType()));
32468 
32469       if (PromoteVT == MVT::v2i32)
32470         Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,
32471                           DAG.getIntPtrConstant(0, dl));
32472 
32473       // Truncate back to the original width.
32474       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
32475 
32476       // Now widen to 128 bits.
32477       unsigned NumConcats = 128 / VT.getSizeInBits();
32478       MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(),
32479                                       VT.getVectorNumElements() * NumConcats);
32480       SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
32481       ConcatOps[0] = Res;
32482       Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
32483       Results.push_back(Res);
32484       if (IsStrict)
32485         Results.push_back(Chain);
32486       return;
32487     }
32488 
32489 
32490     if (VT == MVT::v2i32) {
32491       assert((!IsStrict || IsSigned || Subtarget.hasAVX512()) &&
32492              "Strict unsigned conversion requires AVX512");
32493       assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32494       assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32495              "Unexpected type action!");
32496       if (Src.getValueType() == MVT::v2f64) {
32497         if (!IsSigned && !Subtarget.hasAVX512()) {
32498           SDValue Res =
32499               expandFP_TO_UINT_SSE(MVT::v4i32, Src, dl, DAG, Subtarget);
32500           Results.push_back(Res);
32501           return;
32502         }
32503 
32504         unsigned Opc;
32505         if (IsStrict)
32506           Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
32507         else
32508           Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
32509 
32510         // If we have VLX we can emit a target specific FP_TO_UINT node,.
32511         if (!IsSigned && !Subtarget.hasVLX()) {
32512           // Otherwise we can defer to the generic legalizer which will widen
32513           // the input as well. This will be further widened during op
32514           // legalization to v8i32<-v8f64.
32515           // For strict nodes we'll need to widen ourselves.
32516           // FIXME: Fix the type legalizer to safely widen strict nodes?
32517           if (!IsStrict)
32518             return;
32519           Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, Src,
32520                             DAG.getConstantFP(0.0, dl, MVT::v2f64));
32521           Opc = N->getOpcode();
32522         }
32523         SDValue Res;
32524         SDValue Chain;
32525         if (IsStrict) {
32526           Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},
32527                             {N->getOperand(0), Src});
32528           Chain = Res.getValue(1);
32529         } else {
32530           Res = DAG.getNode(Opc, dl, MVT::v4i32, Src);
32531         }
32532         Results.push_back(Res);
32533         if (IsStrict)
32534           Results.push_back(Chain);
32535         return;
32536       }
32537 
32538       // Custom widen strict v2f32->v2i32 by padding with zeros.
32539       // FIXME: Should generic type legalizer do this?
32540       if (Src.getValueType() == MVT::v2f32 && IsStrict) {
32541         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
32542                           DAG.getConstantFP(0.0, dl, MVT::v2f32));
32543         SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other},
32544                                   {N->getOperand(0), Src});
32545         Results.push_back(Res);
32546         Results.push_back(Res.getValue(1));
32547         return;
32548       }
32549 
32550       // The FP_TO_INTHelper below only handles f32/f64/f80 scalar inputs,
32551       // so early out here.
32552       return;
32553     }
32554 
32555     assert(!VT.isVector() && "Vectors should have been handled above!");
32556 
32557     if ((Subtarget.hasDQI() && VT == MVT::i64 &&
32558          (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
32559         (Subtarget.hasFP16() && SrcVT == MVT::f16)) {
32560       assert(!Subtarget.is64Bit() && "i64 should be legal");
32561       unsigned NumElts = Subtarget.hasVLX() ? 2 : 8;
32562       // If we use a 128-bit result we might need to use a target specific node.
32563       unsigned SrcElts =
32564           std::max(NumElts, 128U / (unsigned)SrcVT.getSizeInBits());
32565       MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts);
32566       MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), SrcElts);
32567       unsigned Opc = N->getOpcode();
32568       if (NumElts != SrcElts) {
32569         if (IsStrict)
32570           Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
32571         else
32572           Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
32573       }
32574 
32575       SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
32576       SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
32577                                 DAG.getConstantFP(0.0, dl, VecInVT), Src,
32578                                 ZeroIdx);
32579       SDValue Chain;
32580       if (IsStrict) {
32581         SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
32582         Res = DAG.getNode(Opc, SDLoc(N), Tys, N->getOperand(0), Res);
32583         Chain = Res.getValue(1);
32584       } else
32585         Res = DAG.getNode(Opc, SDLoc(N), VecVT, Res);
32586       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx);
32587       Results.push_back(Res);
32588       if (IsStrict)
32589         Results.push_back(Chain);
32590       return;
32591     }
32592 
32593     if (VT == MVT::i128 && Subtarget.isTargetWin64()) {
32594       SDValue Chain;
32595       SDValue V = LowerWin64_FP_TO_INT128(SDValue(N, 0), DAG, Chain);
32596       Results.push_back(V);
32597       if (IsStrict)
32598         Results.push_back(Chain);
32599       return;
32600     }
32601 
32602     if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, Chain)) {
32603       Results.push_back(V);
32604       if (IsStrict)
32605         Results.push_back(Chain);
32606     }
32607     return;
32608   }
32609   case ISD::LRINT:
32610   case ISD::LLRINT: {
32611     if (SDValue V = LRINT_LLRINTHelper(N, DAG))
32612       Results.push_back(V);
32613     return;
32614   }
32615 
32616   case ISD::SINT_TO_FP:
32617   case ISD::STRICT_SINT_TO_FP:
32618   case ISD::UINT_TO_FP:
32619   case ISD::STRICT_UINT_TO_FP: {
32620     bool IsStrict = N->isStrictFPOpcode();
32621     bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
32622                     N->getOpcode() == ISD::STRICT_SINT_TO_FP;
32623     EVT VT = N->getValueType(0);
32624     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32625     if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() &&
32626         Subtarget.hasVLX()) {
32627       if (Src.getValueType().getVectorElementType() == MVT::i16)
32628         return;
32629 
32630       if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2i32)
32631         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
32632                           IsStrict ? DAG.getConstant(0, dl, MVT::v2i32)
32633                                    : DAG.getUNDEF(MVT::v2i32));
32634       if (IsStrict) {
32635         unsigned Opc =
32636             IsSigned ? X86ISD::STRICT_CVTSI2P : X86ISD::STRICT_CVTUI2P;
32637         SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
32638                                   {N->getOperand(0), Src});
32639         Results.push_back(Res);
32640         Results.push_back(Res.getValue(1));
32641       } else {
32642         unsigned Opc = IsSigned ? X86ISD::CVTSI2P : X86ISD::CVTUI2P;
32643         Results.push_back(DAG.getNode(Opc, dl, MVT::v8f16, Src));
32644       }
32645       return;
32646     }
32647     if (VT != MVT::v2f32)
32648       return;
32649     EVT SrcVT = Src.getValueType();
32650     if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) {
32651       if (IsStrict) {
32652         unsigned Opc = IsSigned ? X86ISD::STRICT_CVTSI2P
32653                                 : X86ISD::STRICT_CVTUI2P;
32654         SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
32655                                   {N->getOperand(0), Src});
32656         Results.push_back(Res);
32657         Results.push_back(Res.getValue(1));
32658       } else {
32659         unsigned Opc = IsSigned ? X86ISD::CVTSI2P : X86ISD::CVTUI2P;
32660         Results.push_back(DAG.getNode(Opc, dl, MVT::v4f32, Src));
32661       }
32662       return;
32663     }
32664     if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() &&
32665         Subtarget.hasSSE41() && !Subtarget.hasAVX512()) {
32666       SDValue Zero = DAG.getConstant(0, dl, SrcVT);
32667       SDValue One  = DAG.getConstant(1, dl, SrcVT);
32668       SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
32669                                  DAG.getNode(ISD::SRL, dl, SrcVT, Src, One),
32670                                  DAG.getNode(ISD::AND, dl, SrcVT, Src, One));
32671       SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT);
32672       SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src);
32673       SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32));
32674       for (int i = 0; i != 2; ++i) {
32675         SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
32676                                   SignSrc, DAG.getIntPtrConstant(i, dl));
32677         if (IsStrict)
32678           SignCvts[i] =
32679               DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {MVT::f32, MVT::Other},
32680                           {N->getOperand(0), Elt});
32681         else
32682           SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Elt);
32683       };
32684       SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts);
32685       SDValue Slow, Chain;
32686       if (IsStrict) {
32687         Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
32688                             SignCvts[0].getValue(1), SignCvts[1].getValue(1));
32689         Slow = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v4f32, MVT::Other},
32690                            {Chain, SignCvt, SignCvt});
32691         Chain = Slow.getValue(1);
32692       } else {
32693         Slow = DAG.getNode(ISD::FADD, dl, MVT::v4f32, SignCvt, SignCvt);
32694       }
32695       IsNeg = DAG.getBitcast(MVT::v4i32, IsNeg);
32696       IsNeg =
32697           DAG.getVectorShuffle(MVT::v4i32, dl, IsNeg, IsNeg, {1, 3, -1, -1});
32698       SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt);
32699       Results.push_back(Cvt);
32700       if (IsStrict)
32701         Results.push_back(Chain);
32702       return;
32703     }
32704 
32705     if (SrcVT != MVT::v2i32)
32706       return;
32707 
32708     if (IsSigned || Subtarget.hasAVX512()) {
32709       if (!IsStrict)
32710         return;
32711 
32712       // Custom widen strict v2i32->v2f32 to avoid scalarization.
32713       // FIXME: Should generic type legalizer do this?
32714       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
32715                         DAG.getConstant(0, dl, MVT::v2i32));
32716       SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
32717                                 {N->getOperand(0), Src});
32718       Results.push_back(Res);
32719       Results.push_back(Res.getValue(1));
32720       return;
32721     }
32722 
32723     assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32724     SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);
32725     SDValue VBias = DAG.getConstantFP(
32726         llvm::bit_cast<double>(0x4330000000000000ULL), dl, MVT::v2f64);
32727     SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
32728                              DAG.getBitcast(MVT::v2i64, VBias));
32729     Or = DAG.getBitcast(MVT::v2f64, Or);
32730     if (IsStrict) {
32731       SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
32732                                 {N->getOperand(0), Or, VBias});
32733       SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
32734                                 {MVT::v4f32, MVT::Other},
32735                                 {Sub.getValue(1), Sub});
32736       Results.push_back(Res);
32737       Results.push_back(Res.getValue(1));
32738     } else {
32739       // TODO: Are there any fast-math-flags to propagate here?
32740       SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
32741       Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
32742     }
32743     return;
32744   }
32745   case ISD::STRICT_FP_ROUND:
32746   case ISD::FP_ROUND: {
32747     bool IsStrict = N->isStrictFPOpcode();
32748     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
32749     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32750     SDValue Rnd = N->getOperand(IsStrict ? 2 : 1);
32751     EVT SrcVT = Src.getValueType();
32752     EVT VT = N->getValueType(0);
32753     SDValue V;
32754     if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2f32) {
32755       SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f32)
32756                              : DAG.getUNDEF(MVT::v2f32);
32757       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, Ext);
32758     }
32759     if (!Subtarget.hasFP16() && VT.getVectorElementType() == MVT::f16) {
32760       assert(Subtarget.hasF16C() && "Cannot widen f16 without F16C");
32761       if (SrcVT.getVectorElementType() != MVT::f32)
32762         return;
32763 
32764       if (IsStrict)
32765         V = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},
32766                         {Chain, Src, Rnd});
32767       else
32768         V = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Src, Rnd);
32769 
32770       Results.push_back(DAG.getBitcast(MVT::v8f16, V));
32771       if (IsStrict)
32772         Results.push_back(V.getValue(1));
32773       return;
32774     }
32775     if (!isTypeLegal(Src.getValueType()))
32776       return;
32777     EVT NewVT = VT.getVectorElementType() == MVT::f16 ? MVT::v8f16 : MVT::v4f32;
32778     if (IsStrict)
32779       V = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {NewVT, MVT::Other},
32780                       {Chain, Src});
32781     else
32782       V = DAG.getNode(X86ISD::VFPROUND, dl, NewVT, Src);
32783     Results.push_back(V);
32784     if (IsStrict)
32785       Results.push_back(V.getValue(1));
32786     return;
32787   }
32788   case ISD::FP_EXTEND:
32789   case ISD::STRICT_FP_EXTEND: {
32790     // Right now, only MVT::v2f32 has OperationAction for FP_EXTEND.
32791     // No other ValueType for FP_EXTEND should reach this point.
32792     assert(N->getValueType(0) == MVT::v2f32 &&
32793            "Do not know how to legalize this Node");
32794     if (!Subtarget.hasFP16() || !Subtarget.hasVLX())
32795       return;
32796     bool IsStrict = N->isStrictFPOpcode();
32797     SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32798     SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f16)
32799                            : DAG.getUNDEF(MVT::v2f16);
32800     SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f16, Src, Ext);
32801     if (IsStrict)
32802       V = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::v4f32, MVT::Other},
32803                       {N->getOperand(0), V});
32804     else
32805       V = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, V);
32806     Results.push_back(V);
32807     if (IsStrict)
32808       Results.push_back(V.getValue(1));
32809     return;
32810   }
32811   case ISD::INTRINSIC_W_CHAIN: {
32812     unsigned IntNo = N->getConstantOperandVal(1);
32813     switch (IntNo) {
32814     default : llvm_unreachable("Do not know how to custom type "
32815                                "legalize this intrinsic operation!");
32816     case Intrinsic::x86_rdtsc:
32817       return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget,
32818                                      Results);
32819     case Intrinsic::x86_rdtscp:
32820       return getReadTimeStampCounter(N, dl, X86::RDTSCP, DAG, Subtarget,
32821                                      Results);
32822     case Intrinsic::x86_rdpmc:
32823       expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPMC, X86::ECX, Subtarget,
32824                                   Results);
32825       return;
32826     case Intrinsic::x86_rdpru:
32827       expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget,
32828         Results);
32829       return;
32830     case Intrinsic::x86_xgetbv:
32831       expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget,
32832                                   Results);
32833       return;
32834     }
32835   }
32836   case ISD::READCYCLECOUNTER: {
32837     return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, Results);
32838   }
32839   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
32840     EVT T = N->getValueType(0);
32841     assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair");
32842     bool Regs64bit = T == MVT::i128;
32843     assert((!Regs64bit || Subtarget.canUseCMPXCHG16B()) &&
32844            "64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B");
32845     MVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;
32846     SDValue cpInL, cpInH;
32847     std::tie(cpInL, cpInH) =
32848         DAG.SplitScalar(N->getOperand(2), dl, HalfT, HalfT);
32849     cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
32850                              Regs64bit ? X86::RAX : X86::EAX, cpInL, SDValue());
32851     cpInH =
32852         DAG.getCopyToReg(cpInL.getValue(0), dl, Regs64bit ? X86::RDX : X86::EDX,
32853                          cpInH, cpInL.getValue(1));
32854     SDValue swapInL, swapInH;
32855     std::tie(swapInL, swapInH) =
32856         DAG.SplitScalar(N->getOperand(3), dl, HalfT, HalfT);
32857     swapInH =
32858         DAG.getCopyToReg(cpInH.getValue(0), dl, Regs64bit ? X86::RCX : X86::ECX,
32859                          swapInH, cpInH.getValue(1));
32860 
32861     // In 64-bit mode we might need the base pointer in RBX, but we can't know
32862     // until later. So we keep the RBX input in a vreg and use a custom
32863     // inserter.
32864     // Since RBX will be a reserved register the register allocator will not
32865     // make sure its value will be properly saved and restored around this
32866     // live-range.
32867     SDValue Result;
32868     SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
32869     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
32870     if (Regs64bit) {
32871       SDValue Ops[] = {swapInH.getValue(0), N->getOperand(1), swapInL,
32872                        swapInH.getValue(1)};
32873       Result =
32874           DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG16_DAG, dl, Tys, Ops, T, MMO);
32875     } else {
32876       swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl, X86::EBX, swapInL,
32877                                  swapInH.getValue(1));
32878       SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1),
32879                        swapInL.getValue(1)};
32880       Result =
32881           DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, T, MMO);
32882     }
32883 
32884     SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
32885                                         Regs64bit ? X86::RAX : X86::EAX,
32886                                         HalfT, Result.getValue(1));
32887     SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
32888                                         Regs64bit ? X86::RDX : X86::EDX,
32889                                         HalfT, cpOutL.getValue(2));
32890     SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
32891 
32892     SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
32893                                         MVT::i32, cpOutH.getValue(2));
32894     SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
32895     Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1));
32896 
32897     Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF));
32898     Results.push_back(Success);
32899     Results.push_back(EFLAGS.getValue(1));
32900     return;
32901   }
32902   case ISD::ATOMIC_LOAD: {
32903     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
32904     bool NoImplicitFloatOps =
32905         DAG.getMachineFunction().getFunction().hasFnAttribute(
32906             Attribute::NoImplicitFloat);
32907     if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
32908       auto *Node = cast<AtomicSDNode>(N);
32909       if (Subtarget.hasSSE1()) {
32910         // Use a VZEXT_LOAD which will be selected as MOVQ or XORPS+MOVLPS.
32911         // Then extract the lower 64-bits.
32912         MVT LdVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
32913         SDVTList Tys = DAG.getVTList(LdVT, MVT::Other);
32914         SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
32915         SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
32916                                              MVT::i64, Node->getMemOperand());
32917         if (Subtarget.hasSSE2()) {
32918           SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
32919                                     DAG.getIntPtrConstant(0, dl));
32920           Results.push_back(Res);
32921           Results.push_back(Ld.getValue(1));
32922           return;
32923         }
32924         // We use an alternative sequence for SSE1 that extracts as v2f32 and
32925         // then casts to i64. This avoids a 128-bit stack temporary being
32926         // created by type legalization if we were to cast v4f32->v2i64.
32927         SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld,
32928                                   DAG.getIntPtrConstant(0, dl));
32929         Res = DAG.getBitcast(MVT::i64, Res);
32930         Results.push_back(Res);
32931         Results.push_back(Ld.getValue(1));
32932         return;
32933       }
32934       if (Subtarget.hasX87()) {
32935         // First load this into an 80-bit X87 register. This will put the whole
32936         // integer into the significand.
32937         SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
32938         SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
32939         SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD,
32940                                                  dl, Tys, Ops, MVT::i64,
32941                                                  Node->getMemOperand());
32942         SDValue Chain = Result.getValue(1);
32943 
32944         // Now store the X87 register to a stack temporary and convert to i64.
32945         // This store is not atomic and doesn't need to be.
32946         // FIXME: We don't need a stack temporary if the result of the load
32947         // is already being stored. We could just directly store there.
32948         SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
32949         int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
32950         MachinePointerInfo MPI =
32951             MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
32952         SDValue StoreOps[] = { Chain, Result, StackPtr };
32953         Chain = DAG.getMemIntrinsicNode(
32954             X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
32955             MPI, std::nullopt /*Align*/, MachineMemOperand::MOStore);
32956 
32957         // Finally load the value back from the stack temporary and return it.
32958         // This load is not atomic and doesn't need to be.
32959         // This load will be further type legalized.
32960         Result = DAG.getLoad(MVT::i64, dl, Chain, StackPtr, MPI);
32961         Results.push_back(Result);
32962         Results.push_back(Result.getValue(1));
32963         return;
32964       }
32965     }
32966     // TODO: Use MOVLPS when SSE1 is available?
32967     // Delegate to generic TypeLegalization. Situations we can really handle
32968     // should have already been dealt with by AtomicExpandPass.cpp.
32969     break;
32970   }
32971   case ISD::ATOMIC_SWAP:
32972   case ISD::ATOMIC_LOAD_ADD:
32973   case ISD::ATOMIC_LOAD_SUB:
32974   case ISD::ATOMIC_LOAD_AND:
32975   case ISD::ATOMIC_LOAD_OR:
32976   case ISD::ATOMIC_LOAD_XOR:
32977   case ISD::ATOMIC_LOAD_NAND:
32978   case ISD::ATOMIC_LOAD_MIN:
32979   case ISD::ATOMIC_LOAD_MAX:
32980   case ISD::ATOMIC_LOAD_UMIN:
32981   case ISD::ATOMIC_LOAD_UMAX:
32982     // Delegate to generic TypeLegalization. Situations we can really handle
32983     // should have already been dealt with by AtomicExpandPass.cpp.
32984     break;
32985 
32986   case ISD::BITCAST: {
32987     assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32988     EVT DstVT = N->getValueType(0);
32989     EVT SrcVT = N->getOperand(0).getValueType();
32990 
32991     // If this is a bitcast from a v64i1 k-register to a i64 on a 32-bit target
32992     // we can split using the k-register rather than memory.
32993     if (SrcVT == MVT::v64i1 && DstVT == MVT::i64 && Subtarget.hasBWI()) {
32994       assert(!Subtarget.is64Bit() && "Expected 32-bit mode");
32995       SDValue Lo, Hi;
32996       std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
32997       Lo = DAG.getBitcast(MVT::i32, Lo);
32998       Hi = DAG.getBitcast(MVT::i32, Hi);
32999       SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
33000       Results.push_back(Res);
33001       return;
33002     }
33003 
33004     if (DstVT.isVector() && SrcVT == MVT::x86mmx) {
33005       // FIXME: Use v4f32 for SSE1?
33006       assert(Subtarget.hasSSE2() && "Requires SSE2");
33007       assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
33008              "Unexpected type action!");
33009       EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT);
33010       SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,
33011                                 N->getOperand(0));
33012       Res = DAG.getBitcast(WideVT, Res);
33013       Results.push_back(Res);
33014       return;
33015     }
33016 
33017     return;
33018   }
33019   case ISD::MGATHER: {
33020     EVT VT = N->getValueType(0);
33021     if ((VT == MVT::v2f32 || VT == MVT::v2i32) &&
33022         (Subtarget.hasVLX() || !Subtarget.hasAVX512())) {
33023       auto *Gather = cast<MaskedGatherSDNode>(N);
33024       SDValue Index = Gather->getIndex();
33025       if (Index.getValueType() != MVT::v2i64)
33026         return;
33027       assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33028              "Unexpected type action!");
33029       EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
33030       SDValue Mask = Gather->getMask();
33031       assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
33032       SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
33033                                      Gather->getPassThru(),
33034                                      DAG.getUNDEF(VT));
33035       if (!Subtarget.hasVLX()) {
33036         // We need to widen the mask, but the instruction will only use 2
33037         // of its elements. So we can use undef.
33038         Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,
33039                            DAG.getUNDEF(MVT::v2i1));
33040         Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);
33041       }
33042       SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
33043                         Gather->getBasePtr(), Index, Gather->getScale() };
33044       SDValue Res = DAG.getMemIntrinsicNode(
33045           X86ISD::MGATHER, dl, DAG.getVTList(WideVT, MVT::Other), Ops,
33046           Gather->getMemoryVT(), Gather->getMemOperand());
33047       Results.push_back(Res);
33048       Results.push_back(Res.getValue(1));
33049       return;
33050     }
33051     return;
33052   }
33053   case ISD::LOAD: {
33054     // Use an f64/i64 load and a scalar_to_vector for v2f32/v2i32 loads. This
33055     // avoids scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
33056     // cast since type legalization will try to use an i64 load.
33057     MVT VT = N->getSimpleValueType(0);
33058     assert(VT.isVector() && VT.getSizeInBits() == 64 && "Unexpected VT");
33059     assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33060            "Unexpected type action!");
33061     if (!ISD::isNON_EXTLoad(N))
33062       return;
33063     auto *Ld = cast<LoadSDNode>(N);
33064     if (Subtarget.hasSSE2()) {
33065       MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64;
33066       SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(),
33067                                 Ld->getPointerInfo(), Ld->getOriginalAlign(),
33068                                 Ld->getMemOperand()->getFlags());
33069       SDValue Chain = Res.getValue(1);
33070       MVT VecVT = MVT::getVectorVT(LdVT, 2);
33071       Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Res);
33072       EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
33073       Res = DAG.getBitcast(WideVT, Res);
33074       Results.push_back(Res);
33075       Results.push_back(Chain);
33076       return;
33077     }
33078     assert(Subtarget.hasSSE1() && "Expected SSE");
33079     SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other);
33080     SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()};
33081     SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
33082                                           MVT::i64, Ld->getMemOperand());
33083     Results.push_back(Res);
33084     Results.push_back(Res.getValue(1));
33085     return;
33086   }
33087   case ISD::ADDRSPACECAST: {
33088     SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG);
33089     Results.push_back(V);
33090     return;
33091   }
33092   case ISD::BITREVERSE: {
33093     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
33094     assert(Subtarget.hasXOP() && "Expected XOP");
33095     // We can use VPPERM by copying to a vector register and back. We'll need
33096     // to move the scalar in two i32 pieces.
33097     Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG));
33098     return;
33099   }
33100   case ISD::EXTRACT_VECTOR_ELT: {
33101     // f16 = extract vXf16 %vec, i64 %idx
33102     assert(N->getSimpleValueType(0) == MVT::f16 &&
33103            "Unexpected Value type of EXTRACT_VECTOR_ELT!");
33104     assert(Subtarget.hasFP16() && "Expected FP16");
33105     SDValue VecOp = N->getOperand(0);
33106     EVT ExtVT = VecOp.getValueType().changeVectorElementTypeToInteger();
33107     SDValue Split = DAG.getBitcast(ExtVT, N->getOperand(0));
33108     Split = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Split,
33109                         N->getOperand(1));
33110     Split = DAG.getBitcast(MVT::f16, Split);
33111     Results.push_back(Split);
33112     return;
33113   }
33114   }
33115 }
33116 
33117 const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
33118   switch ((X86ISD::NodeType)Opcode) {
33119   case X86ISD::FIRST_NUMBER:       break;
33120 #define NODE_NAME_CASE(NODE) case X86ISD::NODE: return "X86ISD::" #NODE;
33121   NODE_NAME_CASE(BSF)
33122   NODE_NAME_CASE(BSR)
33123   NODE_NAME_CASE(FSHL)
33124   NODE_NAME_CASE(FSHR)
33125   NODE_NAME_CASE(FAND)
33126   NODE_NAME_CASE(FANDN)
33127   NODE_NAME_CASE(FOR)
33128   NODE_NAME_CASE(FXOR)
33129   NODE_NAME_CASE(FILD)
33130   NODE_NAME_CASE(FIST)
33131   NODE_NAME_CASE(FP_TO_INT_IN_MEM)
33132   NODE_NAME_CASE(FLD)
33133   NODE_NAME_CASE(FST)
33134   NODE_NAME_CASE(CALL)
33135   NODE_NAME_CASE(CALL_RVMARKER)
33136   NODE_NAME_CASE(BT)
33137   NODE_NAME_CASE(CMP)
33138   NODE_NAME_CASE(FCMP)
33139   NODE_NAME_CASE(STRICT_FCMP)
33140   NODE_NAME_CASE(STRICT_FCMPS)
33141   NODE_NAME_CASE(COMI)
33142   NODE_NAME_CASE(UCOMI)
33143   NODE_NAME_CASE(CMPM)
33144   NODE_NAME_CASE(CMPMM)
33145   NODE_NAME_CASE(STRICT_CMPM)
33146   NODE_NAME_CASE(CMPMM_SAE)
33147   NODE_NAME_CASE(SETCC)
33148   NODE_NAME_CASE(SETCC_CARRY)
33149   NODE_NAME_CASE(FSETCC)
33150   NODE_NAME_CASE(FSETCCM)
33151   NODE_NAME_CASE(FSETCCM_SAE)
33152   NODE_NAME_CASE(CMOV)
33153   NODE_NAME_CASE(BRCOND)
33154   NODE_NAME_CASE(RET_GLUE)
33155   NODE_NAME_CASE(IRET)
33156   NODE_NAME_CASE(REP_STOS)
33157   NODE_NAME_CASE(REP_MOVS)
33158   NODE_NAME_CASE(GlobalBaseReg)
33159   NODE_NAME_CASE(Wrapper)
33160   NODE_NAME_CASE(WrapperRIP)
33161   NODE_NAME_CASE(MOVQ2DQ)
33162   NODE_NAME_CASE(MOVDQ2Q)
33163   NODE_NAME_CASE(MMX_MOVD2W)
33164   NODE_NAME_CASE(MMX_MOVW2D)
33165   NODE_NAME_CASE(PEXTRB)
33166   NODE_NAME_CASE(PEXTRW)
33167   NODE_NAME_CASE(INSERTPS)
33168   NODE_NAME_CASE(PINSRB)
33169   NODE_NAME_CASE(PINSRW)
33170   NODE_NAME_CASE(PSHUFB)
33171   NODE_NAME_CASE(ANDNP)
33172   NODE_NAME_CASE(BLENDI)
33173   NODE_NAME_CASE(BLENDV)
33174   NODE_NAME_CASE(HADD)
33175   NODE_NAME_CASE(HSUB)
33176   NODE_NAME_CASE(FHADD)
33177   NODE_NAME_CASE(FHSUB)
33178   NODE_NAME_CASE(CONFLICT)
33179   NODE_NAME_CASE(FMAX)
33180   NODE_NAME_CASE(FMAXS)
33181   NODE_NAME_CASE(FMAX_SAE)
33182   NODE_NAME_CASE(FMAXS_SAE)
33183   NODE_NAME_CASE(FMIN)
33184   NODE_NAME_CASE(FMINS)
33185   NODE_NAME_CASE(FMIN_SAE)
33186   NODE_NAME_CASE(FMINS_SAE)
33187   NODE_NAME_CASE(FMAXC)
33188   NODE_NAME_CASE(FMINC)
33189   NODE_NAME_CASE(FRSQRT)
33190   NODE_NAME_CASE(FRCP)
33191   NODE_NAME_CASE(EXTRQI)
33192   NODE_NAME_CASE(INSERTQI)
33193   NODE_NAME_CASE(TLSADDR)
33194   NODE_NAME_CASE(TLSBASEADDR)
33195   NODE_NAME_CASE(TLSCALL)
33196   NODE_NAME_CASE(EH_SJLJ_SETJMP)
33197   NODE_NAME_CASE(EH_SJLJ_LONGJMP)
33198   NODE_NAME_CASE(EH_SJLJ_SETUP_DISPATCH)
33199   NODE_NAME_CASE(EH_RETURN)
33200   NODE_NAME_CASE(TC_RETURN)
33201   NODE_NAME_CASE(FNSTCW16m)
33202   NODE_NAME_CASE(FLDCW16m)
33203   NODE_NAME_CASE(FNSTENVm)
33204   NODE_NAME_CASE(FLDENVm)
33205   NODE_NAME_CASE(LCMPXCHG_DAG)
33206   NODE_NAME_CASE(LCMPXCHG8_DAG)
33207   NODE_NAME_CASE(LCMPXCHG16_DAG)
33208   NODE_NAME_CASE(LCMPXCHG16_SAVE_RBX_DAG)
33209   NODE_NAME_CASE(LADD)
33210   NODE_NAME_CASE(LSUB)
33211   NODE_NAME_CASE(LOR)
33212   NODE_NAME_CASE(LXOR)
33213   NODE_NAME_CASE(LAND)
33214   NODE_NAME_CASE(LBTS)
33215   NODE_NAME_CASE(LBTC)
33216   NODE_NAME_CASE(LBTR)
33217   NODE_NAME_CASE(LBTS_RM)
33218   NODE_NAME_CASE(LBTC_RM)
33219   NODE_NAME_CASE(LBTR_RM)
33220   NODE_NAME_CASE(AADD)
33221   NODE_NAME_CASE(AOR)
33222   NODE_NAME_CASE(AXOR)
33223   NODE_NAME_CASE(AAND)
33224   NODE_NAME_CASE(VZEXT_MOVL)
33225   NODE_NAME_CASE(VZEXT_LOAD)
33226   NODE_NAME_CASE(VEXTRACT_STORE)
33227   NODE_NAME_CASE(VTRUNC)
33228   NODE_NAME_CASE(VTRUNCS)
33229   NODE_NAME_CASE(VTRUNCUS)
33230   NODE_NAME_CASE(VMTRUNC)
33231   NODE_NAME_CASE(VMTRUNCS)
33232   NODE_NAME_CASE(VMTRUNCUS)
33233   NODE_NAME_CASE(VTRUNCSTORES)
33234   NODE_NAME_CASE(VTRUNCSTOREUS)
33235   NODE_NAME_CASE(VMTRUNCSTORES)
33236   NODE_NAME_CASE(VMTRUNCSTOREUS)
33237   NODE_NAME_CASE(VFPEXT)
33238   NODE_NAME_CASE(STRICT_VFPEXT)
33239   NODE_NAME_CASE(VFPEXT_SAE)
33240   NODE_NAME_CASE(VFPEXTS)
33241   NODE_NAME_CASE(VFPEXTS_SAE)
33242   NODE_NAME_CASE(VFPROUND)
33243   NODE_NAME_CASE(STRICT_VFPROUND)
33244   NODE_NAME_CASE(VMFPROUND)
33245   NODE_NAME_CASE(VFPROUND_RND)
33246   NODE_NAME_CASE(VFPROUNDS)
33247   NODE_NAME_CASE(VFPROUNDS_RND)
33248   NODE_NAME_CASE(VSHLDQ)
33249   NODE_NAME_CASE(VSRLDQ)
33250   NODE_NAME_CASE(VSHL)
33251   NODE_NAME_CASE(VSRL)
33252   NODE_NAME_CASE(VSRA)
33253   NODE_NAME_CASE(VSHLI)
33254   NODE_NAME_CASE(VSRLI)
33255   NODE_NAME_CASE(VSRAI)
33256   NODE_NAME_CASE(VSHLV)
33257   NODE_NAME_CASE(VSRLV)
33258   NODE_NAME_CASE(VSRAV)
33259   NODE_NAME_CASE(VROTLI)
33260   NODE_NAME_CASE(VROTRI)
33261   NODE_NAME_CASE(VPPERM)
33262   NODE_NAME_CASE(CMPP)
33263   NODE_NAME_CASE(STRICT_CMPP)
33264   NODE_NAME_CASE(PCMPEQ)
33265   NODE_NAME_CASE(PCMPGT)
33266   NODE_NAME_CASE(PHMINPOS)
33267   NODE_NAME_CASE(ADD)
33268   NODE_NAME_CASE(SUB)
33269   NODE_NAME_CASE(ADC)
33270   NODE_NAME_CASE(SBB)
33271   NODE_NAME_CASE(SMUL)
33272   NODE_NAME_CASE(UMUL)
33273   NODE_NAME_CASE(OR)
33274   NODE_NAME_CASE(XOR)
33275   NODE_NAME_CASE(AND)
33276   NODE_NAME_CASE(BEXTR)
33277   NODE_NAME_CASE(BEXTRI)
33278   NODE_NAME_CASE(BZHI)
33279   NODE_NAME_CASE(PDEP)
33280   NODE_NAME_CASE(PEXT)
33281   NODE_NAME_CASE(MUL_IMM)
33282   NODE_NAME_CASE(MOVMSK)
33283   NODE_NAME_CASE(PTEST)
33284   NODE_NAME_CASE(TESTP)
33285   NODE_NAME_CASE(KORTEST)
33286   NODE_NAME_CASE(KTEST)
33287   NODE_NAME_CASE(KADD)
33288   NODE_NAME_CASE(KSHIFTL)
33289   NODE_NAME_CASE(KSHIFTR)
33290   NODE_NAME_CASE(PACKSS)
33291   NODE_NAME_CASE(PACKUS)
33292   NODE_NAME_CASE(PALIGNR)
33293   NODE_NAME_CASE(VALIGN)
33294   NODE_NAME_CASE(VSHLD)
33295   NODE_NAME_CASE(VSHRD)
33296   NODE_NAME_CASE(VSHLDV)
33297   NODE_NAME_CASE(VSHRDV)
33298   NODE_NAME_CASE(PSHUFD)
33299   NODE_NAME_CASE(PSHUFHW)
33300   NODE_NAME_CASE(PSHUFLW)
33301   NODE_NAME_CASE(SHUFP)
33302   NODE_NAME_CASE(SHUF128)
33303   NODE_NAME_CASE(MOVLHPS)
33304   NODE_NAME_CASE(MOVHLPS)
33305   NODE_NAME_CASE(MOVDDUP)
33306   NODE_NAME_CASE(MOVSHDUP)
33307   NODE_NAME_CASE(MOVSLDUP)
33308   NODE_NAME_CASE(MOVSD)
33309   NODE_NAME_CASE(MOVSS)
33310   NODE_NAME_CASE(MOVSH)
33311   NODE_NAME_CASE(UNPCKL)
33312   NODE_NAME_CASE(UNPCKH)
33313   NODE_NAME_CASE(VBROADCAST)
33314   NODE_NAME_CASE(VBROADCAST_LOAD)
33315   NODE_NAME_CASE(VBROADCASTM)
33316   NODE_NAME_CASE(SUBV_BROADCAST_LOAD)
33317   NODE_NAME_CASE(VPERMILPV)
33318   NODE_NAME_CASE(VPERMILPI)
33319   NODE_NAME_CASE(VPERM2X128)
33320   NODE_NAME_CASE(VPERMV)
33321   NODE_NAME_CASE(VPERMV3)
33322   NODE_NAME_CASE(VPERMI)
33323   NODE_NAME_CASE(VPTERNLOG)
33324   NODE_NAME_CASE(VFIXUPIMM)
33325   NODE_NAME_CASE(VFIXUPIMM_SAE)
33326   NODE_NAME_CASE(VFIXUPIMMS)
33327   NODE_NAME_CASE(VFIXUPIMMS_SAE)
33328   NODE_NAME_CASE(VRANGE)
33329   NODE_NAME_CASE(VRANGE_SAE)
33330   NODE_NAME_CASE(VRANGES)
33331   NODE_NAME_CASE(VRANGES_SAE)
33332   NODE_NAME_CASE(PMULUDQ)
33333   NODE_NAME_CASE(PMULDQ)
33334   NODE_NAME_CASE(PSADBW)
33335   NODE_NAME_CASE(DBPSADBW)
33336   NODE_NAME_CASE(VASTART_SAVE_XMM_REGS)
33337   NODE_NAME_CASE(VAARG_64)
33338   NODE_NAME_CASE(VAARG_X32)
33339   NODE_NAME_CASE(DYN_ALLOCA)
33340   NODE_NAME_CASE(MFENCE)
33341   NODE_NAME_CASE(SEG_ALLOCA)
33342   NODE_NAME_CASE(PROBED_ALLOCA)
33343   NODE_NAME_CASE(RDRAND)
33344   NODE_NAME_CASE(RDSEED)
33345   NODE_NAME_CASE(RDPKRU)
33346   NODE_NAME_CASE(WRPKRU)
33347   NODE_NAME_CASE(VPMADDUBSW)
33348   NODE_NAME_CASE(VPMADDWD)
33349   NODE_NAME_CASE(VPSHA)
33350   NODE_NAME_CASE(VPSHL)
33351   NODE_NAME_CASE(VPCOM)
33352   NODE_NAME_CASE(VPCOMU)
33353   NODE_NAME_CASE(VPERMIL2)
33354   NODE_NAME_CASE(FMSUB)
33355   NODE_NAME_CASE(STRICT_FMSUB)
33356   NODE_NAME_CASE(FNMADD)
33357   NODE_NAME_CASE(STRICT_FNMADD)
33358   NODE_NAME_CASE(FNMSUB)
33359   NODE_NAME_CASE(STRICT_FNMSUB)
33360   NODE_NAME_CASE(FMADDSUB)
33361   NODE_NAME_CASE(FMSUBADD)
33362   NODE_NAME_CASE(FMADD_RND)
33363   NODE_NAME_CASE(FNMADD_RND)
33364   NODE_NAME_CASE(FMSUB_RND)
33365   NODE_NAME_CASE(FNMSUB_RND)
33366   NODE_NAME_CASE(FMADDSUB_RND)
33367   NODE_NAME_CASE(FMSUBADD_RND)
33368   NODE_NAME_CASE(VFMADDC)
33369   NODE_NAME_CASE(VFMADDC_RND)
33370   NODE_NAME_CASE(VFCMADDC)
33371   NODE_NAME_CASE(VFCMADDC_RND)
33372   NODE_NAME_CASE(VFMULC)
33373   NODE_NAME_CASE(VFMULC_RND)
33374   NODE_NAME_CASE(VFCMULC)
33375   NODE_NAME_CASE(VFCMULC_RND)
33376   NODE_NAME_CASE(VFMULCSH)
33377   NODE_NAME_CASE(VFMULCSH_RND)
33378   NODE_NAME_CASE(VFCMULCSH)
33379   NODE_NAME_CASE(VFCMULCSH_RND)
33380   NODE_NAME_CASE(VFMADDCSH)
33381   NODE_NAME_CASE(VFMADDCSH_RND)
33382   NODE_NAME_CASE(VFCMADDCSH)
33383   NODE_NAME_CASE(VFCMADDCSH_RND)
33384   NODE_NAME_CASE(VPMADD52H)
33385   NODE_NAME_CASE(VPMADD52L)
33386   NODE_NAME_CASE(VRNDSCALE)
33387   NODE_NAME_CASE(STRICT_VRNDSCALE)
33388   NODE_NAME_CASE(VRNDSCALE_SAE)
33389   NODE_NAME_CASE(VRNDSCALES)
33390   NODE_NAME_CASE(VRNDSCALES_SAE)
33391   NODE_NAME_CASE(VREDUCE)
33392   NODE_NAME_CASE(VREDUCE_SAE)
33393   NODE_NAME_CASE(VREDUCES)
33394   NODE_NAME_CASE(VREDUCES_SAE)
33395   NODE_NAME_CASE(VGETMANT)
33396   NODE_NAME_CASE(VGETMANT_SAE)
33397   NODE_NAME_CASE(VGETMANTS)
33398   NODE_NAME_CASE(VGETMANTS_SAE)
33399   NODE_NAME_CASE(PCMPESTR)
33400   NODE_NAME_CASE(PCMPISTR)
33401   NODE_NAME_CASE(XTEST)
33402   NODE_NAME_CASE(COMPRESS)
33403   NODE_NAME_CASE(EXPAND)
33404   NODE_NAME_CASE(SELECTS)
33405   NODE_NAME_CASE(ADDSUB)
33406   NODE_NAME_CASE(RCP14)
33407   NODE_NAME_CASE(RCP14S)
33408   NODE_NAME_CASE(RCP28)
33409   NODE_NAME_CASE(RCP28_SAE)
33410   NODE_NAME_CASE(RCP28S)
33411   NODE_NAME_CASE(RCP28S_SAE)
33412   NODE_NAME_CASE(EXP2)
33413   NODE_NAME_CASE(EXP2_SAE)
33414   NODE_NAME_CASE(RSQRT14)
33415   NODE_NAME_CASE(RSQRT14S)
33416   NODE_NAME_CASE(RSQRT28)
33417   NODE_NAME_CASE(RSQRT28_SAE)
33418   NODE_NAME_CASE(RSQRT28S)
33419   NODE_NAME_CASE(RSQRT28S_SAE)
33420   NODE_NAME_CASE(FADD_RND)
33421   NODE_NAME_CASE(FADDS)
33422   NODE_NAME_CASE(FADDS_RND)
33423   NODE_NAME_CASE(FSUB_RND)
33424   NODE_NAME_CASE(FSUBS)
33425   NODE_NAME_CASE(FSUBS_RND)
33426   NODE_NAME_CASE(FMUL_RND)
33427   NODE_NAME_CASE(FMULS)
33428   NODE_NAME_CASE(FMULS_RND)
33429   NODE_NAME_CASE(FDIV_RND)
33430   NODE_NAME_CASE(FDIVS)
33431   NODE_NAME_CASE(FDIVS_RND)
33432   NODE_NAME_CASE(FSQRT_RND)
33433   NODE_NAME_CASE(FSQRTS)
33434   NODE_NAME_CASE(FSQRTS_RND)
33435   NODE_NAME_CASE(FGETEXP)
33436   NODE_NAME_CASE(FGETEXP_SAE)
33437   NODE_NAME_CASE(FGETEXPS)
33438   NODE_NAME_CASE(FGETEXPS_SAE)
33439   NODE_NAME_CASE(SCALEF)
33440   NODE_NAME_CASE(SCALEF_RND)
33441   NODE_NAME_CASE(SCALEFS)
33442   NODE_NAME_CASE(SCALEFS_RND)
33443   NODE_NAME_CASE(MULHRS)
33444   NODE_NAME_CASE(SINT_TO_FP_RND)
33445   NODE_NAME_CASE(UINT_TO_FP_RND)
33446   NODE_NAME_CASE(CVTTP2SI)
33447   NODE_NAME_CASE(CVTTP2UI)
33448   NODE_NAME_CASE(STRICT_CVTTP2SI)
33449   NODE_NAME_CASE(STRICT_CVTTP2UI)
33450   NODE_NAME_CASE(MCVTTP2SI)
33451   NODE_NAME_CASE(MCVTTP2UI)
33452   NODE_NAME_CASE(CVTTP2SI_SAE)
33453   NODE_NAME_CASE(CVTTP2UI_SAE)
33454   NODE_NAME_CASE(CVTTS2SI)
33455   NODE_NAME_CASE(CVTTS2UI)
33456   NODE_NAME_CASE(CVTTS2SI_SAE)
33457   NODE_NAME_CASE(CVTTS2UI_SAE)
33458   NODE_NAME_CASE(CVTSI2P)
33459   NODE_NAME_CASE(CVTUI2P)
33460   NODE_NAME_CASE(STRICT_CVTSI2P)
33461   NODE_NAME_CASE(STRICT_CVTUI2P)
33462   NODE_NAME_CASE(MCVTSI2P)
33463   NODE_NAME_CASE(MCVTUI2P)
33464   NODE_NAME_CASE(VFPCLASS)
33465   NODE_NAME_CASE(VFPCLASSS)
33466   NODE_NAME_CASE(MULTISHIFT)
33467   NODE_NAME_CASE(SCALAR_SINT_TO_FP)
33468   NODE_NAME_CASE(SCALAR_SINT_TO_FP_RND)
33469   NODE_NAME_CASE(SCALAR_UINT_TO_FP)
33470   NODE_NAME_CASE(SCALAR_UINT_TO_FP_RND)
33471   NODE_NAME_CASE(CVTPS2PH)
33472   NODE_NAME_CASE(STRICT_CVTPS2PH)
33473   NODE_NAME_CASE(CVTPS2PH_SAE)
33474   NODE_NAME_CASE(MCVTPS2PH)
33475   NODE_NAME_CASE(MCVTPS2PH_SAE)
33476   NODE_NAME_CASE(CVTPH2PS)
33477   NODE_NAME_CASE(STRICT_CVTPH2PS)
33478   NODE_NAME_CASE(CVTPH2PS_SAE)
33479   NODE_NAME_CASE(CVTP2SI)
33480   NODE_NAME_CASE(CVTP2UI)
33481   NODE_NAME_CASE(MCVTP2SI)
33482   NODE_NAME_CASE(MCVTP2UI)
33483   NODE_NAME_CASE(CVTP2SI_RND)
33484   NODE_NAME_CASE(CVTP2UI_RND)
33485   NODE_NAME_CASE(CVTS2SI)
33486   NODE_NAME_CASE(CVTS2UI)
33487   NODE_NAME_CASE(CVTS2SI_RND)
33488   NODE_NAME_CASE(CVTS2UI_RND)
33489   NODE_NAME_CASE(CVTNE2PS2BF16)
33490   NODE_NAME_CASE(CVTNEPS2BF16)
33491   NODE_NAME_CASE(MCVTNEPS2BF16)
33492   NODE_NAME_CASE(DPBF16PS)
33493   NODE_NAME_CASE(LWPINS)
33494   NODE_NAME_CASE(MGATHER)
33495   NODE_NAME_CASE(MSCATTER)
33496   NODE_NAME_CASE(VPDPBUSD)
33497   NODE_NAME_CASE(VPDPBUSDS)
33498   NODE_NAME_CASE(VPDPWSSD)
33499   NODE_NAME_CASE(VPDPWSSDS)
33500   NODE_NAME_CASE(VPSHUFBITQMB)
33501   NODE_NAME_CASE(GF2P8MULB)
33502   NODE_NAME_CASE(GF2P8AFFINEQB)
33503   NODE_NAME_CASE(GF2P8AFFINEINVQB)
33504   NODE_NAME_CASE(NT_CALL)
33505   NODE_NAME_CASE(NT_BRIND)
33506   NODE_NAME_CASE(UMWAIT)
33507   NODE_NAME_CASE(TPAUSE)
33508   NODE_NAME_CASE(ENQCMD)
33509   NODE_NAME_CASE(ENQCMDS)
33510   NODE_NAME_CASE(VP2INTERSECT)
33511   NODE_NAME_CASE(VPDPBSUD)
33512   NODE_NAME_CASE(VPDPBSUDS)
33513   NODE_NAME_CASE(VPDPBUUD)
33514   NODE_NAME_CASE(VPDPBUUDS)
33515   NODE_NAME_CASE(VPDPBSSD)
33516   NODE_NAME_CASE(VPDPBSSDS)
33517   NODE_NAME_CASE(AESENC128KL)
33518   NODE_NAME_CASE(AESDEC128KL)
33519   NODE_NAME_CASE(AESENC256KL)
33520   NODE_NAME_CASE(AESDEC256KL)
33521   NODE_NAME_CASE(AESENCWIDE128KL)
33522   NODE_NAME_CASE(AESDECWIDE128KL)
33523   NODE_NAME_CASE(AESENCWIDE256KL)
33524   NODE_NAME_CASE(AESDECWIDE256KL)
33525   NODE_NAME_CASE(CMPCCXADD)
33526   NODE_NAME_CASE(TESTUI)
33527   NODE_NAME_CASE(FP80_ADD)
33528   NODE_NAME_CASE(STRICT_FP80_ADD)
33529   }
33530   return nullptr;
33531 #undef NODE_NAME_CASE
33532 }
33533 
33534 /// Return true if the addressing mode represented by AM is legal for this
33535 /// target, for a load/store of the specified type.
33536 bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
33537                                               const AddrMode &AM, Type *Ty,
33538                                               unsigned AS,
33539                                               Instruction *I) const {
33540   // X86 supports extremely general addressing modes.
33541   CodeModel::Model M = getTargetMachine().getCodeModel();
33542 
33543   // X86 allows a sign-extended 32-bit immediate field as a displacement.
33544   if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != nullptr))
33545     return false;
33546 
33547   if (AM.BaseGV) {
33548     unsigned GVFlags = Subtarget.classifyGlobalReference(AM.BaseGV);
33549 
33550     // If a reference to this global requires an extra load, we can't fold it.
33551     if (isGlobalStubReference(GVFlags))
33552       return false;
33553 
33554     // If BaseGV requires a register for the PIC base, we cannot also have a
33555     // BaseReg specified.
33556     if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
33557       return false;
33558 
33559     // If lower 4G is not available, then we must use rip-relative addressing.
33560     if ((M != CodeModel::Small || isPositionIndependent()) &&
33561         Subtarget.is64Bit() && (AM.BaseOffs || AM.Scale > 1))
33562       return false;
33563   }
33564 
33565   switch (AM.Scale) {
33566   case 0:
33567   case 1:
33568   case 2:
33569   case 4:
33570   case 8:
33571     // These scales always work.
33572     break;
33573   case 3:
33574   case 5:
33575   case 9:
33576     // These scales are formed with basereg+scalereg.  Only accept if there is
33577     // no basereg yet.
33578     if (AM.HasBaseReg)
33579       return false;
33580     break;
33581   default:  // Other stuff never works.
33582     return false;
33583   }
33584 
33585   return true;
33586 }
33587 
33588 bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const {
33589   unsigned Bits = Ty->getScalarSizeInBits();
33590 
33591   // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts.
33592   // Splitting for v32i8/v16i16 on XOP+AVX2 targets is still preferred.
33593   if (Subtarget.hasXOP() &&
33594       (Bits == 8 || Bits == 16 || Bits == 32 || Bits == 64))
33595     return false;
33596 
33597   // AVX2 has vpsllv[dq] instructions (and other shifts) that make variable
33598   // shifts just as cheap as scalar ones.
33599   if (Subtarget.hasAVX2() && (Bits == 32 || Bits == 64))
33600     return false;
33601 
33602   // AVX512BW has shifts such as vpsllvw.
33603   if (Subtarget.hasBWI() && Bits == 16)
33604     return false;
33605 
33606   // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
33607   // fully general vector.
33608   return true;
33609 }
33610 
33611 bool X86TargetLowering::isBinOp(unsigned Opcode) const {
33612   switch (Opcode) {
33613   // These are non-commutative binops.
33614   // TODO: Add more X86ISD opcodes once we have test coverage.
33615   case X86ISD::ANDNP:
33616   case X86ISD::PCMPGT:
33617   case X86ISD::FMAX:
33618   case X86ISD::FMIN:
33619   case X86ISD::FANDN:
33620   case X86ISD::VPSHA:
33621   case X86ISD::VPSHL:
33622   case X86ISD::VSHLV:
33623   case X86ISD::VSRLV:
33624   case X86ISD::VSRAV:
33625     return true;
33626   }
33627 
33628   return TargetLoweringBase::isBinOp(Opcode);
33629 }
33630 
33631 bool X86TargetLowering::isCommutativeBinOp(unsigned Opcode) const {
33632   switch (Opcode) {
33633   // TODO: Add more X86ISD opcodes once we have test coverage.
33634   case X86ISD::PCMPEQ:
33635   case X86ISD::PMULDQ:
33636   case X86ISD::PMULUDQ:
33637   case X86ISD::FMAXC:
33638   case X86ISD::FMINC:
33639   case X86ISD::FAND:
33640   case X86ISD::FOR:
33641   case X86ISD::FXOR:
33642     return true;
33643   }
33644 
33645   return TargetLoweringBase::isCommutativeBinOp(Opcode);
33646 }
33647 
33648 bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
33649   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
33650     return false;
33651   unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
33652   unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
33653   return NumBits1 > NumBits2;
33654 }
33655 
33656 bool X86TargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
33657   if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
33658     return false;
33659 
33660   if (!isTypeLegal(EVT::getEVT(Ty1)))
33661     return false;
33662 
33663   assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
33664 
33665   // Assuming the caller doesn't have a zeroext or signext return parameter,
33666   // truncation all the way down to i1 is valid.
33667   return true;
33668 }
33669 
33670 bool X86TargetLowering::isLegalICmpImmediate(int64_t Imm) const {
33671   return isInt<32>(Imm);
33672 }
33673 
33674 bool X86TargetLowering::isLegalAddImmediate(int64_t Imm) const {
33675   // Can also use sub to handle negated immediates.
33676   return isInt<32>(Imm);
33677 }
33678 
33679 bool X86TargetLowering::isLegalStoreImmediate(int64_t Imm) const {
33680   return isInt<32>(Imm);
33681 }
33682 
33683 bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
33684   if (!VT1.isScalarInteger() || !VT2.isScalarInteger())
33685     return false;
33686   unsigned NumBits1 = VT1.getSizeInBits();
33687   unsigned NumBits2 = VT2.getSizeInBits();
33688   return NumBits1 > NumBits2;
33689 }
33690 
33691 bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
33692   // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
33693   return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget.is64Bit();
33694 }
33695 
33696 bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
33697   // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
33698   return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget.is64Bit();
33699 }
33700 
33701 bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
33702   EVT VT1 = Val.getValueType();
33703   if (isZExtFree(VT1, VT2))
33704     return true;
33705 
33706   if (Val.getOpcode() != ISD::LOAD)
33707     return false;
33708 
33709   if (!VT1.isSimple() || !VT1.isInteger() ||
33710       !VT2.isSimple() || !VT2.isInteger())
33711     return false;
33712 
33713   switch (VT1.getSimpleVT().SimpleTy) {
33714   default: break;
33715   case MVT::i8:
33716   case MVT::i16:
33717   case MVT::i32:
33718     // X86 has 8, 16, and 32-bit zero-extending loads.
33719     return true;
33720   }
33721 
33722   return false;
33723 }
33724 
33725 bool X86TargetLowering::shouldSinkOperands(Instruction *I,
33726                                            SmallVectorImpl<Use *> &Ops) const {
33727   using namespace llvm::PatternMatch;
33728 
33729   FixedVectorType *VTy = dyn_cast<FixedVectorType>(I->getType());
33730   if (!VTy)
33731     return false;
33732 
33733   if (I->getOpcode() == Instruction::Mul &&
33734       VTy->getElementType()->isIntegerTy(64)) {
33735     for (auto &Op : I->operands()) {
33736       // Make sure we are not already sinking this operand
33737       if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
33738         continue;
33739 
33740       // Look for PMULDQ pattern where the input is a sext_inreg from vXi32 or
33741       // the PMULUDQ pattern where the input is a zext_inreg from vXi32.
33742       if (Subtarget.hasSSE41() &&
33743           match(Op.get(), m_AShr(m_Shl(m_Value(), m_SpecificInt(32)),
33744                                  m_SpecificInt(32)))) {
33745         Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0));
33746         Ops.push_back(&Op);
33747       } else if (Subtarget.hasSSE2() &&
33748                  match(Op.get(),
33749                        m_And(m_Value(), m_SpecificInt(UINT64_C(0xffffffff))))) {
33750         Ops.push_back(&Op);
33751       }
33752     }
33753 
33754     return !Ops.empty();
33755   }
33756 
33757   // A uniform shift amount in a vector shift or funnel shift may be much
33758   // cheaper than a generic variable vector shift, so make that pattern visible
33759   // to SDAG by sinking the shuffle instruction next to the shift.
33760   int ShiftAmountOpNum = -1;
33761   if (I->isShift())
33762     ShiftAmountOpNum = 1;
33763   else if (auto *II = dyn_cast<IntrinsicInst>(I)) {
33764     if (II->getIntrinsicID() == Intrinsic::fshl ||
33765         II->getIntrinsicID() == Intrinsic::fshr)
33766       ShiftAmountOpNum = 2;
33767   }
33768 
33769   if (ShiftAmountOpNum == -1)
33770     return false;
33771 
33772   auto *Shuf = dyn_cast<ShuffleVectorInst>(I->getOperand(ShiftAmountOpNum));
33773   if (Shuf && getSplatIndex(Shuf->getShuffleMask()) >= 0 &&
33774       isVectorShiftByScalarCheap(I->getType())) {
33775     Ops.push_back(&I->getOperandUse(ShiftAmountOpNum));
33776     return true;
33777   }
33778 
33779   return false;
33780 }
33781 
33782 bool X86TargetLowering::shouldConvertPhiType(Type *From, Type *To) const {
33783   if (!Subtarget.is64Bit())
33784     return false;
33785   return TargetLowering::shouldConvertPhiType(From, To);
33786 }
33787 
33788 bool X86TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
33789   if (isa<MaskedLoadSDNode>(ExtVal.getOperand(0)))
33790     return false;
33791 
33792   EVT SrcVT = ExtVal.getOperand(0).getValueType();
33793 
33794   // There is no extending load for vXi1.
33795   if (SrcVT.getScalarType() == MVT::i1)
33796     return false;
33797 
33798   return true;
33799 }
33800 
33801 bool X86TargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
33802                                                    EVT VT) const {
33803   if (!Subtarget.hasAnyFMA())
33804     return false;
33805 
33806   VT = VT.getScalarType();
33807 
33808   if (!VT.isSimple())
33809     return false;
33810 
33811   switch (VT.getSimpleVT().SimpleTy) {
33812   case MVT::f16:
33813     return Subtarget.hasFP16();
33814   case MVT::f32:
33815   case MVT::f64:
33816     return true;
33817   default:
33818     break;
33819   }
33820 
33821   return false;
33822 }
33823 
33824 bool X86TargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const {
33825   // i16 instructions are longer (0x66 prefix) and potentially slower.
33826   return !(SrcVT == MVT::i32 && DestVT == MVT::i16);
33827 }
33828 
33829 bool X86TargetLowering::shouldFoldSelectWithIdentityConstant(unsigned Opcode,
33830                                                              EVT VT) const {
33831   // TODO: This is too general. There are cases where pre-AVX512 codegen would
33832   //       benefit. The transform may also be profitable for scalar code.
33833   if (!Subtarget.hasAVX512())
33834     return false;
33835   if (!Subtarget.hasVLX() && !VT.is512BitVector())
33836     return false;
33837   if (!VT.isVector() || VT.getScalarType() == MVT::i1)
33838     return false;
33839 
33840   return true;
33841 }
33842 
33843 /// Targets can use this to indicate that they only support *some*
33844 /// VECTOR_SHUFFLE operations, those with specific masks.
33845 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
33846 /// are assumed to be legal.
33847 bool X86TargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const {
33848   if (!VT.isSimple())
33849     return false;
33850 
33851   // Not for i1 vectors
33852   if (VT.getSimpleVT().getScalarType() == MVT::i1)
33853     return false;
33854 
33855   // Very little shuffling can be done for 64-bit vectors right now.
33856   if (VT.getSimpleVT().getSizeInBits() == 64)
33857     return false;
33858 
33859   // We only care that the types being shuffled are legal. The lowering can
33860   // handle any possible shuffle mask that results.
33861   return isTypeLegal(VT.getSimpleVT());
33862 }
33863 
33864 bool X86TargetLowering::isVectorClearMaskLegal(ArrayRef<int> Mask,
33865                                                EVT VT) const {
33866   // Don't convert an 'and' into a shuffle that we don't directly support.
33867   // vpblendw and vpshufb for 256-bit vectors are not available on AVX1.
33868   if (!Subtarget.hasAVX2())
33869     if (VT == MVT::v32i8 || VT == MVT::v16i16)
33870       return false;
33871 
33872   // Just delegate to the generic legality, clear masks aren't special.
33873   return isShuffleMaskLegal(Mask, VT);
33874 }
33875 
33876 bool X86TargetLowering::areJTsAllowed(const Function *Fn) const {
33877   // If the subtarget is using thunks, we need to not generate jump tables.
33878   if (Subtarget.useIndirectThunkBranches())
33879     return false;
33880 
33881   // Otherwise, fallback on the generic logic.
33882   return TargetLowering::areJTsAllowed(Fn);
33883 }
33884 
33885 MVT X86TargetLowering::getPreferredSwitchConditionType(LLVMContext &Context,
33886                                                        EVT ConditionVT) const {
33887   // Avoid 8 and 16 bit types because they increase the chance for unnecessary
33888   // zero-extensions.
33889   if (ConditionVT.getSizeInBits() < 32)
33890     return MVT::i32;
33891   return TargetLoweringBase::getPreferredSwitchConditionType(Context,
33892                                                              ConditionVT);
33893 }
33894 
33895 //===----------------------------------------------------------------------===//
33896 //                           X86 Scheduler Hooks
33897 //===----------------------------------------------------------------------===//
33898 
33899 // Returns true if EFLAG is consumed after this iterator in the rest of the
33900 // basic block or any successors of the basic block.
33901 static bool isEFLAGSLiveAfter(MachineBasicBlock::iterator Itr,
33902                               MachineBasicBlock *BB) {
33903   // Scan forward through BB for a use/def of EFLAGS.
33904   for (const MachineInstr &mi : llvm::make_range(std::next(Itr), BB->end())) {
33905     if (mi.readsRegister(X86::EFLAGS))
33906       return true;
33907     // If we found a def, we can stop searching.
33908     if (mi.definesRegister(X86::EFLAGS))
33909       return false;
33910   }
33911 
33912   // If we hit the end of the block, check whether EFLAGS is live into a
33913   // successor.
33914   for (MachineBasicBlock *Succ : BB->successors())
33915     if (Succ->isLiveIn(X86::EFLAGS))
33916       return true;
33917 
33918   return false;
33919 }
33920 
33921 /// Utility function to emit xbegin specifying the start of an RTM region.
33922 static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB,
33923                                      const TargetInstrInfo *TII) {
33924   const MIMetadata MIMD(MI);
33925 
33926   const BasicBlock *BB = MBB->getBasicBlock();
33927   MachineFunction::iterator I = ++MBB->getIterator();
33928 
33929   // For the v = xbegin(), we generate
33930   //
33931   // thisMBB:
33932   //  xbegin sinkMBB
33933   //
33934   // mainMBB:
33935   //  s0 = -1
33936   //
33937   // fallBB:
33938   //  eax = # XABORT_DEF
33939   //  s1 = eax
33940   //
33941   // sinkMBB:
33942   //  v = phi(s0/mainBB, s1/fallBB)
33943 
33944   MachineBasicBlock *thisMBB = MBB;
33945   MachineFunction *MF = MBB->getParent();
33946   MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
33947   MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB);
33948   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
33949   MF->insert(I, mainMBB);
33950   MF->insert(I, fallMBB);
33951   MF->insert(I, sinkMBB);
33952 
33953   if (isEFLAGSLiveAfter(MI, MBB)) {
33954     mainMBB->addLiveIn(X86::EFLAGS);
33955     fallMBB->addLiveIn(X86::EFLAGS);
33956     sinkMBB->addLiveIn(X86::EFLAGS);
33957   }
33958 
33959   // Transfer the remainder of BB and its successor edges to sinkMBB.
33960   sinkMBB->splice(sinkMBB->begin(), MBB,
33961                   std::next(MachineBasicBlock::iterator(MI)), MBB->end());
33962   sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
33963 
33964   MachineRegisterInfo &MRI = MF->getRegInfo();
33965   Register DstReg = MI.getOperand(0).getReg();
33966   const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
33967   Register mainDstReg = MRI.createVirtualRegister(RC);
33968   Register fallDstReg = MRI.createVirtualRegister(RC);
33969 
33970   // thisMBB:
33971   //  xbegin fallMBB
33972   //  # fallthrough to mainMBB
33973   //  # abortion to fallMBB
33974   BuildMI(thisMBB, MIMD, TII->get(X86::XBEGIN_4)).addMBB(fallMBB);
33975   thisMBB->addSuccessor(mainMBB);
33976   thisMBB->addSuccessor(fallMBB);
33977 
33978   // mainMBB:
33979   //  mainDstReg := -1
33980   BuildMI(mainMBB, MIMD, TII->get(X86::MOV32ri), mainDstReg).addImm(-1);
33981   BuildMI(mainMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
33982   mainMBB->addSuccessor(sinkMBB);
33983 
33984   // fallMBB:
33985   //  ; pseudo instruction to model hardware's definition from XABORT
33986   //  EAX := XABORT_DEF
33987   //  fallDstReg := EAX
33988   BuildMI(fallMBB, MIMD, TII->get(X86::XABORT_DEF));
33989   BuildMI(fallMBB, MIMD, TII->get(TargetOpcode::COPY), fallDstReg)
33990       .addReg(X86::EAX);
33991   fallMBB->addSuccessor(sinkMBB);
33992 
33993   // sinkMBB:
33994   //  DstReg := phi(mainDstReg/mainBB, fallDstReg/fallBB)
33995   BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
33996       .addReg(mainDstReg).addMBB(mainMBB)
33997       .addReg(fallDstReg).addMBB(fallMBB);
33998 
33999   MI.eraseFromParent();
34000   return sinkMBB;
34001 }
34002 
34003 MachineBasicBlock *
34004 X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
34005                                                MachineBasicBlock *MBB) const {
34006   // Emit va_arg instruction on X86-64.
34007 
34008   // Operands to this pseudo-instruction:
34009   // 0  ) Output        : destination address (reg)
34010   // 1-5) Input         : va_list address (addr, i64mem)
34011   // 6  ) ArgSize       : Size (in bytes) of vararg type
34012   // 7  ) ArgMode       : 0=overflow only, 1=use gp_offset, 2=use fp_offset
34013   // 8  ) Align         : Alignment of type
34014   // 9  ) EFLAGS (implicit-def)
34015 
34016   assert(MI.getNumOperands() == 10 && "VAARG should have 10 operands!");
34017   static_assert(X86::AddrNumOperands == 5, "VAARG assumes 5 address operands");
34018 
34019   Register DestReg = MI.getOperand(0).getReg();
34020   MachineOperand &Base = MI.getOperand(1);
34021   MachineOperand &Scale = MI.getOperand(2);
34022   MachineOperand &Index = MI.getOperand(3);
34023   MachineOperand &Disp = MI.getOperand(4);
34024   MachineOperand &Segment = MI.getOperand(5);
34025   unsigned ArgSize = MI.getOperand(6).getImm();
34026   unsigned ArgMode = MI.getOperand(7).getImm();
34027   Align Alignment = Align(MI.getOperand(8).getImm());
34028 
34029   MachineFunction *MF = MBB->getParent();
34030 
34031   // Memory Reference
34032   assert(MI.hasOneMemOperand() && "Expected VAARG to have one memoperand");
34033 
34034   MachineMemOperand *OldMMO = MI.memoperands().front();
34035 
34036   // Clone the MMO into two separate MMOs for loading and storing
34037   MachineMemOperand *LoadOnlyMMO = MF->getMachineMemOperand(
34038       OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOStore);
34039   MachineMemOperand *StoreOnlyMMO = MF->getMachineMemOperand(
34040       OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOLoad);
34041 
34042   // Machine Information
34043   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34044   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
34045   const TargetRegisterClass *AddrRegClass =
34046       getRegClassFor(getPointerTy(MBB->getParent()->getDataLayout()));
34047   const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32);
34048   const MIMetadata MIMD(MI);
34049 
34050   // struct va_list {
34051   //   i32   gp_offset
34052   //   i32   fp_offset
34053   //   i64   overflow_area (address)
34054   //   i64   reg_save_area (address)
34055   // }
34056   // sizeof(va_list) = 24
34057   // alignment(va_list) = 8
34058 
34059   unsigned TotalNumIntRegs = 6;
34060   unsigned TotalNumXMMRegs = 8;
34061   bool UseGPOffset = (ArgMode == 1);
34062   bool UseFPOffset = (ArgMode == 2);
34063   unsigned MaxOffset = TotalNumIntRegs * 8 +
34064                        (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
34065 
34066   /* Align ArgSize to a multiple of 8 */
34067   unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
34068   bool NeedsAlign = (Alignment > 8);
34069 
34070   MachineBasicBlock *thisMBB = MBB;
34071   MachineBasicBlock *overflowMBB;
34072   MachineBasicBlock *offsetMBB;
34073   MachineBasicBlock *endMBB;
34074 
34075   unsigned OffsetDestReg = 0;    // Argument address computed by offsetMBB
34076   unsigned OverflowDestReg = 0;  // Argument address computed by overflowMBB
34077   unsigned OffsetReg = 0;
34078 
34079   if (!UseGPOffset && !UseFPOffset) {
34080     // If we only pull from the overflow region, we don't create a branch.
34081     // We don't need to alter control flow.
34082     OffsetDestReg = 0; // unused
34083     OverflowDestReg = DestReg;
34084 
34085     offsetMBB = nullptr;
34086     overflowMBB = thisMBB;
34087     endMBB = thisMBB;
34088   } else {
34089     // First emit code to check if gp_offset (or fp_offset) is below the bound.
34090     // If so, pull the argument from reg_save_area. (branch to offsetMBB)
34091     // If not, pull from overflow_area. (branch to overflowMBB)
34092     //
34093     //       thisMBB
34094     //         |     .
34095     //         |        .
34096     //     offsetMBB   overflowMBB
34097     //         |        .
34098     //         |     .
34099     //        endMBB
34100 
34101     // Registers for the PHI in endMBB
34102     OffsetDestReg = MRI.createVirtualRegister(AddrRegClass);
34103     OverflowDestReg = MRI.createVirtualRegister(AddrRegClass);
34104 
34105     const BasicBlock *LLVM_BB = MBB->getBasicBlock();
34106     overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34107     offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34108     endMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34109 
34110     MachineFunction::iterator MBBIter = ++MBB->getIterator();
34111 
34112     // Insert the new basic blocks
34113     MF->insert(MBBIter, offsetMBB);
34114     MF->insert(MBBIter, overflowMBB);
34115     MF->insert(MBBIter, endMBB);
34116 
34117     // Transfer the remainder of MBB and its successor edges to endMBB.
34118     endMBB->splice(endMBB->begin(), thisMBB,
34119                    std::next(MachineBasicBlock::iterator(MI)), thisMBB->end());
34120     endMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
34121 
34122     // Make offsetMBB and overflowMBB successors of thisMBB
34123     thisMBB->addSuccessor(offsetMBB);
34124     thisMBB->addSuccessor(overflowMBB);
34125 
34126     // endMBB is a successor of both offsetMBB and overflowMBB
34127     offsetMBB->addSuccessor(endMBB);
34128     overflowMBB->addSuccessor(endMBB);
34129 
34130     // Load the offset value into a register
34131     OffsetReg = MRI.createVirtualRegister(OffsetRegClass);
34132     BuildMI(thisMBB, MIMD, TII->get(X86::MOV32rm), OffsetReg)
34133         .add(Base)
34134         .add(Scale)
34135         .add(Index)
34136         .addDisp(Disp, UseFPOffset ? 4 : 0)
34137         .add(Segment)
34138         .setMemRefs(LoadOnlyMMO);
34139 
34140     // Check if there is enough room left to pull this argument.
34141     BuildMI(thisMBB, MIMD, TII->get(X86::CMP32ri))
34142       .addReg(OffsetReg)
34143       .addImm(MaxOffset + 8 - ArgSizeA8);
34144 
34145     // Branch to "overflowMBB" if offset >= max
34146     // Fall through to "offsetMBB" otherwise
34147     BuildMI(thisMBB, MIMD, TII->get(X86::JCC_1))
34148       .addMBB(overflowMBB).addImm(X86::COND_AE);
34149   }
34150 
34151   // In offsetMBB, emit code to use the reg_save_area.
34152   if (offsetMBB) {
34153     assert(OffsetReg != 0);
34154 
34155     // Read the reg_save_area address.
34156     Register RegSaveReg = MRI.createVirtualRegister(AddrRegClass);
34157     BuildMI(
34158         offsetMBB, MIMD,
34159         TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm),
34160         RegSaveReg)
34161         .add(Base)
34162         .add(Scale)
34163         .add(Index)
34164         .addDisp(Disp, Subtarget.isTarget64BitLP64() ? 16 : 12)
34165         .add(Segment)
34166         .setMemRefs(LoadOnlyMMO);
34167 
34168     if (Subtarget.isTarget64BitLP64()) {
34169       // Zero-extend the offset
34170       Register OffsetReg64 = MRI.createVirtualRegister(AddrRegClass);
34171       BuildMI(offsetMBB, MIMD, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
34172           .addImm(0)
34173           .addReg(OffsetReg)
34174           .addImm(X86::sub_32bit);
34175 
34176       // Add the offset to the reg_save_area to get the final address.
34177       BuildMI(offsetMBB, MIMD, TII->get(X86::ADD64rr), OffsetDestReg)
34178           .addReg(OffsetReg64)
34179           .addReg(RegSaveReg);
34180     } else {
34181       // Add the offset to the reg_save_area to get the final address.
34182       BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32rr), OffsetDestReg)
34183           .addReg(OffsetReg)
34184           .addReg(RegSaveReg);
34185     }
34186 
34187     // Compute the offset for the next argument
34188     Register NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass);
34189     BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32ri), NextOffsetReg)
34190       .addReg(OffsetReg)
34191       .addImm(UseFPOffset ? 16 : 8);
34192 
34193     // Store it back into the va_list.
34194     BuildMI(offsetMBB, MIMD, TII->get(X86::MOV32mr))
34195         .add(Base)
34196         .add(Scale)
34197         .add(Index)
34198         .addDisp(Disp, UseFPOffset ? 4 : 0)
34199         .add(Segment)
34200         .addReg(NextOffsetReg)
34201         .setMemRefs(StoreOnlyMMO);
34202 
34203     // Jump to endMBB
34204     BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1))
34205       .addMBB(endMBB);
34206   }
34207 
34208   //
34209   // Emit code to use overflow area
34210   //
34211 
34212   // Load the overflow_area address into a register.
34213   Register OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass);
34214   BuildMI(overflowMBB, MIMD,
34215           TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm),
34216           OverflowAddrReg)
34217       .add(Base)
34218       .add(Scale)
34219       .add(Index)
34220       .addDisp(Disp, 8)
34221       .add(Segment)
34222       .setMemRefs(LoadOnlyMMO);
34223 
34224   // If we need to align it, do so. Otherwise, just copy the address
34225   // to OverflowDestReg.
34226   if (NeedsAlign) {
34227     // Align the overflow address
34228     Register TmpReg = MRI.createVirtualRegister(AddrRegClass);
34229 
34230     // aligned_addr = (addr + (align-1)) & ~(align-1)
34231     BuildMI(
34232         overflowMBB, MIMD,
34233         TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri),
34234         TmpReg)
34235         .addReg(OverflowAddrReg)
34236         .addImm(Alignment.value() - 1);
34237 
34238     BuildMI(
34239         overflowMBB, MIMD,
34240         TII->get(Subtarget.isTarget64BitLP64() ? X86::AND64ri32 : X86::AND32ri),
34241         OverflowDestReg)
34242         .addReg(TmpReg)
34243         .addImm(~(uint64_t)(Alignment.value() - 1));
34244   } else {
34245     BuildMI(overflowMBB, MIMD, TII->get(TargetOpcode::COPY), OverflowDestReg)
34246       .addReg(OverflowAddrReg);
34247   }
34248 
34249   // Compute the next overflow address after this argument.
34250   // (the overflow address should be kept 8-byte aligned)
34251   Register NextAddrReg = MRI.createVirtualRegister(AddrRegClass);
34252   BuildMI(
34253       overflowMBB, MIMD,
34254       TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri),
34255       NextAddrReg)
34256       .addReg(OverflowDestReg)
34257       .addImm(ArgSizeA8);
34258 
34259   // Store the new overflow address.
34260   BuildMI(overflowMBB, MIMD,
34261           TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64mr : X86::MOV32mr))
34262       .add(Base)
34263       .add(Scale)
34264       .add(Index)
34265       .addDisp(Disp, 8)
34266       .add(Segment)
34267       .addReg(NextAddrReg)
34268       .setMemRefs(StoreOnlyMMO);
34269 
34270   // If we branched, emit the PHI to the front of endMBB.
34271   if (offsetMBB) {
34272     BuildMI(*endMBB, endMBB->begin(), MIMD,
34273             TII->get(X86::PHI), DestReg)
34274       .addReg(OffsetDestReg).addMBB(offsetMBB)
34275       .addReg(OverflowDestReg).addMBB(overflowMBB);
34276   }
34277 
34278   // Erase the pseudo instruction
34279   MI.eraseFromParent();
34280 
34281   return endMBB;
34282 }
34283 
34284 // The EFLAGS operand of SelectItr might be missing a kill marker
34285 // because there were multiple uses of EFLAGS, and ISel didn't know
34286 // which to mark. Figure out whether SelectItr should have had a
34287 // kill marker, and set it if it should. Returns the correct kill
34288 // marker value.
34289 static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
34290                                      MachineBasicBlock* BB,
34291                                      const TargetRegisterInfo* TRI) {
34292   if (isEFLAGSLiveAfter(SelectItr, BB))
34293     return false;
34294 
34295   // We found a def, or hit the end of the basic block and EFLAGS wasn't live
34296   // out. SelectMI should have a kill flag on EFLAGS.
34297   SelectItr->addRegisterKilled(X86::EFLAGS, TRI);
34298   return true;
34299 }
34300 
34301 // Return true if it is OK for this CMOV pseudo-opcode to be cascaded
34302 // together with other CMOV pseudo-opcodes into a single basic-block with
34303 // conditional jump around it.
34304 static bool isCMOVPseudo(MachineInstr &MI) {
34305   switch (MI.getOpcode()) {
34306   case X86::CMOV_FR16:
34307   case X86::CMOV_FR16X:
34308   case X86::CMOV_FR32:
34309   case X86::CMOV_FR32X:
34310   case X86::CMOV_FR64:
34311   case X86::CMOV_FR64X:
34312   case X86::CMOV_GR8:
34313   case X86::CMOV_GR16:
34314   case X86::CMOV_GR32:
34315   case X86::CMOV_RFP32:
34316   case X86::CMOV_RFP64:
34317   case X86::CMOV_RFP80:
34318   case X86::CMOV_VR64:
34319   case X86::CMOV_VR128:
34320   case X86::CMOV_VR128X:
34321   case X86::CMOV_VR256:
34322   case X86::CMOV_VR256X:
34323   case X86::CMOV_VR512:
34324   case X86::CMOV_VK1:
34325   case X86::CMOV_VK2:
34326   case X86::CMOV_VK4:
34327   case X86::CMOV_VK8:
34328   case X86::CMOV_VK16:
34329   case X86::CMOV_VK32:
34330   case X86::CMOV_VK64:
34331     return true;
34332 
34333   default:
34334     return false;
34335   }
34336 }
34337 
34338 // Helper function, which inserts PHI functions into SinkMBB:
34339 //   %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
34340 // where %FalseValue(i) and %TrueValue(i) are taken from the consequent CMOVs
34341 // in [MIItBegin, MIItEnd) range. It returns the last MachineInstrBuilder for
34342 // the last PHI function inserted.
34343 static MachineInstrBuilder createPHIsForCMOVsInSinkBB(
34344     MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd,
34345     MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB,
34346     MachineBasicBlock *SinkMBB) {
34347   MachineFunction *MF = TrueMBB->getParent();
34348   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
34349   const MIMetadata MIMD(*MIItBegin);
34350 
34351   X86::CondCode CC = X86::CondCode(MIItBegin->getOperand(3).getImm());
34352   X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
34353 
34354   MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
34355 
34356   // As we are creating the PHIs, we have to be careful if there is more than
34357   // one.  Later CMOVs may reference the results of earlier CMOVs, but later
34358   // PHIs have to reference the individual true/false inputs from earlier PHIs.
34359   // That also means that PHI construction must work forward from earlier to
34360   // later, and that the code must maintain a mapping from earlier PHI's
34361   // destination registers, and the registers that went into the PHI.
34362   DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
34363   MachineInstrBuilder MIB;
34364 
34365   for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd; ++MIIt) {
34366     Register DestReg = MIIt->getOperand(0).getReg();
34367     Register Op1Reg = MIIt->getOperand(1).getReg();
34368     Register Op2Reg = MIIt->getOperand(2).getReg();
34369 
34370     // If this CMOV we are generating is the opposite condition from
34371     // the jump we generated, then we have to swap the operands for the
34372     // PHI that is going to be generated.
34373     if (MIIt->getOperand(3).getImm() == OppCC)
34374       std::swap(Op1Reg, Op2Reg);
34375 
34376     if (RegRewriteTable.contains(Op1Reg))
34377       Op1Reg = RegRewriteTable[Op1Reg].first;
34378 
34379     if (RegRewriteTable.contains(Op2Reg))
34380       Op2Reg = RegRewriteTable[Op2Reg].second;
34381 
34382     MIB =
34383         BuildMI(*SinkMBB, SinkInsertionPoint, MIMD, TII->get(X86::PHI), DestReg)
34384             .addReg(Op1Reg)
34385             .addMBB(FalseMBB)
34386             .addReg(Op2Reg)
34387             .addMBB(TrueMBB);
34388 
34389     // Add this PHI to the rewrite table.
34390     RegRewriteTable[DestReg] = std::make_pair(Op1Reg, Op2Reg);
34391   }
34392 
34393   return MIB;
34394 }
34395 
34396 // Lower cascaded selects in form of (SecondCmov (FirstCMOV F, T, cc1), T, cc2).
34397 MachineBasicBlock *
34398 X86TargetLowering::EmitLoweredCascadedSelect(MachineInstr &FirstCMOV,
34399                                              MachineInstr &SecondCascadedCMOV,
34400                                              MachineBasicBlock *ThisMBB) const {
34401   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34402   const MIMetadata MIMD(FirstCMOV);
34403 
34404   // We lower cascaded CMOVs such as
34405   //
34406   //   (SecondCascadedCMOV (FirstCMOV F, T, cc1), T, cc2)
34407   //
34408   // to two successive branches.
34409   //
34410   // Without this, we would add a PHI between the two jumps, which ends up
34411   // creating a few copies all around. For instance, for
34412   //
34413   //    (sitofp (zext (fcmp une)))
34414   //
34415   // we would generate:
34416   //
34417   //         ucomiss %xmm1, %xmm0
34418   //         movss  <1.0f>, %xmm0
34419   //         movaps  %xmm0, %xmm1
34420   //         jne     .LBB5_2
34421   //         xorps   %xmm1, %xmm1
34422   // .LBB5_2:
34423   //         jp      .LBB5_4
34424   //         movaps  %xmm1, %xmm0
34425   // .LBB5_4:
34426   //         retq
34427   //
34428   // because this custom-inserter would have generated:
34429   //
34430   //   A
34431   //   | \
34432   //   |  B
34433   //   | /
34434   //   C
34435   //   | \
34436   //   |  D
34437   //   | /
34438   //   E
34439   //
34440   // A: X = ...; Y = ...
34441   // B: empty
34442   // C: Z = PHI [X, A], [Y, B]
34443   // D: empty
34444   // E: PHI [X, C], [Z, D]
34445   //
34446   // If we lower both CMOVs in a single step, we can instead generate:
34447   //
34448   //   A
34449   //   | \
34450   //   |  C
34451   //   | /|
34452   //   |/ |
34453   //   |  |
34454   //   |  D
34455   //   | /
34456   //   E
34457   //
34458   // A: X = ...; Y = ...
34459   // D: empty
34460   // E: PHI [X, A], [X, C], [Y, D]
34461   //
34462   // Which, in our sitofp/fcmp example, gives us something like:
34463   //
34464   //         ucomiss %xmm1, %xmm0
34465   //         movss  <1.0f>, %xmm0
34466   //         jne     .LBB5_4
34467   //         jp      .LBB5_4
34468   //         xorps   %xmm0, %xmm0
34469   // .LBB5_4:
34470   //         retq
34471   //
34472 
34473   // We lower cascaded CMOV into two successive branches to the same block.
34474   // EFLAGS is used by both, so mark it as live in the second.
34475   const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
34476   MachineFunction *F = ThisMBB->getParent();
34477   MachineBasicBlock *FirstInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB);
34478   MachineBasicBlock *SecondInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB);
34479   MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
34480 
34481   MachineFunction::iterator It = ++ThisMBB->getIterator();
34482   F->insert(It, FirstInsertedMBB);
34483   F->insert(It, SecondInsertedMBB);
34484   F->insert(It, SinkMBB);
34485 
34486   // For a cascaded CMOV, we lower it to two successive branches to
34487   // the same block (SinkMBB).  EFLAGS is used by both, so mark it as live in
34488   // the FirstInsertedMBB.
34489   FirstInsertedMBB->addLiveIn(X86::EFLAGS);
34490 
34491   // If the EFLAGS register isn't dead in the terminator, then claim that it's
34492   // live into the sink and copy blocks.
34493   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
34494   if (!SecondCascadedCMOV.killsRegister(X86::EFLAGS) &&
34495       !checkAndUpdateEFLAGSKill(SecondCascadedCMOV, ThisMBB, TRI)) {
34496     SecondInsertedMBB->addLiveIn(X86::EFLAGS);
34497     SinkMBB->addLiveIn(X86::EFLAGS);
34498   }
34499 
34500   // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
34501   SinkMBB->splice(SinkMBB->begin(), ThisMBB,
34502                   std::next(MachineBasicBlock::iterator(FirstCMOV)),
34503                   ThisMBB->end());
34504   SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
34505 
34506   // Fallthrough block for ThisMBB.
34507   ThisMBB->addSuccessor(FirstInsertedMBB);
34508   // The true block target of the first branch is always SinkMBB.
34509   ThisMBB->addSuccessor(SinkMBB);
34510   // Fallthrough block for FirstInsertedMBB.
34511   FirstInsertedMBB->addSuccessor(SecondInsertedMBB);
34512   // The true block for the branch of FirstInsertedMBB.
34513   FirstInsertedMBB->addSuccessor(SinkMBB);
34514   // This is fallthrough.
34515   SecondInsertedMBB->addSuccessor(SinkMBB);
34516 
34517   // Create the conditional branch instructions.
34518   X86::CondCode FirstCC = X86::CondCode(FirstCMOV.getOperand(3).getImm());
34519   BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(FirstCC);
34520 
34521   X86::CondCode SecondCC =
34522       X86::CondCode(SecondCascadedCMOV.getOperand(3).getImm());
34523   BuildMI(FirstInsertedMBB, MIMD, TII->get(X86::JCC_1))
34524       .addMBB(SinkMBB)
34525       .addImm(SecondCC);
34526 
34527   //  SinkMBB:
34528   //   %Result = phi [ %FalseValue, SecondInsertedMBB ], [ %TrueValue, ThisMBB ]
34529   Register DestReg = SecondCascadedCMOV.getOperand(0).getReg();
34530   Register Op1Reg = FirstCMOV.getOperand(1).getReg();
34531   Register Op2Reg = FirstCMOV.getOperand(2).getReg();
34532   MachineInstrBuilder MIB =
34533       BuildMI(*SinkMBB, SinkMBB->begin(), MIMD, TII->get(X86::PHI), DestReg)
34534           .addReg(Op1Reg)
34535           .addMBB(SecondInsertedMBB)
34536           .addReg(Op2Reg)
34537           .addMBB(ThisMBB);
34538 
34539   // The second SecondInsertedMBB provides the same incoming value as the
34540   // FirstInsertedMBB (the True operand of the SELECT_CC/CMOV nodes).
34541   MIB.addReg(FirstCMOV.getOperand(2).getReg()).addMBB(FirstInsertedMBB);
34542 
34543   // Now remove the CMOVs.
34544   FirstCMOV.eraseFromParent();
34545   SecondCascadedCMOV.eraseFromParent();
34546 
34547   return SinkMBB;
34548 }
34549 
34550 MachineBasicBlock *
34551 X86TargetLowering::EmitLoweredSelect(MachineInstr &MI,
34552                                      MachineBasicBlock *ThisMBB) const {
34553   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34554   const MIMetadata MIMD(MI);
34555 
34556   // To "insert" a SELECT_CC instruction, we actually have to insert the
34557   // diamond control-flow pattern.  The incoming instruction knows the
34558   // destination vreg to set, the condition code register to branch on, the
34559   // true/false values to select between and a branch opcode to use.
34560 
34561   //  ThisMBB:
34562   //  ...
34563   //   TrueVal = ...
34564   //   cmpTY ccX, r1, r2
34565   //   bCC copy1MBB
34566   //   fallthrough --> FalseMBB
34567 
34568   // This code lowers all pseudo-CMOV instructions. Generally it lowers these
34569   // as described above, by inserting a BB, and then making a PHI at the join
34570   // point to select the true and false operands of the CMOV in the PHI.
34571   //
34572   // The code also handles two different cases of multiple CMOV opcodes
34573   // in a row.
34574   //
34575   // Case 1:
34576   // In this case, there are multiple CMOVs in a row, all which are based on
34577   // the same condition setting (or the exact opposite condition setting).
34578   // In this case we can lower all the CMOVs using a single inserted BB, and
34579   // then make a number of PHIs at the join point to model the CMOVs. The only
34580   // trickiness here, is that in a case like:
34581   //
34582   // t2 = CMOV cond1 t1, f1
34583   // t3 = CMOV cond1 t2, f2
34584   //
34585   // when rewriting this into PHIs, we have to perform some renaming on the
34586   // temps since you cannot have a PHI operand refer to a PHI result earlier
34587   // in the same block.  The "simple" but wrong lowering would be:
34588   //
34589   // t2 = PHI t1(BB1), f1(BB2)
34590   // t3 = PHI t2(BB1), f2(BB2)
34591   //
34592   // but clearly t2 is not defined in BB1, so that is incorrect. The proper
34593   // renaming is to note that on the path through BB1, t2 is really just a
34594   // copy of t1, and do that renaming, properly generating:
34595   //
34596   // t2 = PHI t1(BB1), f1(BB2)
34597   // t3 = PHI t1(BB1), f2(BB2)
34598   //
34599   // Case 2:
34600   // CMOV ((CMOV F, T, cc1), T, cc2) is checked here and handled by a separate
34601   // function - EmitLoweredCascadedSelect.
34602 
34603   X86::CondCode CC = X86::CondCode(MI.getOperand(3).getImm());
34604   X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
34605   MachineInstr *LastCMOV = &MI;
34606   MachineBasicBlock::iterator NextMIIt = MachineBasicBlock::iterator(MI);
34607 
34608   // Check for case 1, where there are multiple CMOVs with the same condition
34609   // first.  Of the two cases of multiple CMOV lowerings, case 1 reduces the
34610   // number of jumps the most.
34611 
34612   if (isCMOVPseudo(MI)) {
34613     // See if we have a string of CMOVS with the same condition. Skip over
34614     // intervening debug insts.
34615     while (NextMIIt != ThisMBB->end() && isCMOVPseudo(*NextMIIt) &&
34616            (NextMIIt->getOperand(3).getImm() == CC ||
34617             NextMIIt->getOperand(3).getImm() == OppCC)) {
34618       LastCMOV = &*NextMIIt;
34619       NextMIIt = next_nodbg(NextMIIt, ThisMBB->end());
34620     }
34621   }
34622 
34623   // This checks for case 2, but only do this if we didn't already find
34624   // case 1, as indicated by LastCMOV == MI.
34625   if (LastCMOV == &MI && NextMIIt != ThisMBB->end() &&
34626       NextMIIt->getOpcode() == MI.getOpcode() &&
34627       NextMIIt->getOperand(2).getReg() == MI.getOperand(2).getReg() &&
34628       NextMIIt->getOperand(1).getReg() == MI.getOperand(0).getReg() &&
34629       NextMIIt->getOperand(1).isKill()) {
34630     return EmitLoweredCascadedSelect(MI, *NextMIIt, ThisMBB);
34631   }
34632 
34633   const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
34634   MachineFunction *F = ThisMBB->getParent();
34635   MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
34636   MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
34637 
34638   MachineFunction::iterator It = ++ThisMBB->getIterator();
34639   F->insert(It, FalseMBB);
34640   F->insert(It, SinkMBB);
34641 
34642   // Set the call frame size on entry to the new basic blocks.
34643   unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
34644   FalseMBB->setCallFrameSize(CallFrameSize);
34645   SinkMBB->setCallFrameSize(CallFrameSize);
34646 
34647   // If the EFLAGS register isn't dead in the terminator, then claim that it's
34648   // live into the sink and copy blocks.
34649   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
34650   if (!LastCMOV->killsRegister(X86::EFLAGS) &&
34651       !checkAndUpdateEFLAGSKill(LastCMOV, ThisMBB, TRI)) {
34652     FalseMBB->addLiveIn(X86::EFLAGS);
34653     SinkMBB->addLiveIn(X86::EFLAGS);
34654   }
34655 
34656   // Transfer any debug instructions inside the CMOV sequence to the sunk block.
34657   auto DbgRange = llvm::make_range(MachineBasicBlock::iterator(MI),
34658                                    MachineBasicBlock::iterator(LastCMOV));
34659   for (MachineInstr &MI : llvm::make_early_inc_range(DbgRange))
34660     if (MI.isDebugInstr())
34661       SinkMBB->push_back(MI.removeFromParent());
34662 
34663   // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
34664   SinkMBB->splice(SinkMBB->end(), ThisMBB,
34665                   std::next(MachineBasicBlock::iterator(LastCMOV)),
34666                   ThisMBB->end());
34667   SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
34668 
34669   // Fallthrough block for ThisMBB.
34670   ThisMBB->addSuccessor(FalseMBB);
34671   // The true block target of the first (or only) branch is always a SinkMBB.
34672   ThisMBB->addSuccessor(SinkMBB);
34673   // Fallthrough block for FalseMBB.
34674   FalseMBB->addSuccessor(SinkMBB);
34675 
34676   // Create the conditional branch instruction.
34677   BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(CC);
34678 
34679   //  SinkMBB:
34680   //   %Result = phi [ %FalseValue, FalseMBB ], [ %TrueValue, ThisMBB ]
34681   //  ...
34682   MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI);
34683   MachineBasicBlock::iterator MIItEnd =
34684       std::next(MachineBasicBlock::iterator(LastCMOV));
34685   createPHIsForCMOVsInSinkBB(MIItBegin, MIItEnd, ThisMBB, FalseMBB, SinkMBB);
34686 
34687   // Now remove the CMOV(s).
34688   ThisMBB->erase(MIItBegin, MIItEnd);
34689 
34690   return SinkMBB;
34691 }
34692 
34693 static unsigned getSUBriOpcode(bool IsLP64) {
34694   if (IsLP64)
34695     return X86::SUB64ri32;
34696   else
34697     return X86::SUB32ri;
34698 }
34699 
34700 MachineBasicBlock *
34701 X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI,
34702                                            MachineBasicBlock *MBB) const {
34703   MachineFunction *MF = MBB->getParent();
34704   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34705   const X86FrameLowering &TFI = *Subtarget.getFrameLowering();
34706   const MIMetadata MIMD(MI);
34707   const BasicBlock *LLVM_BB = MBB->getBasicBlock();
34708 
34709   const unsigned ProbeSize = getStackProbeSize(*MF);
34710 
34711   MachineRegisterInfo &MRI = MF->getRegInfo();
34712   MachineBasicBlock *testMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34713   MachineBasicBlock *tailMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34714   MachineBasicBlock *blockMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34715 
34716   MachineFunction::iterator MBBIter = ++MBB->getIterator();
34717   MF->insert(MBBIter, testMBB);
34718   MF->insert(MBBIter, blockMBB);
34719   MF->insert(MBBIter, tailMBB);
34720 
34721   Register sizeVReg = MI.getOperand(1).getReg();
34722 
34723   Register physSPReg = TFI.Uses64BitFramePtr ? X86::RSP : X86::ESP;
34724 
34725   Register TmpStackPtr = MRI.createVirtualRegister(
34726       TFI.Uses64BitFramePtr ? &X86::GR64RegClass : &X86::GR32RegClass);
34727   Register FinalStackPtr = MRI.createVirtualRegister(
34728       TFI.Uses64BitFramePtr ? &X86::GR64RegClass : &X86::GR32RegClass);
34729 
34730   BuildMI(*MBB, {MI}, MIMD, TII->get(TargetOpcode::COPY), TmpStackPtr)
34731       .addReg(physSPReg);
34732   {
34733     const unsigned Opc = TFI.Uses64BitFramePtr ? X86::SUB64rr : X86::SUB32rr;
34734     BuildMI(*MBB, {MI}, MIMD, TII->get(Opc), FinalStackPtr)
34735         .addReg(TmpStackPtr)
34736         .addReg(sizeVReg);
34737   }
34738 
34739   // test rsp size
34740 
34741   BuildMI(testMBB, MIMD,
34742           TII->get(TFI.Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
34743       .addReg(FinalStackPtr)
34744       .addReg(physSPReg);
34745 
34746   BuildMI(testMBB, MIMD, TII->get(X86::JCC_1))
34747       .addMBB(tailMBB)
34748       .addImm(X86::COND_GE);
34749   testMBB->addSuccessor(blockMBB);
34750   testMBB->addSuccessor(tailMBB);
34751 
34752   // Touch the block then extend it. This is done on the opposite side of
34753   // static probe where we allocate then touch, to avoid the need of probing the
34754   // tail of the static alloca. Possible scenarios are:
34755   //
34756   //       + ---- <- ------------ <- ------------- <- ------------ +
34757   //       |                                                       |
34758   // [free probe] -> [page alloc] -> [alloc probe] -> [tail alloc] + -> [dyn probe] -> [page alloc] -> [dyn probe] -> [tail alloc] +
34759   //                                                               |                                                               |
34760   //                                                               + <- ----------- <- ------------ <- ----------- <- ------------ +
34761   //
34762   // The property we want to enforce is to never have more than [page alloc] between two probes.
34763 
34764   const unsigned XORMIOpc =
34765       TFI.Uses64BitFramePtr ? X86::XOR64mi32 : X86::XOR32mi;
34766   addRegOffset(BuildMI(blockMBB, MIMD, TII->get(XORMIOpc)), physSPReg, false, 0)
34767       .addImm(0);
34768 
34769   BuildMI(blockMBB, MIMD, TII->get(getSUBriOpcode(TFI.Uses64BitFramePtr)),
34770           physSPReg)
34771       .addReg(physSPReg)
34772       .addImm(ProbeSize);
34773 
34774   BuildMI(blockMBB, MIMD, TII->get(X86::JMP_1)).addMBB(testMBB);
34775   blockMBB->addSuccessor(testMBB);
34776 
34777   // Replace original instruction by the expected stack ptr
34778   BuildMI(tailMBB, MIMD, TII->get(TargetOpcode::COPY),
34779           MI.getOperand(0).getReg())
34780       .addReg(FinalStackPtr);
34781 
34782   tailMBB->splice(tailMBB->end(), MBB,
34783                   std::next(MachineBasicBlock::iterator(MI)), MBB->end());
34784   tailMBB->transferSuccessorsAndUpdatePHIs(MBB);
34785   MBB->addSuccessor(testMBB);
34786 
34787   // Delete the original pseudo instruction.
34788   MI.eraseFromParent();
34789 
34790   // And we're done.
34791   return tailMBB;
34792 }
34793 
34794 MachineBasicBlock *
34795 X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI,
34796                                         MachineBasicBlock *BB) const {
34797   MachineFunction *MF = BB->getParent();
34798   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34799   const MIMetadata MIMD(MI);
34800   const BasicBlock *LLVM_BB = BB->getBasicBlock();
34801 
34802   assert(MF->shouldSplitStack());
34803 
34804   const bool Is64Bit = Subtarget.is64Bit();
34805   const bool IsLP64 = Subtarget.isTarget64BitLP64();
34806 
34807   const unsigned TlsReg = Is64Bit ? X86::FS : X86::GS;
34808   const unsigned TlsOffset = IsLP64 ? 0x70 : Is64Bit ? 0x40 : 0x30;
34809 
34810   // BB:
34811   //  ... [Till the alloca]
34812   // If stacklet is not large enough, jump to mallocMBB
34813   //
34814   // bumpMBB:
34815   //  Allocate by subtracting from RSP
34816   //  Jump to continueMBB
34817   //
34818   // mallocMBB:
34819   //  Allocate by call to runtime
34820   //
34821   // continueMBB:
34822   //  ...
34823   //  [rest of original BB]
34824   //
34825 
34826   MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34827   MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34828   MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34829 
34830   MachineRegisterInfo &MRI = MF->getRegInfo();
34831   const TargetRegisterClass *AddrRegClass =
34832       getRegClassFor(getPointerTy(MF->getDataLayout()));
34833 
34834   Register mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass),
34835            bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass),
34836            tmpSPVReg = MRI.createVirtualRegister(AddrRegClass),
34837            SPLimitVReg = MRI.createVirtualRegister(AddrRegClass),
34838            sizeVReg = MI.getOperand(1).getReg(),
34839            physSPReg =
34840                IsLP64 || Subtarget.isTargetNaCl64() ? X86::RSP : X86::ESP;
34841 
34842   MachineFunction::iterator MBBIter = ++BB->getIterator();
34843 
34844   MF->insert(MBBIter, bumpMBB);
34845   MF->insert(MBBIter, mallocMBB);
34846   MF->insert(MBBIter, continueMBB);
34847 
34848   continueMBB->splice(continueMBB->begin(), BB,
34849                       std::next(MachineBasicBlock::iterator(MI)), BB->end());
34850   continueMBB->transferSuccessorsAndUpdatePHIs(BB);
34851 
34852   // Add code to the main basic block to check if the stack limit has been hit,
34853   // and if so, jump to mallocMBB otherwise to bumpMBB.
34854   BuildMI(BB, MIMD, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
34855   BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
34856     .addReg(tmpSPVReg).addReg(sizeVReg);
34857   BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
34858     .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg)
34859     .addReg(SPLimitVReg);
34860   BuildMI(BB, MIMD, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G);
34861 
34862   // bumpMBB simply decreases the stack pointer, since we know the current
34863   // stacklet has enough space.
34864   BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), physSPReg)
34865     .addReg(SPLimitVReg);
34866   BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
34867     .addReg(SPLimitVReg);
34868   BuildMI(bumpMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
34869 
34870   // Calls into a routine in libgcc to allocate more space from the heap.
34871   const uint32_t *RegMask =
34872       Subtarget.getRegisterInfo()->getCallPreservedMask(*MF, CallingConv::C);
34873   if (IsLP64) {
34874     BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI)
34875       .addReg(sizeVReg);
34876     BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
34877       .addExternalSymbol("__morestack_allocate_stack_space")
34878       .addRegMask(RegMask)
34879       .addReg(X86::RDI, RegState::Implicit)
34880       .addReg(X86::RAX, RegState::ImplicitDefine);
34881   } else if (Is64Bit) {
34882     BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI)
34883       .addReg(sizeVReg);
34884     BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
34885       .addExternalSymbol("__morestack_allocate_stack_space")
34886       .addRegMask(RegMask)
34887       .addReg(X86::EDI, RegState::Implicit)
34888       .addReg(X86::EAX, RegState::ImplicitDefine);
34889   } else {
34890     BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
34891       .addImm(12);
34892     BuildMI(mallocMBB, MIMD, TII->get(X86::PUSH32r)).addReg(sizeVReg);
34893     BuildMI(mallocMBB, MIMD, TII->get(X86::CALLpcrel32))
34894       .addExternalSymbol("__morestack_allocate_stack_space")
34895       .addRegMask(RegMask)
34896       .addReg(X86::EAX, RegState::ImplicitDefine);
34897   }
34898 
34899   if (!Is64Bit)
34900     BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
34901       .addImm(16);
34902 
34903   BuildMI(mallocMBB, MIMD, TII->get(TargetOpcode::COPY), mallocPtrVReg)
34904     .addReg(IsLP64 ? X86::RAX : X86::EAX);
34905   BuildMI(mallocMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
34906 
34907   // Set up the CFG correctly.
34908   BB->addSuccessor(bumpMBB);
34909   BB->addSuccessor(mallocMBB);
34910   mallocMBB->addSuccessor(continueMBB);
34911   bumpMBB->addSuccessor(continueMBB);
34912 
34913   // Take care of the PHI nodes.
34914   BuildMI(*continueMBB, continueMBB->begin(), MIMD, TII->get(X86::PHI),
34915           MI.getOperand(0).getReg())
34916       .addReg(mallocPtrVReg)
34917       .addMBB(mallocMBB)
34918       .addReg(bumpSPPtrVReg)
34919       .addMBB(bumpMBB);
34920 
34921   // Delete the original pseudo instruction.
34922   MI.eraseFromParent();
34923 
34924   // And we're done.
34925   return continueMBB;
34926 }
34927 
34928 MachineBasicBlock *
34929 X86TargetLowering::EmitLoweredCatchRet(MachineInstr &MI,
34930                                        MachineBasicBlock *BB) const {
34931   MachineFunction *MF = BB->getParent();
34932   const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
34933   MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
34934   const MIMetadata MIMD(MI);
34935 
34936   assert(!isAsynchronousEHPersonality(
34937              classifyEHPersonality(MF->getFunction().getPersonalityFn())) &&
34938          "SEH does not use catchret!");
34939 
34940   // Only 32-bit EH needs to worry about manually restoring stack pointers.
34941   if (!Subtarget.is32Bit())
34942     return BB;
34943 
34944   // C++ EH creates a new target block to hold the restore code, and wires up
34945   // the new block to the return destination with a normal JMP_4.
34946   MachineBasicBlock *RestoreMBB =
34947       MF->CreateMachineBasicBlock(BB->getBasicBlock());
34948   assert(BB->succ_size() == 1);
34949   MF->insert(std::next(BB->getIterator()), RestoreMBB);
34950   RestoreMBB->transferSuccessorsAndUpdatePHIs(BB);
34951   BB->addSuccessor(RestoreMBB);
34952   MI.getOperand(0).setMBB(RestoreMBB);
34953 
34954   // Marking this as an EH pad but not a funclet entry block causes PEI to
34955   // restore stack pointers in the block.
34956   RestoreMBB->setIsEHPad(true);
34957 
34958   auto RestoreMBBI = RestoreMBB->begin();
34959   BuildMI(*RestoreMBB, RestoreMBBI, MIMD, TII.get(X86::JMP_4)).addMBB(TargetMBB);
34960   return BB;
34961 }
34962 
34963 MachineBasicBlock *
34964 X86TargetLowering::EmitLoweredTLSAddr(MachineInstr &MI,
34965                                       MachineBasicBlock *BB) const {
34966   // So, here we replace TLSADDR with the sequence:
34967   // adjust_stackdown -> TLSADDR -> adjust_stackup.
34968   // We need this because TLSADDR is lowered into calls
34969   // inside MC, therefore without the two markers shrink-wrapping
34970   // may push the prologue/epilogue pass them.
34971   const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
34972   const MIMetadata MIMD(MI);
34973   MachineFunction &MF = *BB->getParent();
34974 
34975   // Emit CALLSEQ_START right before the instruction.
34976   unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
34977   MachineInstrBuilder CallseqStart =
34978       BuildMI(MF, MIMD, TII.get(AdjStackDown)).addImm(0).addImm(0).addImm(0);
34979   BB->insert(MachineBasicBlock::iterator(MI), CallseqStart);
34980 
34981   // Emit CALLSEQ_END right after the instruction.
34982   // We don't call erase from parent because we want to keep the
34983   // original instruction around.
34984   unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
34985   MachineInstrBuilder CallseqEnd =
34986       BuildMI(MF, MIMD, TII.get(AdjStackUp)).addImm(0).addImm(0);
34987   BB->insertAfter(MachineBasicBlock::iterator(MI), CallseqEnd);
34988 
34989   return BB;
34990 }
34991 
34992 MachineBasicBlock *
34993 X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI,
34994                                       MachineBasicBlock *BB) const {
34995   // This is pretty easy.  We're taking the value that we received from
34996   // our load from the relocation, sticking it in either RDI (x86-64)
34997   // or EAX and doing an indirect call.  The return value will then
34998   // be in the normal return register.
34999   MachineFunction *F = BB->getParent();
35000   const X86InstrInfo *TII = Subtarget.getInstrInfo();
35001   const MIMetadata MIMD(MI);
35002 
35003   assert(Subtarget.isTargetDarwin() && "Darwin only instr emitted?");
35004   assert(MI.getOperand(3).isGlobal() && "This should be a global");
35005 
35006   // Get a register mask for the lowered call.
35007   // FIXME: The 32-bit calls have non-standard calling conventions. Use a
35008   // proper register mask.
35009   const uint32_t *RegMask =
35010       Subtarget.is64Bit() ?
35011       Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask() :
35012       Subtarget.getRegisterInfo()->getCallPreservedMask(*F, CallingConv::C);
35013   if (Subtarget.is64Bit()) {
35014     MachineInstrBuilder MIB =
35015         BuildMI(*BB, MI, MIMD, TII->get(X86::MOV64rm), X86::RDI)
35016             .addReg(X86::RIP)
35017             .addImm(0)
35018             .addReg(0)
35019             .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35020                               MI.getOperand(3).getTargetFlags())
35021             .addReg(0);
35022     MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL64m));
35023     addDirectMem(MIB, X86::RDI);
35024     MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
35025   } else if (!isPositionIndependent()) {
35026     MachineInstrBuilder MIB =
35027         BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX)
35028             .addReg(0)
35029             .addImm(0)
35030             .addReg(0)
35031             .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35032                               MI.getOperand(3).getTargetFlags())
35033             .addReg(0);
35034     MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m));
35035     addDirectMem(MIB, X86::EAX);
35036     MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
35037   } else {
35038     MachineInstrBuilder MIB =
35039         BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX)
35040             .addReg(TII->getGlobalBaseReg(F))
35041             .addImm(0)
35042             .addReg(0)
35043             .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35044                               MI.getOperand(3).getTargetFlags())
35045             .addReg(0);
35046     MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m));
35047     addDirectMem(MIB, X86::EAX);
35048     MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
35049   }
35050 
35051   MI.eraseFromParent(); // The pseudo instruction is gone now.
35052   return BB;
35053 }
35054 
35055 static unsigned getOpcodeForIndirectThunk(unsigned RPOpc) {
35056   switch (RPOpc) {
35057   case X86::INDIRECT_THUNK_CALL32:
35058     return X86::CALLpcrel32;
35059   case X86::INDIRECT_THUNK_CALL64:
35060     return X86::CALL64pcrel32;
35061   case X86::INDIRECT_THUNK_TCRETURN32:
35062     return X86::TCRETURNdi;
35063   case X86::INDIRECT_THUNK_TCRETURN64:
35064     return X86::TCRETURNdi64;
35065   }
35066   llvm_unreachable("not indirect thunk opcode");
35067 }
35068 
35069 static const char *getIndirectThunkSymbol(const X86Subtarget &Subtarget,
35070                                           unsigned Reg) {
35071   if (Subtarget.useRetpolineExternalThunk()) {
35072     // When using an external thunk for retpolines, we pick names that match the
35073     // names GCC happens to use as well. This helps simplify the implementation
35074     // of the thunks for kernels where they have no easy ability to create
35075     // aliases and are doing non-trivial configuration of the thunk's body. For
35076     // example, the Linux kernel will do boot-time hot patching of the thunk
35077     // bodies and cannot easily export aliases of these to loaded modules.
35078     //
35079     // Note that at any point in the future, we may need to change the semantics
35080     // of how we implement retpolines and at that time will likely change the
35081     // name of the called thunk. Essentially, there is no hard guarantee that
35082     // LLVM will generate calls to specific thunks, we merely make a best-effort
35083     // attempt to help out kernels and other systems where duplicating the
35084     // thunks is costly.
35085     switch (Reg) {
35086     case X86::EAX:
35087       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35088       return "__x86_indirect_thunk_eax";
35089     case X86::ECX:
35090       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35091       return "__x86_indirect_thunk_ecx";
35092     case X86::EDX:
35093       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35094       return "__x86_indirect_thunk_edx";
35095     case X86::EDI:
35096       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35097       return "__x86_indirect_thunk_edi";
35098     case X86::R11:
35099       assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35100       return "__x86_indirect_thunk_r11";
35101     }
35102     llvm_unreachable("unexpected reg for external indirect thunk");
35103   }
35104 
35105   if (Subtarget.useRetpolineIndirectCalls() ||
35106       Subtarget.useRetpolineIndirectBranches()) {
35107     // When targeting an internal COMDAT thunk use an LLVM-specific name.
35108     switch (Reg) {
35109     case X86::EAX:
35110       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35111       return "__llvm_retpoline_eax";
35112     case X86::ECX:
35113       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35114       return "__llvm_retpoline_ecx";
35115     case X86::EDX:
35116       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35117       return "__llvm_retpoline_edx";
35118     case X86::EDI:
35119       assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35120       return "__llvm_retpoline_edi";
35121     case X86::R11:
35122       assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35123       return "__llvm_retpoline_r11";
35124     }
35125     llvm_unreachable("unexpected reg for retpoline");
35126   }
35127 
35128   if (Subtarget.useLVIControlFlowIntegrity()) {
35129     assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35130     return "__llvm_lvi_thunk_r11";
35131   }
35132   llvm_unreachable("getIndirectThunkSymbol() invoked without thunk feature");
35133 }
35134 
35135 MachineBasicBlock *
35136 X86TargetLowering::EmitLoweredIndirectThunk(MachineInstr &MI,
35137                                             MachineBasicBlock *BB) const {
35138   // Copy the virtual register into the R11 physical register and
35139   // call the retpoline thunk.
35140   const MIMetadata MIMD(MI);
35141   const X86InstrInfo *TII = Subtarget.getInstrInfo();
35142   Register CalleeVReg = MI.getOperand(0).getReg();
35143   unsigned Opc = getOpcodeForIndirectThunk(MI.getOpcode());
35144 
35145   // Find an available scratch register to hold the callee. On 64-bit, we can
35146   // just use R11, but we scan for uses anyway to ensure we don't generate
35147   // incorrect code. On 32-bit, we use one of EAX, ECX, or EDX that isn't
35148   // already a register use operand to the call to hold the callee. If none
35149   // are available, use EDI instead. EDI is chosen because EBX is the PIC base
35150   // register and ESI is the base pointer to realigned stack frames with VLAs.
35151   SmallVector<unsigned, 3> AvailableRegs;
35152   if (Subtarget.is64Bit())
35153     AvailableRegs.push_back(X86::R11);
35154   else
35155     AvailableRegs.append({X86::EAX, X86::ECX, X86::EDX, X86::EDI});
35156 
35157   // Zero out any registers that are already used.
35158   for (const auto &MO : MI.operands()) {
35159     if (MO.isReg() && MO.isUse())
35160       for (unsigned &Reg : AvailableRegs)
35161         if (Reg == MO.getReg())
35162           Reg = 0;
35163   }
35164 
35165   // Choose the first remaining non-zero available register.
35166   unsigned AvailableReg = 0;
35167   for (unsigned MaybeReg : AvailableRegs) {
35168     if (MaybeReg) {
35169       AvailableReg = MaybeReg;
35170       break;
35171     }
35172   }
35173   if (!AvailableReg)
35174     report_fatal_error("calling convention incompatible with retpoline, no "
35175                        "available registers");
35176 
35177   const char *Symbol = getIndirectThunkSymbol(Subtarget, AvailableReg);
35178 
35179   BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), AvailableReg)
35180       .addReg(CalleeVReg);
35181   MI.getOperand(0).ChangeToES(Symbol);
35182   MI.setDesc(TII->get(Opc));
35183   MachineInstrBuilder(*BB->getParent(), &MI)
35184       .addReg(AvailableReg, RegState::Implicit | RegState::Kill);
35185   return BB;
35186 }
35187 
35188 /// SetJmp implies future control flow change upon calling the corresponding
35189 /// LongJmp.
35190 /// Instead of using the 'return' instruction, the long jump fixes the stack and
35191 /// performs an indirect branch. To do so it uses the registers that were stored
35192 /// in the jump buffer (when calling SetJmp).
35193 /// In case the shadow stack is enabled we need to fix it as well, because some
35194 /// return addresses will be skipped.
35195 /// The function will save the SSP for future fixing in the function
35196 /// emitLongJmpShadowStackFix.
35197 /// \sa emitLongJmpShadowStackFix
35198 /// \param [in] MI The temporary Machine Instruction for the builtin.
35199 /// \param [in] MBB The Machine Basic Block that will be modified.
35200 void X86TargetLowering::emitSetJmpShadowStackFix(MachineInstr &MI,
35201                                                  MachineBasicBlock *MBB) const {
35202   const MIMetadata MIMD(MI);
35203   MachineFunction *MF = MBB->getParent();
35204   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35205   MachineRegisterInfo &MRI = MF->getRegInfo();
35206   MachineInstrBuilder MIB;
35207 
35208   // Memory Reference.
35209   SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
35210                                            MI.memoperands_end());
35211 
35212   // Initialize a register with zero.
35213   MVT PVT = getPointerTy(MF->getDataLayout());
35214   const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
35215   Register ZReg = MRI.createVirtualRegister(PtrRC);
35216   unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr;
35217   BuildMI(*MBB, MI, MIMD, TII->get(XorRROpc))
35218       .addDef(ZReg)
35219       .addReg(ZReg, RegState::Undef)
35220       .addReg(ZReg, RegState::Undef);
35221 
35222   // Read the current SSP Register value to the zeroed register.
35223   Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
35224   unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
35225   BuildMI(*MBB, MI, MIMD, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
35226 
35227   // Write the SSP register value to offset 3 in input memory buffer.
35228   unsigned PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
35229   MIB = BuildMI(*MBB, MI, MIMD, TII->get(PtrStoreOpc));
35230   const int64_t SSPOffset = 3 * PVT.getStoreSize();
35231   const unsigned MemOpndSlot = 1;
35232   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35233     if (i == X86::AddrDisp)
35234       MIB.addDisp(MI.getOperand(MemOpndSlot + i), SSPOffset);
35235     else
35236       MIB.add(MI.getOperand(MemOpndSlot + i));
35237   }
35238   MIB.addReg(SSPCopyReg);
35239   MIB.setMemRefs(MMOs);
35240 }
35241 
35242 MachineBasicBlock *
35243 X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
35244                                     MachineBasicBlock *MBB) const {
35245   const MIMetadata MIMD(MI);
35246   MachineFunction *MF = MBB->getParent();
35247   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35248   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
35249   MachineRegisterInfo &MRI = MF->getRegInfo();
35250 
35251   const BasicBlock *BB = MBB->getBasicBlock();
35252   MachineFunction::iterator I = ++MBB->getIterator();
35253 
35254   // Memory Reference
35255   SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
35256                                            MI.memoperands_end());
35257 
35258   unsigned DstReg;
35259   unsigned MemOpndSlot = 0;
35260 
35261   unsigned CurOp = 0;
35262 
35263   DstReg = MI.getOperand(CurOp++).getReg();
35264   const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
35265   assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
35266   (void)TRI;
35267   Register mainDstReg = MRI.createVirtualRegister(RC);
35268   Register restoreDstReg = MRI.createVirtualRegister(RC);
35269 
35270   MemOpndSlot = CurOp;
35271 
35272   MVT PVT = getPointerTy(MF->getDataLayout());
35273   assert((PVT == MVT::i64 || PVT == MVT::i32) &&
35274          "Invalid Pointer Size!");
35275 
35276   // For v = setjmp(buf), we generate
35277   //
35278   // thisMBB:
35279   //  buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB
35280   //  SjLjSetup restoreMBB
35281   //
35282   // mainMBB:
35283   //  v_main = 0
35284   //
35285   // sinkMBB:
35286   //  v = phi(main, restore)
35287   //
35288   // restoreMBB:
35289   //  if base pointer being used, load it from frame
35290   //  v_restore = 1
35291 
35292   MachineBasicBlock *thisMBB = MBB;
35293   MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
35294   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
35295   MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB);
35296   MF->insert(I, mainMBB);
35297   MF->insert(I, sinkMBB);
35298   MF->push_back(restoreMBB);
35299   restoreMBB->setMachineBlockAddressTaken();
35300 
35301   MachineInstrBuilder MIB;
35302 
35303   // Transfer the remainder of BB and its successor edges to sinkMBB.
35304   sinkMBB->splice(sinkMBB->begin(), MBB,
35305                   std::next(MachineBasicBlock::iterator(MI)), MBB->end());
35306   sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
35307 
35308   // thisMBB:
35309   unsigned PtrStoreOpc = 0;
35310   unsigned LabelReg = 0;
35311   const int64_t LabelOffset = 1 * PVT.getStoreSize();
35312   bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
35313                      !isPositionIndependent();
35314 
35315   // Prepare IP either in reg or imm.
35316   if (!UseImmLabel) {
35317     PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
35318     const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
35319     LabelReg = MRI.createVirtualRegister(PtrRC);
35320     if (Subtarget.is64Bit()) {
35321       MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA64r), LabelReg)
35322               .addReg(X86::RIP)
35323               .addImm(0)
35324               .addReg(0)
35325               .addMBB(restoreMBB)
35326               .addReg(0);
35327     } else {
35328       const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
35329       MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA32r), LabelReg)
35330               .addReg(XII->getGlobalBaseReg(MF))
35331               .addImm(0)
35332               .addReg(0)
35333               .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference())
35334               .addReg(0);
35335     }
35336   } else
35337     PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
35338   // Store IP
35339   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrStoreOpc));
35340   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35341     if (i == X86::AddrDisp)
35342       MIB.addDisp(MI.getOperand(MemOpndSlot + i), LabelOffset);
35343     else
35344       MIB.add(MI.getOperand(MemOpndSlot + i));
35345   }
35346   if (!UseImmLabel)
35347     MIB.addReg(LabelReg);
35348   else
35349     MIB.addMBB(restoreMBB);
35350   MIB.setMemRefs(MMOs);
35351 
35352   if (MF->getMMI().getModule()->getModuleFlag("cf-protection-return")) {
35353     emitSetJmpShadowStackFix(MI, thisMBB);
35354   }
35355 
35356   // Setup
35357   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::EH_SjLj_Setup))
35358           .addMBB(restoreMBB);
35359 
35360   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
35361   MIB.addRegMask(RegInfo->getNoPreservedMask());
35362   thisMBB->addSuccessor(mainMBB);
35363   thisMBB->addSuccessor(restoreMBB);
35364 
35365   // mainMBB:
35366   //  EAX = 0
35367   BuildMI(mainMBB, MIMD, TII->get(X86::MOV32r0), mainDstReg);
35368   mainMBB->addSuccessor(sinkMBB);
35369 
35370   // sinkMBB:
35371   BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
35372       .addReg(mainDstReg)
35373       .addMBB(mainMBB)
35374       .addReg(restoreDstReg)
35375       .addMBB(restoreMBB);
35376 
35377   // restoreMBB:
35378   if (RegInfo->hasBasePointer(*MF)) {
35379     const bool Uses64BitFramePtr =
35380         Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64();
35381     X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
35382     X86FI->setRestoreBasePointer(MF);
35383     Register FramePtr = RegInfo->getFrameRegister(*MF);
35384     Register BasePtr = RegInfo->getBaseRegister();
35385     unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
35386     addRegOffset(BuildMI(restoreMBB, MIMD, TII->get(Opm), BasePtr),
35387                  FramePtr, true, X86FI->getRestoreBasePointerOffset())
35388       .setMIFlag(MachineInstr::FrameSetup);
35389   }
35390   BuildMI(restoreMBB, MIMD, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
35391   BuildMI(restoreMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
35392   restoreMBB->addSuccessor(sinkMBB);
35393 
35394   MI.eraseFromParent();
35395   return sinkMBB;
35396 }
35397 
35398 /// Fix the shadow stack using the previously saved SSP pointer.
35399 /// \sa emitSetJmpShadowStackFix
35400 /// \param [in] MI The temporary Machine Instruction for the builtin.
35401 /// \param [in] MBB The Machine Basic Block that will be modified.
35402 /// \return The sink MBB that will perform the future indirect branch.
35403 MachineBasicBlock *
35404 X86TargetLowering::emitLongJmpShadowStackFix(MachineInstr &MI,
35405                                              MachineBasicBlock *MBB) const {
35406   const MIMetadata MIMD(MI);
35407   MachineFunction *MF = MBB->getParent();
35408   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35409   MachineRegisterInfo &MRI = MF->getRegInfo();
35410 
35411   // Memory Reference
35412   SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
35413                                            MI.memoperands_end());
35414 
35415   MVT PVT = getPointerTy(MF->getDataLayout());
35416   const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
35417 
35418   // checkSspMBB:
35419   //         xor vreg1, vreg1
35420   //         rdssp vreg1
35421   //         test vreg1, vreg1
35422   //         je sinkMBB   # Jump if Shadow Stack is not supported
35423   // fallMBB:
35424   //         mov buf+24/12(%rip), vreg2
35425   //         sub vreg1, vreg2
35426   //         jbe sinkMBB  # No need to fix the Shadow Stack
35427   // fixShadowMBB:
35428   //         shr 3/2, vreg2
35429   //         incssp vreg2  # fix the SSP according to the lower 8 bits
35430   //         shr 8, vreg2
35431   //         je sinkMBB
35432   // fixShadowLoopPrepareMBB:
35433   //         shl vreg2
35434   //         mov 128, vreg3
35435   // fixShadowLoopMBB:
35436   //         incssp vreg3
35437   //         dec vreg2
35438   //         jne fixShadowLoopMBB # Iterate until you finish fixing
35439   //                              # the Shadow Stack
35440   // sinkMBB:
35441 
35442   MachineFunction::iterator I = ++MBB->getIterator();
35443   const BasicBlock *BB = MBB->getBasicBlock();
35444 
35445   MachineBasicBlock *checkSspMBB = MF->CreateMachineBasicBlock(BB);
35446   MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB);
35447   MachineBasicBlock *fixShadowMBB = MF->CreateMachineBasicBlock(BB);
35448   MachineBasicBlock *fixShadowLoopPrepareMBB = MF->CreateMachineBasicBlock(BB);
35449   MachineBasicBlock *fixShadowLoopMBB = MF->CreateMachineBasicBlock(BB);
35450   MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
35451   MF->insert(I, checkSspMBB);
35452   MF->insert(I, fallMBB);
35453   MF->insert(I, fixShadowMBB);
35454   MF->insert(I, fixShadowLoopPrepareMBB);
35455   MF->insert(I, fixShadowLoopMBB);
35456   MF->insert(I, sinkMBB);
35457 
35458   // Transfer the remainder of BB and its successor edges to sinkMBB.
35459   sinkMBB->splice(sinkMBB->begin(), MBB, MachineBasicBlock::iterator(MI),
35460                   MBB->end());
35461   sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
35462 
35463   MBB->addSuccessor(checkSspMBB);
35464 
35465   // Initialize a register with zero.
35466   Register ZReg = MRI.createVirtualRegister(&X86::GR32RegClass);
35467   BuildMI(checkSspMBB, MIMD, TII->get(X86::MOV32r0), ZReg);
35468 
35469   if (PVT == MVT::i64) {
35470     Register TmpZReg = MRI.createVirtualRegister(PtrRC);
35471     BuildMI(checkSspMBB, MIMD, TII->get(X86::SUBREG_TO_REG), TmpZReg)
35472       .addImm(0)
35473       .addReg(ZReg)
35474       .addImm(X86::sub_32bit);
35475     ZReg = TmpZReg;
35476   }
35477 
35478   // Read the current SSP Register value to the zeroed register.
35479   Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
35480   unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
35481   BuildMI(checkSspMBB, MIMD, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
35482 
35483   // Check whether the result of the SSP register is zero and jump directly
35484   // to the sink.
35485   unsigned TestRROpc = (PVT == MVT::i64) ? X86::TEST64rr : X86::TEST32rr;
35486   BuildMI(checkSspMBB, MIMD, TII->get(TestRROpc))
35487       .addReg(SSPCopyReg)
35488       .addReg(SSPCopyReg);
35489   BuildMI(checkSspMBB, MIMD, TII->get(X86::JCC_1))
35490       .addMBB(sinkMBB)
35491       .addImm(X86::COND_E);
35492   checkSspMBB->addSuccessor(sinkMBB);
35493   checkSspMBB->addSuccessor(fallMBB);
35494 
35495   // Reload the previously saved SSP register value.
35496   Register PrevSSPReg = MRI.createVirtualRegister(PtrRC);
35497   unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
35498   const int64_t SPPOffset = 3 * PVT.getStoreSize();
35499   MachineInstrBuilder MIB =
35500       BuildMI(fallMBB, MIMD, TII->get(PtrLoadOpc), PrevSSPReg);
35501   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35502     const MachineOperand &MO = MI.getOperand(i);
35503     if (i == X86::AddrDisp)
35504       MIB.addDisp(MO, SPPOffset);
35505     else if (MO.isReg()) // Don't add the whole operand, we don't want to
35506                          // preserve kill flags.
35507       MIB.addReg(MO.getReg());
35508     else
35509       MIB.add(MO);
35510   }
35511   MIB.setMemRefs(MMOs);
35512 
35513   // Subtract the current SSP from the previous SSP.
35514   Register SspSubReg = MRI.createVirtualRegister(PtrRC);
35515   unsigned SubRROpc = (PVT == MVT::i64) ? X86::SUB64rr : X86::SUB32rr;
35516   BuildMI(fallMBB, MIMD, TII->get(SubRROpc), SspSubReg)
35517       .addReg(PrevSSPReg)
35518       .addReg(SSPCopyReg);
35519 
35520   // Jump to sink in case PrevSSPReg <= SSPCopyReg.
35521   BuildMI(fallMBB, MIMD, TII->get(X86::JCC_1))
35522       .addMBB(sinkMBB)
35523       .addImm(X86::COND_BE);
35524   fallMBB->addSuccessor(sinkMBB);
35525   fallMBB->addSuccessor(fixShadowMBB);
35526 
35527   // Shift right by 2/3 for 32/64 because incssp multiplies the argument by 4/8.
35528   unsigned ShrRIOpc = (PVT == MVT::i64) ? X86::SHR64ri : X86::SHR32ri;
35529   unsigned Offset = (PVT == MVT::i64) ? 3 : 2;
35530   Register SspFirstShrReg = MRI.createVirtualRegister(PtrRC);
35531   BuildMI(fixShadowMBB, MIMD, TII->get(ShrRIOpc), SspFirstShrReg)
35532       .addReg(SspSubReg)
35533       .addImm(Offset);
35534 
35535   // Increase SSP when looking only on the lower 8 bits of the delta.
35536   unsigned IncsspOpc = (PVT == MVT::i64) ? X86::INCSSPQ : X86::INCSSPD;
35537   BuildMI(fixShadowMBB, MIMD, TII->get(IncsspOpc)).addReg(SspFirstShrReg);
35538 
35539   // Reset the lower 8 bits.
35540   Register SspSecondShrReg = MRI.createVirtualRegister(PtrRC);
35541   BuildMI(fixShadowMBB, MIMD, TII->get(ShrRIOpc), SspSecondShrReg)
35542       .addReg(SspFirstShrReg)
35543       .addImm(8);
35544 
35545   // Jump if the result of the shift is zero.
35546   BuildMI(fixShadowMBB, MIMD, TII->get(X86::JCC_1))
35547       .addMBB(sinkMBB)
35548       .addImm(X86::COND_E);
35549   fixShadowMBB->addSuccessor(sinkMBB);
35550   fixShadowMBB->addSuccessor(fixShadowLoopPrepareMBB);
35551 
35552   // Do a single shift left.
35553   unsigned ShlR1Opc = (PVT == MVT::i64) ? X86::SHL64ri : X86::SHL32ri;
35554   Register SspAfterShlReg = MRI.createVirtualRegister(PtrRC);
35555   BuildMI(fixShadowLoopPrepareMBB, MIMD, TII->get(ShlR1Opc), SspAfterShlReg)
35556       .addReg(SspSecondShrReg)
35557       .addImm(1);
35558 
35559   // Save the value 128 to a register (will be used next with incssp).
35560   Register Value128InReg = MRI.createVirtualRegister(PtrRC);
35561   unsigned MovRIOpc = (PVT == MVT::i64) ? X86::MOV64ri32 : X86::MOV32ri;
35562   BuildMI(fixShadowLoopPrepareMBB, MIMD, TII->get(MovRIOpc), Value128InReg)
35563       .addImm(128);
35564   fixShadowLoopPrepareMBB->addSuccessor(fixShadowLoopMBB);
35565 
35566   // Since incssp only looks at the lower 8 bits, we might need to do several
35567   // iterations of incssp until we finish fixing the shadow stack.
35568   Register DecReg = MRI.createVirtualRegister(PtrRC);
35569   Register CounterReg = MRI.createVirtualRegister(PtrRC);
35570   BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::PHI), CounterReg)
35571       .addReg(SspAfterShlReg)
35572       .addMBB(fixShadowLoopPrepareMBB)
35573       .addReg(DecReg)
35574       .addMBB(fixShadowLoopMBB);
35575 
35576   // Every iteration we increase the SSP by 128.
35577   BuildMI(fixShadowLoopMBB, MIMD, TII->get(IncsspOpc)).addReg(Value128InReg);
35578 
35579   // Every iteration we decrement the counter by 1.
35580   unsigned DecROpc = (PVT == MVT::i64) ? X86::DEC64r : X86::DEC32r;
35581   BuildMI(fixShadowLoopMBB, MIMD, TII->get(DecROpc), DecReg).addReg(CounterReg);
35582 
35583   // Jump if the counter is not zero yet.
35584   BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::JCC_1))
35585       .addMBB(fixShadowLoopMBB)
35586       .addImm(X86::COND_NE);
35587   fixShadowLoopMBB->addSuccessor(sinkMBB);
35588   fixShadowLoopMBB->addSuccessor(fixShadowLoopMBB);
35589 
35590   return sinkMBB;
35591 }
35592 
35593 MachineBasicBlock *
35594 X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
35595                                      MachineBasicBlock *MBB) const {
35596   const MIMetadata MIMD(MI);
35597   MachineFunction *MF = MBB->getParent();
35598   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35599   MachineRegisterInfo &MRI = MF->getRegInfo();
35600 
35601   // Memory Reference
35602   SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
35603                                            MI.memoperands_end());
35604 
35605   MVT PVT = getPointerTy(MF->getDataLayout());
35606   assert((PVT == MVT::i64 || PVT == MVT::i32) &&
35607          "Invalid Pointer Size!");
35608 
35609   const TargetRegisterClass *RC =
35610     (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
35611   Register Tmp = MRI.createVirtualRegister(RC);
35612   // Since FP is only updated here but NOT referenced, it's treated as GPR.
35613   const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
35614   Register FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
35615   Register SP = RegInfo->getStackRegister();
35616 
35617   MachineInstrBuilder MIB;
35618 
35619   const int64_t LabelOffset = 1 * PVT.getStoreSize();
35620   const int64_t SPOffset = 2 * PVT.getStoreSize();
35621 
35622   unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
35623   unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;
35624 
35625   MachineBasicBlock *thisMBB = MBB;
35626 
35627   // When CET and shadow stack is enabled, we need to fix the Shadow Stack.
35628   if (MF->getMMI().getModule()->getModuleFlag("cf-protection-return")) {
35629     thisMBB = emitLongJmpShadowStackFix(MI, thisMBB);
35630   }
35631 
35632   // Reload FP
35633   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), FP);
35634   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35635     const MachineOperand &MO = MI.getOperand(i);
35636     if (MO.isReg()) // Don't add the whole operand, we don't want to
35637                     // preserve kill flags.
35638       MIB.addReg(MO.getReg());
35639     else
35640       MIB.add(MO);
35641   }
35642   MIB.setMemRefs(MMOs);
35643 
35644   // Reload IP
35645   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), Tmp);
35646   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35647     const MachineOperand &MO = MI.getOperand(i);
35648     if (i == X86::AddrDisp)
35649       MIB.addDisp(MO, LabelOffset);
35650     else if (MO.isReg()) // Don't add the whole operand, we don't want to
35651                          // preserve kill flags.
35652       MIB.addReg(MO.getReg());
35653     else
35654       MIB.add(MO);
35655   }
35656   MIB.setMemRefs(MMOs);
35657 
35658   // Reload SP
35659   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), SP);
35660   for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35661     if (i == X86::AddrDisp)
35662       MIB.addDisp(MI.getOperand(i), SPOffset);
35663     else
35664       MIB.add(MI.getOperand(i)); // We can preserve the kill flags here, it's
35665                                  // the last instruction of the expansion.
35666   }
35667   MIB.setMemRefs(MMOs);
35668 
35669   // Jump
35670   BuildMI(*thisMBB, MI, MIMD, TII->get(IJmpOpc)).addReg(Tmp);
35671 
35672   MI.eraseFromParent();
35673   return thisMBB;
35674 }
35675 
35676 void X86TargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
35677                                                MachineBasicBlock *MBB,
35678                                                MachineBasicBlock *DispatchBB,
35679                                                int FI) const {
35680   const MIMetadata MIMD(MI);
35681   MachineFunction *MF = MBB->getParent();
35682   MachineRegisterInfo *MRI = &MF->getRegInfo();
35683   const X86InstrInfo *TII = Subtarget.getInstrInfo();
35684 
35685   MVT PVT = getPointerTy(MF->getDataLayout());
35686   assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
35687 
35688   unsigned Op = 0;
35689   unsigned VR = 0;
35690 
35691   bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
35692                      !isPositionIndependent();
35693 
35694   if (UseImmLabel) {
35695     Op = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
35696   } else {
35697     const TargetRegisterClass *TRC =
35698         (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
35699     VR = MRI->createVirtualRegister(TRC);
35700     Op = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
35701 
35702     if (Subtarget.is64Bit())
35703       BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA64r), VR)
35704           .addReg(X86::RIP)
35705           .addImm(1)
35706           .addReg(0)
35707           .addMBB(DispatchBB)
35708           .addReg(0);
35709     else
35710       BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA32r), VR)
35711           .addReg(0) /* TII->getGlobalBaseReg(MF) */
35712           .addImm(1)
35713           .addReg(0)
35714           .addMBB(DispatchBB, Subtarget.classifyBlockAddressReference())
35715           .addReg(0);
35716   }
35717 
35718   MachineInstrBuilder MIB = BuildMI(*MBB, MI, MIMD, TII->get(Op));
35719   addFrameReference(MIB, FI, Subtarget.is64Bit() ? 56 : 36);
35720   if (UseImmLabel)
35721     MIB.addMBB(DispatchBB);
35722   else
35723     MIB.addReg(VR);
35724 }
35725 
35726 MachineBasicBlock *
35727 X86TargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
35728                                          MachineBasicBlock *BB) const {
35729   const MIMetadata MIMD(MI);
35730   MachineFunction *MF = BB->getParent();
35731   MachineRegisterInfo *MRI = &MF->getRegInfo();
35732   const X86InstrInfo *TII = Subtarget.getInstrInfo();
35733   int FI = MF->getFrameInfo().getFunctionContextIndex();
35734 
35735   // Get a mapping of the call site numbers to all of the landing pads they're
35736   // associated with.
35737   DenseMap<unsigned, SmallVector<MachineBasicBlock *, 2>> CallSiteNumToLPad;
35738   unsigned MaxCSNum = 0;
35739   for (auto &MBB : *MF) {
35740     if (!MBB.isEHPad())
35741       continue;
35742 
35743     MCSymbol *Sym = nullptr;
35744     for (const auto &MI : MBB) {
35745       if (MI.isDebugInstr())
35746         continue;
35747 
35748       assert(MI.isEHLabel() && "expected EH_LABEL");
35749       Sym = MI.getOperand(0).getMCSymbol();
35750       break;
35751     }
35752 
35753     if (!MF->hasCallSiteLandingPad(Sym))
35754       continue;
35755 
35756     for (unsigned CSI : MF->getCallSiteLandingPad(Sym)) {
35757       CallSiteNumToLPad[CSI].push_back(&MBB);
35758       MaxCSNum = std::max(MaxCSNum, CSI);
35759     }
35760   }
35761 
35762   // Get an ordered list of the machine basic blocks for the jump table.
35763   std::vector<MachineBasicBlock *> LPadList;
35764   SmallPtrSet<MachineBasicBlock *, 32> InvokeBBs;
35765   LPadList.reserve(CallSiteNumToLPad.size());
35766 
35767   for (unsigned CSI = 1; CSI <= MaxCSNum; ++CSI) {
35768     for (auto &LP : CallSiteNumToLPad[CSI]) {
35769       LPadList.push_back(LP);
35770       InvokeBBs.insert(LP->pred_begin(), LP->pred_end());
35771     }
35772   }
35773 
35774   assert(!LPadList.empty() &&
35775          "No landing pad destinations for the dispatch jump table!");
35776 
35777   // Create the MBBs for the dispatch code.
35778 
35779   // Shove the dispatch's address into the return slot in the function context.
35780   MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
35781   DispatchBB->setIsEHPad(true);
35782 
35783   MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
35784   BuildMI(TrapBB, MIMD, TII->get(X86::TRAP));
35785   DispatchBB->addSuccessor(TrapBB);
35786 
35787   MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
35788   DispatchBB->addSuccessor(DispContBB);
35789 
35790   // Insert MBBs.
35791   MF->push_back(DispatchBB);
35792   MF->push_back(DispContBB);
35793   MF->push_back(TrapBB);
35794 
35795   // Insert code into the entry block that creates and registers the function
35796   // context.
35797   SetupEntryBlockForSjLj(MI, BB, DispatchBB, FI);
35798 
35799   // Create the jump table and associated information
35800   unsigned JTE = getJumpTableEncoding();
35801   MachineJumpTableInfo *JTI = MF->getOrCreateJumpTableInfo(JTE);
35802   unsigned MJTI = JTI->createJumpTableIndex(LPadList);
35803 
35804   const X86RegisterInfo &RI = TII->getRegisterInfo();
35805   // Add a register mask with no preserved registers.  This results in all
35806   // registers being marked as clobbered.
35807   if (RI.hasBasePointer(*MF)) {
35808     const bool FPIs64Bit =
35809         Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64();
35810     X86MachineFunctionInfo *MFI = MF->getInfo<X86MachineFunctionInfo>();
35811     MFI->setRestoreBasePointer(MF);
35812 
35813     Register FP = RI.getFrameRegister(*MF);
35814     Register BP = RI.getBaseRegister();
35815     unsigned Op = FPIs64Bit ? X86::MOV64rm : X86::MOV32rm;
35816     addRegOffset(BuildMI(DispatchBB, MIMD, TII->get(Op), BP), FP, true,
35817                  MFI->getRestoreBasePointerOffset())
35818         .addRegMask(RI.getNoPreservedMask());
35819   } else {
35820     BuildMI(DispatchBB, MIMD, TII->get(X86::NOOP))
35821         .addRegMask(RI.getNoPreservedMask());
35822   }
35823 
35824   // IReg is used as an index in a memory operand and therefore can't be SP
35825   Register IReg = MRI->createVirtualRegister(&X86::GR32_NOSPRegClass);
35826   addFrameReference(BuildMI(DispatchBB, MIMD, TII->get(X86::MOV32rm), IReg), FI,
35827                     Subtarget.is64Bit() ? 8 : 4);
35828   BuildMI(DispatchBB, MIMD, TII->get(X86::CMP32ri))
35829       .addReg(IReg)
35830       .addImm(LPadList.size());
35831   BuildMI(DispatchBB, MIMD, TII->get(X86::JCC_1))
35832       .addMBB(TrapBB)
35833       .addImm(X86::COND_AE);
35834 
35835   if (Subtarget.is64Bit()) {
35836     Register BReg = MRI->createVirtualRegister(&X86::GR64RegClass);
35837     Register IReg64 = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
35838 
35839     // leaq .LJTI0_0(%rip), BReg
35840     BuildMI(DispContBB, MIMD, TII->get(X86::LEA64r), BReg)
35841         .addReg(X86::RIP)
35842         .addImm(1)
35843         .addReg(0)
35844         .addJumpTableIndex(MJTI)
35845         .addReg(0);
35846     // movzx IReg64, IReg
35847     BuildMI(DispContBB, MIMD, TII->get(TargetOpcode::SUBREG_TO_REG), IReg64)
35848         .addImm(0)
35849         .addReg(IReg)
35850         .addImm(X86::sub_32bit);
35851 
35852     switch (JTE) {
35853     case MachineJumpTableInfo::EK_BlockAddress:
35854       // jmpq *(BReg,IReg64,8)
35855       BuildMI(DispContBB, MIMD, TII->get(X86::JMP64m))
35856           .addReg(BReg)
35857           .addImm(8)
35858           .addReg(IReg64)
35859           .addImm(0)
35860           .addReg(0);
35861       break;
35862     case MachineJumpTableInfo::EK_LabelDifference32: {
35863       Register OReg = MRI->createVirtualRegister(&X86::GR32RegClass);
35864       Register OReg64 = MRI->createVirtualRegister(&X86::GR64RegClass);
35865       Register TReg = MRI->createVirtualRegister(&X86::GR64RegClass);
35866 
35867       // movl (BReg,IReg64,4), OReg
35868       BuildMI(DispContBB, MIMD, TII->get(X86::MOV32rm), OReg)
35869           .addReg(BReg)
35870           .addImm(4)
35871           .addReg(IReg64)
35872           .addImm(0)
35873           .addReg(0);
35874       // movsx OReg64, OReg
35875       BuildMI(DispContBB, MIMD, TII->get(X86::MOVSX64rr32), OReg64)
35876           .addReg(OReg);
35877       // addq BReg, OReg64, TReg
35878       BuildMI(DispContBB, MIMD, TII->get(X86::ADD64rr), TReg)
35879           .addReg(OReg64)
35880           .addReg(BReg);
35881       // jmpq *TReg
35882       BuildMI(DispContBB, MIMD, TII->get(X86::JMP64r)).addReg(TReg);
35883       break;
35884     }
35885     default:
35886       llvm_unreachable("Unexpected jump table encoding");
35887     }
35888   } else {
35889     // jmpl *.LJTI0_0(,IReg,4)
35890     BuildMI(DispContBB, MIMD, TII->get(X86::JMP32m))
35891         .addReg(0)
35892         .addImm(4)
35893         .addReg(IReg)
35894         .addJumpTableIndex(MJTI)
35895         .addReg(0);
35896   }
35897 
35898   // Add the jump table entries as successors to the MBB.
35899   SmallPtrSet<MachineBasicBlock *, 8> SeenMBBs;
35900   for (auto &LP : LPadList)
35901     if (SeenMBBs.insert(LP).second)
35902       DispContBB->addSuccessor(LP);
35903 
35904   // N.B. the order the invoke BBs are processed in doesn't matter here.
35905   SmallVector<MachineBasicBlock *, 64> MBBLPads;
35906   const MCPhysReg *SavedRegs = MF->getRegInfo().getCalleeSavedRegs();
35907   for (MachineBasicBlock *MBB : InvokeBBs) {
35908     // Remove the landing pad successor from the invoke block and replace it
35909     // with the new dispatch block.
35910     // Keep a copy of Successors since it's modified inside the loop.
35911     SmallVector<MachineBasicBlock *, 8> Successors(MBB->succ_rbegin(),
35912                                                    MBB->succ_rend());
35913     // FIXME: Avoid quadratic complexity.
35914     for (auto *MBBS : Successors) {
35915       if (MBBS->isEHPad()) {
35916         MBB->removeSuccessor(MBBS);
35917         MBBLPads.push_back(MBBS);
35918       }
35919     }
35920 
35921     MBB->addSuccessor(DispatchBB);
35922 
35923     // Find the invoke call and mark all of the callee-saved registers as
35924     // 'implicit defined' so that they're spilled.  This prevents code from
35925     // moving instructions to before the EH block, where they will never be
35926     // executed.
35927     for (auto &II : reverse(*MBB)) {
35928       if (!II.isCall())
35929         continue;
35930 
35931       DenseMap<unsigned, bool> DefRegs;
35932       for (auto &MOp : II.operands())
35933         if (MOp.isReg())
35934           DefRegs[MOp.getReg()] = true;
35935 
35936       MachineInstrBuilder MIB(*MF, &II);
35937       for (unsigned RegIdx = 0; SavedRegs[RegIdx]; ++RegIdx) {
35938         unsigned Reg = SavedRegs[RegIdx];
35939         if (!DefRegs[Reg])
35940           MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead);
35941       }
35942 
35943       break;
35944     }
35945   }
35946 
35947   // Mark all former landing pads as non-landing pads.  The dispatch is the only
35948   // landing pad now.
35949   for (auto &LP : MBBLPads)
35950     LP->setIsEHPad(false);
35951 
35952   // The instruction is gone now.
35953   MI.eraseFromParent();
35954   return BB;
35955 }
35956 
35957 MachineBasicBlock *
35958 X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
35959                                                MachineBasicBlock *BB) const {
35960   MachineFunction *MF = BB->getParent();
35961   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35962   const MIMetadata MIMD(MI);
35963 
35964   auto TMMImmToTMMReg = [](unsigned Imm) {
35965     assert (Imm < 8 && "Illegal tmm index");
35966     return X86::TMM0 + Imm;
35967   };
35968   switch (MI.getOpcode()) {
35969   default: llvm_unreachable("Unexpected instr type to insert");
35970   case X86::TLS_addr32:
35971   case X86::TLS_addr64:
35972   case X86::TLS_addrX32:
35973   case X86::TLS_base_addr32:
35974   case X86::TLS_base_addr64:
35975   case X86::TLS_base_addrX32:
35976     return EmitLoweredTLSAddr(MI, BB);
35977   case X86::INDIRECT_THUNK_CALL32:
35978   case X86::INDIRECT_THUNK_CALL64:
35979   case X86::INDIRECT_THUNK_TCRETURN32:
35980   case X86::INDIRECT_THUNK_TCRETURN64:
35981     return EmitLoweredIndirectThunk(MI, BB);
35982   case X86::CATCHRET:
35983     return EmitLoweredCatchRet(MI, BB);
35984   case X86::SEG_ALLOCA_32:
35985   case X86::SEG_ALLOCA_64:
35986     return EmitLoweredSegAlloca(MI, BB);
35987   case X86::PROBED_ALLOCA_32:
35988   case X86::PROBED_ALLOCA_64:
35989     return EmitLoweredProbedAlloca(MI, BB);
35990   case X86::TLSCall_32:
35991   case X86::TLSCall_64:
35992     return EmitLoweredTLSCall(MI, BB);
35993   case X86::CMOV_FR16:
35994   case X86::CMOV_FR16X:
35995   case X86::CMOV_FR32:
35996   case X86::CMOV_FR32X:
35997   case X86::CMOV_FR64:
35998   case X86::CMOV_FR64X:
35999   case X86::CMOV_GR8:
36000   case X86::CMOV_GR16:
36001   case X86::CMOV_GR32:
36002   case X86::CMOV_RFP32:
36003   case X86::CMOV_RFP64:
36004   case X86::CMOV_RFP80:
36005   case X86::CMOV_VR64:
36006   case X86::CMOV_VR128:
36007   case X86::CMOV_VR128X:
36008   case X86::CMOV_VR256:
36009   case X86::CMOV_VR256X:
36010   case X86::CMOV_VR512:
36011   case X86::CMOV_VK1:
36012   case X86::CMOV_VK2:
36013   case X86::CMOV_VK4:
36014   case X86::CMOV_VK8:
36015   case X86::CMOV_VK16:
36016   case X86::CMOV_VK32:
36017   case X86::CMOV_VK64:
36018     return EmitLoweredSelect(MI, BB);
36019 
36020   case X86::FP80_ADDr:
36021   case X86::FP80_ADDm32: {
36022     // Change the floating point control register to use double extended
36023     // precision when performing the addition.
36024     int OrigCWFrameIdx =
36025         MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36026     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)),
36027                       OrigCWFrameIdx);
36028 
36029     // Load the old value of the control word...
36030     Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36031     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW),
36032                       OrigCWFrameIdx);
36033 
36034     // OR 0b11 into bit 8 and 9. 0b11 is the encoding for double extended
36035     // precision.
36036     Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36037     BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
36038         .addReg(OldCW, RegState::Kill)
36039         .addImm(0x300);
36040 
36041     // Extract to 16 bits.
36042     Register NewCW16 =
36043         MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
36044     BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16)
36045         .addReg(NewCW, RegState::Kill, X86::sub_16bit);
36046 
36047     // Prepare memory for FLDCW.
36048     int NewCWFrameIdx =
36049         MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36050     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
36051                       NewCWFrameIdx)
36052         .addReg(NewCW16, RegState::Kill);
36053 
36054     // Reload the modified control word now...
36055     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36056                       NewCWFrameIdx);
36057 
36058     // Do the addition.
36059     if (MI.getOpcode() == X86::FP80_ADDr) {
36060       BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80))
36061           .add(MI.getOperand(0))
36062           .add(MI.getOperand(1))
36063           .add(MI.getOperand(2));
36064     } else {
36065       BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80m32))
36066           .add(MI.getOperand(0))
36067           .add(MI.getOperand(1))
36068           .add(MI.getOperand(2))
36069           .add(MI.getOperand(3))
36070           .add(MI.getOperand(4))
36071           .add(MI.getOperand(5))
36072           .add(MI.getOperand(6));
36073     }
36074 
36075     // Reload the original control word now.
36076     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36077                       OrigCWFrameIdx);
36078 
36079     MI.eraseFromParent(); // The pseudo instruction is gone now.
36080     return BB;
36081   }
36082 
36083   case X86::FP32_TO_INT16_IN_MEM:
36084   case X86::FP32_TO_INT32_IN_MEM:
36085   case X86::FP32_TO_INT64_IN_MEM:
36086   case X86::FP64_TO_INT16_IN_MEM:
36087   case X86::FP64_TO_INT32_IN_MEM:
36088   case X86::FP64_TO_INT64_IN_MEM:
36089   case X86::FP80_TO_INT16_IN_MEM:
36090   case X86::FP80_TO_INT32_IN_MEM:
36091   case X86::FP80_TO_INT64_IN_MEM: {
36092     // Change the floating point control register to use "round towards zero"
36093     // mode when truncating to an integer value.
36094     int OrigCWFrameIdx =
36095         MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36096     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)),
36097                       OrigCWFrameIdx);
36098 
36099     // Load the old value of the control word...
36100     Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36101     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW),
36102                       OrigCWFrameIdx);
36103 
36104     // OR 0b11 into bit 10 and 11. 0b11 is the encoding for round toward zero.
36105     Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36106     BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
36107       .addReg(OldCW, RegState::Kill).addImm(0xC00);
36108 
36109     // Extract to 16 bits.
36110     Register NewCW16 =
36111         MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
36112     BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16)
36113       .addReg(NewCW, RegState::Kill, X86::sub_16bit);
36114 
36115     // Prepare memory for FLDCW.
36116     int NewCWFrameIdx =
36117         MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36118     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
36119                       NewCWFrameIdx)
36120       .addReg(NewCW16, RegState::Kill);
36121 
36122     // Reload the modified control word now...
36123     addFrameReference(BuildMI(*BB, MI, MIMD,
36124                               TII->get(X86::FLDCW16m)), NewCWFrameIdx);
36125 
36126     // Get the X86 opcode to use.
36127     unsigned Opc;
36128     switch (MI.getOpcode()) {
36129     default: llvm_unreachable("illegal opcode!");
36130     case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
36131     case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
36132     case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
36133     case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
36134     case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
36135     case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
36136     case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
36137     case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
36138     case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
36139     }
36140 
36141     X86AddressMode AM = getAddressFromInstr(&MI, 0);
36142     addFullAddress(BuildMI(*BB, MI, MIMD, TII->get(Opc)), AM)
36143         .addReg(MI.getOperand(X86::AddrNumOperands).getReg());
36144 
36145     // Reload the original control word now.
36146     addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36147                       OrigCWFrameIdx);
36148 
36149     MI.eraseFromParent(); // The pseudo instruction is gone now.
36150     return BB;
36151   }
36152 
36153   // xbegin
36154   case X86::XBEGIN:
36155     return emitXBegin(MI, BB, Subtarget.getInstrInfo());
36156 
36157   case X86::VAARG_64:
36158   case X86::VAARG_X32:
36159     return EmitVAARGWithCustomInserter(MI, BB);
36160 
36161   case X86::EH_SjLj_SetJmp32:
36162   case X86::EH_SjLj_SetJmp64:
36163     return emitEHSjLjSetJmp(MI, BB);
36164 
36165   case X86::EH_SjLj_LongJmp32:
36166   case X86::EH_SjLj_LongJmp64:
36167     return emitEHSjLjLongJmp(MI, BB);
36168 
36169   case X86::Int_eh_sjlj_setup_dispatch:
36170     return EmitSjLjDispatchBlock(MI, BB);
36171 
36172   case TargetOpcode::STATEPOINT:
36173     // As an implementation detail, STATEPOINT shares the STACKMAP format at
36174     // this point in the process.  We diverge later.
36175     return emitPatchPoint(MI, BB);
36176 
36177   case TargetOpcode::STACKMAP:
36178   case TargetOpcode::PATCHPOINT:
36179     return emitPatchPoint(MI, BB);
36180 
36181   case TargetOpcode::PATCHABLE_EVENT_CALL:
36182   case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
36183     return BB;
36184 
36185   case X86::LCMPXCHG8B: {
36186     const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36187     // In addition to 4 E[ABCD] registers implied by encoding, CMPXCHG8B
36188     // requires a memory operand. If it happens that current architecture is
36189     // i686 and for current function we need a base pointer
36190     // - which is ESI for i686 - register allocator would not be able to
36191     // allocate registers for an address in form of X(%reg, %reg, Y)
36192     // - there never would be enough unreserved registers during regalloc
36193     // (without the need for base ptr the only option would be X(%edi, %esi, Y).
36194     // We are giving a hand to register allocator by precomputing the address in
36195     // a new vreg using LEA.
36196 
36197     // If it is not i686 or there is no base pointer - nothing to do here.
36198     if (!Subtarget.is32Bit() || !TRI->hasBasePointer(*MF))
36199       return BB;
36200 
36201     // Even though this code does not necessarily needs the base pointer to
36202     // be ESI, we check for that. The reason: if this assert fails, there are
36203     // some changes happened in the compiler base pointer handling, which most
36204     // probably have to be addressed somehow here.
36205     assert(TRI->getBaseRegister() == X86::ESI &&
36206            "LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a "
36207            "base pointer in mind");
36208 
36209     MachineRegisterInfo &MRI = MF->getRegInfo();
36210     MVT SPTy = getPointerTy(MF->getDataLayout());
36211     const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
36212     Register computedAddrVReg = MRI.createVirtualRegister(AddrRegClass);
36213 
36214     X86AddressMode AM = getAddressFromInstr(&MI, 0);
36215     // Regalloc does not need any help when the memory operand of CMPXCHG8B
36216     // does not use index register.
36217     if (AM.IndexReg == X86::NoRegister)
36218       return BB;
36219 
36220     // After X86TargetLowering::ReplaceNodeResults CMPXCHG8B is glued to its
36221     // four operand definitions that are E[ABCD] registers. We skip them and
36222     // then insert the LEA.
36223     MachineBasicBlock::reverse_iterator RMBBI(MI.getReverseIterator());
36224     while (RMBBI != BB->rend() && (RMBBI->definesRegister(X86::EAX) ||
36225                                    RMBBI->definesRegister(X86::EBX) ||
36226                                    RMBBI->definesRegister(X86::ECX) ||
36227                                    RMBBI->definesRegister(X86::EDX))) {
36228       ++RMBBI;
36229     }
36230     MachineBasicBlock::iterator MBBI(RMBBI);
36231     addFullAddress(
36232         BuildMI(*BB, *MBBI, MIMD, TII->get(X86::LEA32r), computedAddrVReg), AM);
36233 
36234     setDirectAddressInInstr(&MI, 0, computedAddrVReg);
36235 
36236     return BB;
36237   }
36238   case X86::LCMPXCHG16B_NO_RBX: {
36239     const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36240     Register BasePtr = TRI->getBaseRegister();
36241     if (TRI->hasBasePointer(*MF) &&
36242         (BasePtr == X86::RBX || BasePtr == X86::EBX)) {
36243       if (!BB->isLiveIn(BasePtr))
36244         BB->addLiveIn(BasePtr);
36245       // Save RBX into a virtual register.
36246       Register SaveRBX =
36247           MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36248       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), SaveRBX)
36249           .addReg(X86::RBX);
36250       Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36251       MachineInstrBuilder MIB =
36252           BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B_SAVE_RBX), Dst);
36253       for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx)
36254         MIB.add(MI.getOperand(Idx));
36255       MIB.add(MI.getOperand(X86::AddrNumOperands));
36256       MIB.addReg(SaveRBX);
36257     } else {
36258       // Simple case, just copy the virtual register to RBX.
36259       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::RBX)
36260           .add(MI.getOperand(X86::AddrNumOperands));
36261       MachineInstrBuilder MIB =
36262           BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B));
36263       for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx)
36264         MIB.add(MI.getOperand(Idx));
36265     }
36266     MI.eraseFromParent();
36267     return BB;
36268   }
36269   case X86::MWAITX: {
36270     const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36271     Register BasePtr = TRI->getBaseRegister();
36272     bool IsRBX = (BasePtr == X86::RBX || BasePtr == X86::EBX);
36273     // If no need to save the base pointer, we generate MWAITXrrr,
36274     // else we generate pseudo MWAITX_SAVE_RBX.
36275     if (!IsRBX || !TRI->hasBasePointer(*MF)) {
36276       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX)
36277           .addReg(MI.getOperand(0).getReg());
36278       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX)
36279           .addReg(MI.getOperand(1).getReg());
36280       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EBX)
36281           .addReg(MI.getOperand(2).getReg());
36282       BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITXrrr));
36283       MI.eraseFromParent();
36284     } else {
36285       if (!BB->isLiveIn(BasePtr)) {
36286         BB->addLiveIn(BasePtr);
36287       }
36288       // Parameters can be copied into ECX and EAX but not EBX yet.
36289       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX)
36290           .addReg(MI.getOperand(0).getReg());
36291       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX)
36292           .addReg(MI.getOperand(1).getReg());
36293       assert(Subtarget.is64Bit() && "Expected 64-bit mode!");
36294       // Save RBX into a virtual register.
36295       Register SaveRBX =
36296           MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36297       BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), SaveRBX)
36298           .addReg(X86::RBX);
36299       // Generate mwaitx pseudo.
36300       Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36301       BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITX_SAVE_RBX))
36302           .addDef(Dst) // Destination tied in with SaveRBX.
36303           .addReg(MI.getOperand(2).getReg()) // input value of EBX.
36304           .addUse(SaveRBX);                  // Save of base pointer.
36305       MI.eraseFromParent();
36306     }
36307     return BB;
36308   }
36309   case TargetOpcode::PREALLOCATED_SETUP: {
36310     assert(Subtarget.is32Bit() && "preallocated only used in 32-bit");
36311     auto MFI = MF->getInfo<X86MachineFunctionInfo>();
36312     MFI->setHasPreallocatedCall(true);
36313     int64_t PreallocatedId = MI.getOperand(0).getImm();
36314     size_t StackAdjustment = MFI->getPreallocatedStackSize(PreallocatedId);
36315     assert(StackAdjustment != 0 && "0 stack adjustment");
36316     LLVM_DEBUG(dbgs() << "PREALLOCATED_SETUP stack adjustment "
36317                       << StackAdjustment << "\n");
36318     BuildMI(*BB, MI, MIMD, TII->get(X86::SUB32ri), X86::ESP)
36319         .addReg(X86::ESP)
36320         .addImm(StackAdjustment);
36321     MI.eraseFromParent();
36322     return BB;
36323   }
36324   case TargetOpcode::PREALLOCATED_ARG: {
36325     assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit");
36326     int64_t PreallocatedId = MI.getOperand(1).getImm();
36327     int64_t ArgIdx = MI.getOperand(2).getImm();
36328     auto MFI = MF->getInfo<X86MachineFunctionInfo>();
36329     size_t ArgOffset = MFI->getPreallocatedArgOffsets(PreallocatedId)[ArgIdx];
36330     LLVM_DEBUG(dbgs() << "PREALLOCATED_ARG arg index " << ArgIdx
36331                       << ", arg offset " << ArgOffset << "\n");
36332     // stack pointer + offset
36333     addRegOffset(BuildMI(*BB, MI, MIMD, TII->get(X86::LEA32r),
36334                          MI.getOperand(0).getReg()),
36335                  X86::ESP, false, ArgOffset);
36336     MI.eraseFromParent();
36337     return BB;
36338   }
36339   case X86::PTDPBSSD:
36340   case X86::PTDPBSUD:
36341   case X86::PTDPBUSD:
36342   case X86::PTDPBUUD:
36343   case X86::PTDPBF16PS:
36344   case X86::PTDPFP16PS: {
36345     unsigned Opc;
36346     switch (MI.getOpcode()) {
36347     default: llvm_unreachable("illegal opcode!");
36348     case X86::PTDPBSSD: Opc = X86::TDPBSSD; break;
36349     case X86::PTDPBSUD: Opc = X86::TDPBSUD; break;
36350     case X86::PTDPBUSD: Opc = X86::TDPBUSD; break;
36351     case X86::PTDPBUUD: Opc = X86::TDPBUUD; break;
36352     case X86::PTDPBF16PS: Opc = X86::TDPBF16PS; break;
36353     case X86::PTDPFP16PS: Opc = X86::TDPFP16PS; break;
36354     }
36355 
36356     MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
36357     MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Define);
36358     MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Undef);
36359     MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef);
36360     MIB.addReg(TMMImmToTMMReg(MI.getOperand(2).getImm()), RegState::Undef);
36361 
36362     MI.eraseFromParent(); // The pseudo is gone now.
36363     return BB;
36364   }
36365   case X86::PTILEZERO: {
36366     unsigned Imm = MI.getOperand(0).getImm();
36367     BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
36368     MI.eraseFromParent(); // The pseudo is gone now.
36369     return BB;
36370   }
36371   case X86::PTILELOADD:
36372   case X86::PTILELOADDT1:
36373   case X86::PTILESTORED: {
36374     unsigned Opc;
36375     switch (MI.getOpcode()) {
36376     default: llvm_unreachable("illegal opcode!");
36377 #define GET_EGPR_IF_ENABLED(OPC) (Subtarget.hasEGPR() ? OPC##_EVEX : OPC)
36378     case X86::PTILELOADD:
36379       Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
36380       break;
36381     case X86::PTILELOADDT1:
36382       Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDT1);
36383       break;
36384     case X86::PTILESTORED:
36385       Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED);
36386       break;
36387 #undef GET_EGPR_IF_ENABLED
36388     }
36389 
36390     MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
36391     unsigned CurOp = 0;
36392     if (Opc != X86::TILESTORED && Opc != X86::TILESTORED_EVEX)
36393       MIB.addReg(TMMImmToTMMReg(MI.getOperand(CurOp++).getImm()),
36394                  RegState::Define);
36395 
36396     MIB.add(MI.getOperand(CurOp++)); // base
36397     MIB.add(MI.getOperand(CurOp++)); // scale
36398     MIB.add(MI.getOperand(CurOp++)); // index -- stride
36399     MIB.add(MI.getOperand(CurOp++)); // displacement
36400     MIB.add(MI.getOperand(CurOp++)); // segment
36401 
36402     if (Opc == X86::TILESTORED || Opc == X86::TILESTORED_EVEX)
36403       MIB.addReg(TMMImmToTMMReg(MI.getOperand(CurOp++).getImm()),
36404                  RegState::Undef);
36405 
36406     MI.eraseFromParent(); // The pseudo is gone now.
36407     return BB;
36408   }
36409   case X86::PTCMMIMFP16PS:
36410   case X86::PTCMMRLFP16PS: {
36411     const MIMetadata MIMD(MI);
36412     unsigned Opc;
36413     switch (MI.getOpcode()) {
36414     default: llvm_unreachable("Unexpected instruction!");
36415     case X86::PTCMMIMFP16PS:     Opc = X86::TCMMIMFP16PS;     break;
36416     case X86::PTCMMRLFP16PS:     Opc = X86::TCMMRLFP16PS;     break;
36417     }
36418     MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
36419     MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Define);
36420     MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Undef);
36421     MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef);
36422     MIB.addReg(TMMImmToTMMReg(MI.getOperand(2).getImm()), RegState::Undef);
36423     MI.eraseFromParent(); // The pseudo is gone now.
36424     return BB;
36425   }
36426   }
36427 }
36428 
36429 //===----------------------------------------------------------------------===//
36430 //                           X86 Optimization Hooks
36431 //===----------------------------------------------------------------------===//
36432 
36433 bool
36434 X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
36435                                                 const APInt &DemandedBits,
36436                                                 const APInt &DemandedElts,
36437                                                 TargetLoweringOpt &TLO) const {
36438   EVT VT = Op.getValueType();
36439   unsigned Opcode = Op.getOpcode();
36440   unsigned EltSize = VT.getScalarSizeInBits();
36441 
36442   if (VT.isVector()) {
36443     // If the constant is only all signbits in the active bits, then we should
36444     // extend it to the entire constant to allow it act as a boolean constant
36445     // vector.
36446     auto NeedsSignExtension = [&](SDValue V, unsigned ActiveBits) {
36447       if (!ISD::isBuildVectorOfConstantSDNodes(V.getNode()))
36448         return false;
36449       for (unsigned i = 0, e = V.getNumOperands(); i != e; ++i) {
36450         if (!DemandedElts[i] || V.getOperand(i).isUndef())
36451           continue;
36452         const APInt &Val = V.getConstantOperandAPInt(i);
36453         if (Val.getBitWidth() > Val.getNumSignBits() &&
36454             Val.trunc(ActiveBits).getNumSignBits() == ActiveBits)
36455           return true;
36456       }
36457       return false;
36458     };
36459     // For vectors - if we have a constant, then try to sign extend.
36460     // TODO: Handle AND cases.
36461     unsigned ActiveBits = DemandedBits.getActiveBits();
36462     if (EltSize > ActiveBits && EltSize > 1 && isTypeLegal(VT) &&
36463         (Opcode == ISD::OR || Opcode == ISD::XOR || Opcode == X86ISD::ANDNP) &&
36464         NeedsSignExtension(Op.getOperand(1), ActiveBits)) {
36465       EVT ExtSVT = EVT::getIntegerVT(*TLO.DAG.getContext(), ActiveBits);
36466       EVT ExtVT = EVT::getVectorVT(*TLO.DAG.getContext(), ExtSVT,
36467                                    VT.getVectorNumElements());
36468       SDValue NewC =
36469           TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT,
36470                           Op.getOperand(1), TLO.DAG.getValueType(ExtVT));
36471       SDValue NewOp =
36472           TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC);
36473       return TLO.CombineTo(Op, NewOp);
36474     }
36475     return false;
36476   }
36477 
36478   // Only optimize Ands to prevent shrinking a constant that could be
36479   // matched by movzx.
36480   if (Opcode != ISD::AND)
36481     return false;
36482 
36483   // Make sure the RHS really is a constant.
36484   ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
36485   if (!C)
36486     return false;
36487 
36488   const APInt &Mask = C->getAPIntValue();
36489 
36490   // Clear all non-demanded bits initially.
36491   APInt ShrunkMask = Mask & DemandedBits;
36492 
36493   // Find the width of the shrunk mask.
36494   unsigned Width = ShrunkMask.getActiveBits();
36495 
36496   // If the mask is all 0s there's nothing to do here.
36497   if (Width == 0)
36498     return false;
36499 
36500   // Find the next power of 2 width, rounding up to a byte.
36501   Width = llvm::bit_ceil(std::max(Width, 8U));
36502   // Truncate the width to size to handle illegal types.
36503   Width = std::min(Width, EltSize);
36504 
36505   // Calculate a possible zero extend mask for this constant.
36506   APInt ZeroExtendMask = APInt::getLowBitsSet(EltSize, Width);
36507 
36508   // If we aren't changing the mask, just return true to keep it and prevent
36509   // the caller from optimizing.
36510   if (ZeroExtendMask == Mask)
36511     return true;
36512 
36513   // Make sure the new mask can be represented by a combination of mask bits
36514   // and non-demanded bits.
36515   if (!ZeroExtendMask.isSubsetOf(Mask | ~DemandedBits))
36516     return false;
36517 
36518   // Replace the constant with the zero extend mask.
36519   SDLoc DL(Op);
36520   SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
36521   SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
36522   return TLO.CombineTo(Op, NewOp);
36523 }
36524 
36525 void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
36526                                                       KnownBits &Known,
36527                                                       const APInt &DemandedElts,
36528                                                       const SelectionDAG &DAG,
36529                                                       unsigned Depth) const {
36530   unsigned BitWidth = Known.getBitWidth();
36531   unsigned NumElts = DemandedElts.getBitWidth();
36532   unsigned Opc = Op.getOpcode();
36533   EVT VT = Op.getValueType();
36534   assert((Opc >= ISD::BUILTIN_OP_END ||
36535           Opc == ISD::INTRINSIC_WO_CHAIN ||
36536           Opc == ISD::INTRINSIC_W_CHAIN ||
36537           Opc == ISD::INTRINSIC_VOID) &&
36538          "Should use MaskedValueIsZero if you don't know whether Op"
36539          " is a target node!");
36540 
36541   Known.resetAll();
36542   switch (Opc) {
36543   default: break;
36544   case X86ISD::MUL_IMM: {
36545     KnownBits Known2;
36546     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36547     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36548     Known = KnownBits::mul(Known, Known2);
36549     break;
36550   }
36551   case X86ISD::SETCC:
36552     Known.Zero.setBitsFrom(1);
36553     break;
36554   case X86ISD::MOVMSK: {
36555     unsigned NumLoBits = Op.getOperand(0).getValueType().getVectorNumElements();
36556     Known.Zero.setBitsFrom(NumLoBits);
36557     break;
36558   }
36559   case X86ISD::PEXTRB:
36560   case X86ISD::PEXTRW: {
36561     SDValue Src = Op.getOperand(0);
36562     EVT SrcVT = Src.getValueType();
36563     APInt DemandedElt = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
36564                                             Op.getConstantOperandVal(1));
36565     Known = DAG.computeKnownBits(Src, DemandedElt, Depth + 1);
36566     Known = Known.anyextOrTrunc(BitWidth);
36567     Known.Zero.setBitsFrom(SrcVT.getScalarSizeInBits());
36568     break;
36569   }
36570   case X86ISD::VSRAI:
36571   case X86ISD::VSHLI:
36572   case X86ISD::VSRLI: {
36573     unsigned ShAmt = Op.getConstantOperandVal(1);
36574     if (ShAmt >= VT.getScalarSizeInBits()) {
36575       // Out of range logical bit shifts are guaranteed to be zero.
36576       // Out of range arithmetic bit shifts splat the sign bit.
36577       if (Opc != X86ISD::VSRAI) {
36578         Known.setAllZero();
36579         break;
36580       }
36581 
36582       ShAmt = VT.getScalarSizeInBits() - 1;
36583     }
36584 
36585     Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36586     if (Opc == X86ISD::VSHLI) {
36587       Known.Zero <<= ShAmt;
36588       Known.One <<= ShAmt;
36589       // Low bits are known zero.
36590       Known.Zero.setLowBits(ShAmt);
36591     } else if (Opc == X86ISD::VSRLI) {
36592       Known.Zero.lshrInPlace(ShAmt);
36593       Known.One.lshrInPlace(ShAmt);
36594       // High bits are known zero.
36595       Known.Zero.setHighBits(ShAmt);
36596     } else {
36597       Known.Zero.ashrInPlace(ShAmt);
36598       Known.One.ashrInPlace(ShAmt);
36599     }
36600     break;
36601   }
36602   case X86ISD::PACKUS: {
36603     // PACKUS is just a truncation if the upper half is zero.
36604     APInt DemandedLHS, DemandedRHS;
36605     getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
36606 
36607     Known.One = APInt::getAllOnes(BitWidth * 2);
36608     Known.Zero = APInt::getAllOnes(BitWidth * 2);
36609 
36610     KnownBits Known2;
36611     if (!!DemandedLHS) {
36612       Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedLHS, Depth + 1);
36613       Known = Known.intersectWith(Known2);
36614     }
36615     if (!!DemandedRHS) {
36616       Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedRHS, Depth + 1);
36617       Known = Known.intersectWith(Known2);
36618     }
36619 
36620     if (Known.countMinLeadingZeros() < BitWidth)
36621       Known.resetAll();
36622     Known = Known.trunc(BitWidth);
36623     break;
36624   }
36625   case X86ISD::VBROADCAST: {
36626     SDValue Src = Op.getOperand(0);
36627     if (!Src.getSimpleValueType().isVector()) {
36628       Known = DAG.computeKnownBits(Src, Depth + 1);
36629       return;
36630     }
36631     break;
36632   }
36633   case X86ISD::AND: {
36634     if (Op.getResNo() == 0) {
36635       KnownBits Known2;
36636       Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36637       Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36638       Known &= Known2;
36639     }
36640     break;
36641   }
36642   case X86ISD::ANDNP: {
36643     KnownBits Known2;
36644     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36645     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36646 
36647     // ANDNP = (~X & Y);
36648     Known.One &= Known2.Zero;
36649     Known.Zero |= Known2.One;
36650     break;
36651   }
36652   case X86ISD::FOR: {
36653     KnownBits Known2;
36654     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36655     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36656 
36657     Known |= Known2;
36658     break;
36659   }
36660   case X86ISD::PSADBW: {
36661     assert(VT.getScalarType() == MVT::i64 &&
36662            Op.getOperand(0).getValueType().getScalarType() == MVT::i8 &&
36663            "Unexpected PSADBW types");
36664 
36665     // PSADBW - fills low 16 bits and zeros upper 48 bits of each i64 result.
36666     Known.Zero.setBitsFrom(16);
36667     break;
36668   }
36669   case X86ISD::PCMPGT:
36670   case X86ISD::PCMPEQ: {
36671     KnownBits KnownLhs =
36672         DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36673     KnownBits KnownRhs =
36674         DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36675     std::optional<bool> Res = Opc == X86ISD::PCMPEQ
36676                                   ? KnownBits::eq(KnownLhs, KnownRhs)
36677                                   : KnownBits::sgt(KnownLhs, KnownRhs);
36678     if (Res) {
36679       if (*Res)
36680         Known.setAllOnes();
36681       else
36682         Known.setAllZero();
36683     }
36684     break;
36685   }
36686   case X86ISD::PMULUDQ: {
36687     KnownBits Known2;
36688     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36689     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36690 
36691     Known = Known.trunc(BitWidth / 2).zext(BitWidth);
36692     Known2 = Known2.trunc(BitWidth / 2).zext(BitWidth);
36693     Known = KnownBits::mul(Known, Known2);
36694     break;
36695   }
36696   case X86ISD::CMOV: {
36697     Known = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
36698     // If we don't know any bits, early out.
36699     if (Known.isUnknown())
36700       break;
36701     KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
36702 
36703     // Only known if known in both the LHS and RHS.
36704     Known = Known.intersectWith(Known2);
36705     break;
36706   }
36707   case X86ISD::BEXTR:
36708   case X86ISD::BEXTRI: {
36709     SDValue Op0 = Op.getOperand(0);
36710     SDValue Op1 = Op.getOperand(1);
36711 
36712     if (auto* Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
36713       unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0);
36714       unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8);
36715 
36716       // If the length is 0, the result is 0.
36717       if (Length == 0) {
36718         Known.setAllZero();
36719         break;
36720       }
36721 
36722       if ((Shift + Length) <= BitWidth) {
36723         Known = DAG.computeKnownBits(Op0, Depth + 1);
36724         Known = Known.extractBits(Length, Shift);
36725         Known = Known.zextOrTrunc(BitWidth);
36726       }
36727     }
36728     break;
36729   }
36730   case X86ISD::PDEP: {
36731     KnownBits Known2;
36732     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36733     Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
36734     // Zeros are retained from the mask operand. But not ones.
36735     Known.One.clearAllBits();
36736     // The result will have at least as many trailing zeros as the non-mask
36737     // operand since bits can only map to the same or higher bit position.
36738     Known.Zero.setLowBits(Known2.countMinTrailingZeros());
36739     break;
36740   }
36741   case X86ISD::PEXT: {
36742     Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
36743     // The result has as many leading zeros as the number of zeroes in the mask.
36744     unsigned Count = Known.Zero.popcount();
36745     Known.Zero = APInt::getHighBitsSet(BitWidth, Count);
36746     Known.One.clearAllBits();
36747     break;
36748   }
36749   case X86ISD::VTRUNC:
36750   case X86ISD::VTRUNCS:
36751   case X86ISD::VTRUNCUS:
36752   case X86ISD::CVTSI2P:
36753   case X86ISD::CVTUI2P:
36754   case X86ISD::CVTP2SI:
36755   case X86ISD::CVTP2UI:
36756   case X86ISD::MCVTP2SI:
36757   case X86ISD::MCVTP2UI:
36758   case X86ISD::CVTTP2SI:
36759   case X86ISD::CVTTP2UI:
36760   case X86ISD::MCVTTP2SI:
36761   case X86ISD::MCVTTP2UI:
36762   case X86ISD::MCVTSI2P:
36763   case X86ISD::MCVTUI2P:
36764   case X86ISD::VFPROUND:
36765   case X86ISD::VMFPROUND:
36766   case X86ISD::CVTPS2PH:
36767   case X86ISD::MCVTPS2PH: {
36768     // Truncations/Conversions - upper elements are known zero.
36769     EVT SrcVT = Op.getOperand(0).getValueType();
36770     if (SrcVT.isVector()) {
36771       unsigned NumSrcElts = SrcVT.getVectorNumElements();
36772       if (NumElts > NumSrcElts && DemandedElts.countr_zero() >= NumSrcElts)
36773         Known.setAllZero();
36774     }
36775     break;
36776   }
36777   case X86ISD::STRICT_CVTTP2SI:
36778   case X86ISD::STRICT_CVTTP2UI:
36779   case X86ISD::STRICT_CVTSI2P:
36780   case X86ISD::STRICT_CVTUI2P:
36781   case X86ISD::STRICT_VFPROUND:
36782   case X86ISD::STRICT_CVTPS2PH: {
36783     // Strict Conversions - upper elements are known zero.
36784     EVT SrcVT = Op.getOperand(1).getValueType();
36785     if (SrcVT.isVector()) {
36786       unsigned NumSrcElts = SrcVT.getVectorNumElements();
36787       if (NumElts > NumSrcElts && DemandedElts.countr_zero() >= NumSrcElts)
36788         Known.setAllZero();
36789     }
36790     break;
36791   }
36792   case X86ISD::MOVQ2DQ: {
36793     // Move from MMX to XMM. Upper half of XMM should be 0.
36794     if (DemandedElts.countr_zero() >= (NumElts / 2))
36795       Known.setAllZero();
36796     break;
36797   }
36798   case X86ISD::VBROADCAST_LOAD: {
36799     APInt UndefElts;
36800     SmallVector<APInt, 16> EltBits;
36801     if (getTargetConstantBitsFromNode(Op, BitWidth, UndefElts, EltBits,
36802                                       /*AllowWholeUndefs*/ false,
36803                                       /*AllowPartialUndefs*/ false)) {
36804       Known.Zero.setAllBits();
36805       Known.One.setAllBits();
36806       for (unsigned I = 0; I != NumElts; ++I) {
36807         if (!DemandedElts[I])
36808           continue;
36809         if (UndefElts[I]) {
36810           Known.resetAll();
36811           break;
36812         }
36813         KnownBits Known2 = KnownBits::makeConstant(EltBits[I]);
36814         Known = Known.intersectWith(Known2);
36815       }
36816       return;
36817     }
36818     break;
36819   }
36820   }
36821 
36822   // Handle target shuffles.
36823   // TODO - use resolveTargetShuffleInputs once we can limit recursive depth.
36824   if (isTargetShuffle(Opc)) {
36825     SmallVector<int, 64> Mask;
36826     SmallVector<SDValue, 2> Ops;
36827     if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask)) {
36828       unsigned NumOps = Ops.size();
36829       unsigned NumElts = VT.getVectorNumElements();
36830       if (Mask.size() == NumElts) {
36831         SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
36832         Known.Zero.setAllBits(); Known.One.setAllBits();
36833         for (unsigned i = 0; i != NumElts; ++i) {
36834           if (!DemandedElts[i])
36835             continue;
36836           int M = Mask[i];
36837           if (M == SM_SentinelUndef) {
36838             // For UNDEF elements, we don't know anything about the common state
36839             // of the shuffle result.
36840             Known.resetAll();
36841             break;
36842           }
36843           if (M == SM_SentinelZero) {
36844             Known.One.clearAllBits();
36845             continue;
36846           }
36847           assert(0 <= M && (unsigned)M < (NumOps * NumElts) &&
36848                  "Shuffle index out of range");
36849 
36850           unsigned OpIdx = (unsigned)M / NumElts;
36851           unsigned EltIdx = (unsigned)M % NumElts;
36852           if (Ops[OpIdx].getValueType() != VT) {
36853             // TODO - handle target shuffle ops with different value types.
36854             Known.resetAll();
36855             break;
36856           }
36857           DemandedOps[OpIdx].setBit(EltIdx);
36858         }
36859         // Known bits are the values that are shared by every demanded element.
36860         for (unsigned i = 0; i != NumOps && !Known.isUnknown(); ++i) {
36861           if (!DemandedOps[i])
36862             continue;
36863           KnownBits Known2 =
36864               DAG.computeKnownBits(Ops[i], DemandedOps[i], Depth + 1);
36865           Known = Known.intersectWith(Known2);
36866         }
36867       }
36868     }
36869   }
36870 }
36871 
36872 unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
36873     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
36874     unsigned Depth) const {
36875   EVT VT = Op.getValueType();
36876   unsigned VTBits = VT.getScalarSizeInBits();
36877   unsigned Opcode = Op.getOpcode();
36878   switch (Opcode) {
36879   case X86ISD::SETCC_CARRY:
36880     // SETCC_CARRY sets the dest to ~0 for true or 0 for false.
36881     return VTBits;
36882 
36883   case X86ISD::VTRUNC: {
36884     SDValue Src = Op.getOperand(0);
36885     MVT SrcVT = Src.getSimpleValueType();
36886     unsigned NumSrcBits = SrcVT.getScalarSizeInBits();
36887     assert(VTBits < NumSrcBits && "Illegal truncation input type");
36888     APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
36889     unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
36890     if (Tmp > (NumSrcBits - VTBits))
36891       return Tmp - (NumSrcBits - VTBits);
36892     return 1;
36893   }
36894 
36895   case X86ISD::PACKSS: {
36896     // PACKSS is just a truncation if the sign bits extend to the packed size.
36897     APInt DemandedLHS, DemandedRHS;
36898     getPackDemandedElts(Op.getValueType(), DemandedElts, DemandedLHS,
36899                         DemandedRHS);
36900 
36901     // Helper to detect PACKSSDW(BITCAST(PACKSSDW(X)),BITCAST(PACKSSDW(Y)))
36902     // patterns often used to compact vXi64 allsignbit patterns.
36903     auto NumSignBitsPACKSS = [&](SDValue V, const APInt &Elts) -> unsigned {
36904       SDValue BC = peekThroughBitcasts(V);
36905       if (BC.getOpcode() == X86ISD::PACKSS &&
36906           BC.getScalarValueSizeInBits() == 16 &&
36907           V.getScalarValueSizeInBits() == 32) {
36908         SDValue BC0 = peekThroughBitcasts(BC.getOperand(0));
36909         SDValue BC1 = peekThroughBitcasts(BC.getOperand(1));
36910         if (BC0.getScalarValueSizeInBits() == 64 &&
36911             BC1.getScalarValueSizeInBits() == 64 &&
36912             DAG.ComputeNumSignBits(BC0, Depth + 1) == 64 &&
36913             DAG.ComputeNumSignBits(BC1, Depth + 1) == 64)
36914           return 32;
36915       }
36916       return DAG.ComputeNumSignBits(V, Elts, Depth + 1);
36917     };
36918 
36919     unsigned SrcBits = Op.getOperand(0).getScalarValueSizeInBits();
36920     unsigned Tmp0 = SrcBits, Tmp1 = SrcBits;
36921     if (!!DemandedLHS)
36922       Tmp0 = NumSignBitsPACKSS(Op.getOperand(0), DemandedLHS);
36923     if (!!DemandedRHS)
36924       Tmp1 = NumSignBitsPACKSS(Op.getOperand(1), DemandedRHS);
36925     unsigned Tmp = std::min(Tmp0, Tmp1);
36926     if (Tmp > (SrcBits - VTBits))
36927       return Tmp - (SrcBits - VTBits);
36928     return 1;
36929   }
36930 
36931   case X86ISD::VBROADCAST: {
36932     SDValue Src = Op.getOperand(0);
36933     if (!Src.getSimpleValueType().isVector())
36934       return DAG.ComputeNumSignBits(Src, Depth + 1);
36935     break;
36936   }
36937 
36938   case X86ISD::VSHLI: {
36939     SDValue Src = Op.getOperand(0);
36940     const APInt &ShiftVal = Op.getConstantOperandAPInt(1);
36941     if (ShiftVal.uge(VTBits))
36942       return VTBits; // Shifted all bits out --> zero.
36943     unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
36944     if (ShiftVal.uge(Tmp))
36945       return 1; // Shifted all sign bits out --> unknown.
36946     return Tmp - ShiftVal.getZExtValue();
36947   }
36948 
36949   case X86ISD::VSRAI: {
36950     SDValue Src = Op.getOperand(0);
36951     APInt ShiftVal = Op.getConstantOperandAPInt(1);
36952     if (ShiftVal.uge(VTBits - 1))
36953       return VTBits; // Sign splat.
36954     unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
36955     ShiftVal += Tmp;
36956     return ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
36957   }
36958 
36959   case X86ISD::FSETCC:
36960     // cmpss/cmpsd return zero/all-bits result values in the bottom element.
36961     if (VT == MVT::f32 || VT == MVT::f64 ||
36962         ((VT == MVT::v4f32 || VT == MVT::v2f64) && DemandedElts == 1))
36963       return VTBits;
36964     break;
36965 
36966   case X86ISD::PCMPGT:
36967   case X86ISD::PCMPEQ:
36968   case X86ISD::CMPP:
36969   case X86ISD::VPCOM:
36970   case X86ISD::VPCOMU:
36971     // Vector compares return zero/all-bits result values.
36972     return VTBits;
36973 
36974   case X86ISD::ANDNP: {
36975     unsigned Tmp0 =
36976         DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
36977     if (Tmp0 == 1) return 1; // Early out.
36978     unsigned Tmp1 =
36979         DAG.ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
36980     return std::min(Tmp0, Tmp1);
36981   }
36982 
36983   case X86ISD::CMOV: {
36984     unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1);
36985     if (Tmp0 == 1) return 1;  // Early out.
36986     unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1);
36987     return std::min(Tmp0, Tmp1);
36988   }
36989   }
36990 
36991   // Handle target shuffles.
36992   // TODO - use resolveTargetShuffleInputs once we can limit recursive depth.
36993   if (isTargetShuffle(Opcode)) {
36994     SmallVector<int, 64> Mask;
36995     SmallVector<SDValue, 2> Ops;
36996     if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask)) {
36997       unsigned NumOps = Ops.size();
36998       unsigned NumElts = VT.getVectorNumElements();
36999       if (Mask.size() == NumElts) {
37000         SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
37001         for (unsigned i = 0; i != NumElts; ++i) {
37002           if (!DemandedElts[i])
37003             continue;
37004           int M = Mask[i];
37005           if (M == SM_SentinelUndef) {
37006             // For UNDEF elements, we don't know anything about the common state
37007             // of the shuffle result.
37008             return 1;
37009           } else if (M == SM_SentinelZero) {
37010             // Zero = all sign bits.
37011             continue;
37012           }
37013           assert(0 <= M && (unsigned)M < (NumOps * NumElts) &&
37014                  "Shuffle index out of range");
37015 
37016           unsigned OpIdx = (unsigned)M / NumElts;
37017           unsigned EltIdx = (unsigned)M % NumElts;
37018           if (Ops[OpIdx].getValueType() != VT) {
37019             // TODO - handle target shuffle ops with different value types.
37020             return 1;
37021           }
37022           DemandedOps[OpIdx].setBit(EltIdx);
37023         }
37024         unsigned Tmp0 = VTBits;
37025         for (unsigned i = 0; i != NumOps && Tmp0 > 1; ++i) {
37026           if (!DemandedOps[i])
37027             continue;
37028           unsigned Tmp1 =
37029               DAG.ComputeNumSignBits(Ops[i], DemandedOps[i], Depth + 1);
37030           Tmp0 = std::min(Tmp0, Tmp1);
37031         }
37032         return Tmp0;
37033       }
37034     }
37035   }
37036 
37037   // Fallback case.
37038   return 1;
37039 }
37040 
37041 SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
37042   if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP)
37043     return N->getOperand(0);
37044   return N;
37045 }
37046 
37047 // Helper to look for a normal load that can be narrowed into a vzload with the
37048 // specified VT and memory VT. Returns SDValue() on failure.
37049 static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT,
37050                                   SelectionDAG &DAG) {
37051   // Can't if the load is volatile or atomic.
37052   if (!LN->isSimple())
37053     return SDValue();
37054 
37055   SDVTList Tys = DAG.getVTList(VT, MVT::Other);
37056   SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
37057   return DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, SDLoc(LN), Tys, Ops, MemVT,
37058                                  LN->getPointerInfo(), LN->getOriginalAlign(),
37059                                  LN->getMemOperand()->getFlags());
37060 }
37061 
37062 // Attempt to match a combined shuffle mask against supported unary shuffle
37063 // instructions.
37064 // TODO: Investigate sharing more of this with shuffle lowering.
37065 static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
37066                               bool AllowFloatDomain, bool AllowIntDomain,
37067                               SDValue V1, const SelectionDAG &DAG,
37068                               const X86Subtarget &Subtarget, unsigned &Shuffle,
37069                               MVT &SrcVT, MVT &DstVT) {
37070   unsigned NumMaskElts = Mask.size();
37071   unsigned MaskEltSize = MaskVT.getScalarSizeInBits();
37072 
37073   // Match against a VZEXT_MOVL vXi32 and vXi16 zero-extending instruction.
37074   if (Mask[0] == 0 &&
37075       (MaskEltSize == 32 || (MaskEltSize == 16 && Subtarget.hasFP16()))) {
37076     if ((isUndefOrZero(Mask[1]) && isUndefInRange(Mask, 2, NumMaskElts - 2)) ||
37077         (V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
37078          isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1))) {
37079       Shuffle = X86ISD::VZEXT_MOVL;
37080       if (MaskEltSize == 16)
37081         SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);
37082       else
37083         SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
37084       return true;
37085     }
37086   }
37087 
37088   // Match against a ANY/SIGN/ZERO_EXTEND_VECTOR_INREG instruction.
37089   // TODO: Add 512-bit vector support (split AVX512F and AVX512BW).
37090   if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE41()) ||
37091                          (MaskVT.is256BitVector() && Subtarget.hasInt256()))) {
37092     unsigned MaxScale = 64 / MaskEltSize;
37093     bool UseSign = V1.getScalarValueSizeInBits() == MaskEltSize &&
37094                    DAG.ComputeNumSignBits(V1) == MaskEltSize;
37095     for (unsigned Scale = 2; Scale <= MaxScale; Scale *= 2) {
37096       bool MatchAny = true;
37097       bool MatchZero = true;
37098       bool MatchSign = UseSign;
37099       unsigned NumDstElts = NumMaskElts / Scale;
37100       for (unsigned i = 0;
37101            i != NumDstElts && (MatchAny || MatchSign || MatchZero); ++i) {
37102         if (!isUndefOrEqual(Mask[i * Scale], (int)i)) {
37103           MatchAny = MatchSign = MatchZero = false;
37104           break;
37105         }
37106         unsigned Pos = (i * Scale) + 1;
37107         unsigned Len = Scale - 1;
37108         MatchAny &= isUndefInRange(Mask, Pos, Len);
37109         MatchZero &= isUndefOrZeroInRange(Mask, Pos, Len);
37110         MatchSign &= isUndefOrEqualInRange(Mask, (int)i, Pos, Len);
37111       }
37112       if (MatchAny || MatchSign || MatchZero) {
37113         assert((MatchSign || MatchZero) &&
37114                "Failed to match sext/zext but matched aext?");
37115         unsigned SrcSize = std::max(128u, NumDstElts * MaskEltSize);
37116         MVT ScalarTy = MaskVT.isInteger() ? MaskVT.getScalarType()
37117                                           : MVT::getIntegerVT(MaskEltSize);
37118         SrcVT = MVT::getVectorVT(ScalarTy, SrcSize / MaskEltSize);
37119 
37120         Shuffle = unsigned(
37121             MatchAny ? ISD::ANY_EXTEND
37122                      : (MatchSign ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND));
37123         if (SrcVT.getVectorNumElements() != NumDstElts)
37124           Shuffle = DAG.getOpcode_EXTEND_VECTOR_INREG(Shuffle);
37125 
37126         DstVT = MVT::getIntegerVT(Scale * MaskEltSize);
37127         DstVT = MVT::getVectorVT(DstVT, NumDstElts);
37128         return true;
37129       }
37130     }
37131   }
37132 
37133   // Match against a VZEXT_MOVL instruction, SSE1 only supports 32-bits (MOVSS).
37134   if (((MaskEltSize == 32) || (MaskEltSize == 64 && Subtarget.hasSSE2()) ||
37135        (MaskEltSize == 16 && Subtarget.hasFP16())) &&
37136       isUndefOrEqual(Mask[0], 0) &&
37137       isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1)) {
37138     Shuffle = X86ISD::VZEXT_MOVL;
37139     if (MaskEltSize == 16)
37140       SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);
37141     else
37142       SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
37143     return true;
37144   }
37145 
37146   // Check if we have SSE3 which will let us use MOVDDUP etc. The
37147   // instructions are no slower than UNPCKLPD but has the option to
37148   // fold the input operand into even an unaligned memory load.
37149   if (MaskVT.is128BitVector() && Subtarget.hasSSE3() && AllowFloatDomain) {
37150     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG, V1)) {
37151       Shuffle = X86ISD::MOVDDUP;
37152       SrcVT = DstVT = MVT::v2f64;
37153       return true;
37154     }
37155     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) {
37156       Shuffle = X86ISD::MOVSLDUP;
37157       SrcVT = DstVT = MVT::v4f32;
37158       return true;
37159     }
37160     if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3}, DAG, V1)) {
37161       Shuffle = X86ISD::MOVSHDUP;
37162       SrcVT = DstVT = MVT::v4f32;
37163       return true;
37164     }
37165   }
37166 
37167   if (MaskVT.is256BitVector() && AllowFloatDomain) {
37168     assert(Subtarget.hasAVX() && "AVX required for 256-bit vector shuffles");
37169     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) {
37170       Shuffle = X86ISD::MOVDDUP;
37171       SrcVT = DstVT = MVT::v4f64;
37172       return true;
37173     }
37174     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG,
37175                                   V1)) {
37176       Shuffle = X86ISD::MOVSLDUP;
37177       SrcVT = DstVT = MVT::v8f32;
37178       return true;
37179     }
37180     if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3, 5, 5, 7, 7}, DAG,
37181                                   V1)) {
37182       Shuffle = X86ISD::MOVSHDUP;
37183       SrcVT = DstVT = MVT::v8f32;
37184       return true;
37185     }
37186   }
37187 
37188   if (MaskVT.is512BitVector() && AllowFloatDomain) {
37189     assert(Subtarget.hasAVX512() &&
37190            "AVX512 required for 512-bit vector shuffles");
37191     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG,
37192                                   V1)) {
37193       Shuffle = X86ISD::MOVDDUP;
37194       SrcVT = DstVT = MVT::v8f64;
37195       return true;
37196     }
37197     if (isTargetShuffleEquivalent(
37198             MaskVT, Mask,
37199             {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}, DAG, V1)) {
37200       Shuffle = X86ISD::MOVSLDUP;
37201       SrcVT = DstVT = MVT::v16f32;
37202       return true;
37203     }
37204     if (isTargetShuffleEquivalent(
37205             MaskVT, Mask,
37206             {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15}, DAG, V1)) {
37207       Shuffle = X86ISD::MOVSHDUP;
37208       SrcVT = DstVT = MVT::v16f32;
37209       return true;
37210     }
37211   }
37212 
37213   return false;
37214 }
37215 
37216 // Attempt to match a combined shuffle mask against supported unary immediate
37217 // permute instructions.
37218 // TODO: Investigate sharing more of this with shuffle lowering.
37219 static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
37220                                      const APInt &Zeroable,
37221                                      bool AllowFloatDomain, bool AllowIntDomain,
37222                                      const SelectionDAG &DAG,
37223                                      const X86Subtarget &Subtarget,
37224                                      unsigned &Shuffle, MVT &ShuffleVT,
37225                                      unsigned &PermuteImm) {
37226   unsigned NumMaskElts = Mask.size();
37227   unsigned InputSizeInBits = MaskVT.getSizeInBits();
37228   unsigned MaskScalarSizeInBits = InputSizeInBits / NumMaskElts;
37229   MVT MaskEltVT = MVT::getIntegerVT(MaskScalarSizeInBits);
37230   bool ContainsZeros = isAnyZero(Mask);
37231 
37232   // Handle VPERMI/VPERMILPD vXi64/vXi64 patterns.
37233   if (!ContainsZeros && MaskScalarSizeInBits == 64) {
37234     // Check for lane crossing permutes.
37235     if (is128BitLaneCrossingShuffleMask(MaskEltVT, Mask)) {
37236       // PERMPD/PERMQ permutes within a 256-bit vector (AVX2+).
37237       if (Subtarget.hasAVX2() && MaskVT.is256BitVector()) {
37238         Shuffle = X86ISD::VPERMI;
37239         ShuffleVT = (AllowFloatDomain ? MVT::v4f64 : MVT::v4i64);
37240         PermuteImm = getV4X86ShuffleImm(Mask);
37241         return true;
37242       }
37243       if (Subtarget.hasAVX512() && MaskVT.is512BitVector()) {
37244         SmallVector<int, 4> RepeatedMask;
37245         if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) {
37246           Shuffle = X86ISD::VPERMI;
37247           ShuffleVT = (AllowFloatDomain ? MVT::v8f64 : MVT::v8i64);
37248           PermuteImm = getV4X86ShuffleImm(RepeatedMask);
37249           return true;
37250         }
37251       }
37252     } else if (AllowFloatDomain && Subtarget.hasAVX()) {
37253       // VPERMILPD can permute with a non-repeating shuffle.
37254       Shuffle = X86ISD::VPERMILPI;
37255       ShuffleVT = MVT::getVectorVT(MVT::f64, Mask.size());
37256       PermuteImm = 0;
37257       for (int i = 0, e = Mask.size(); i != e; ++i) {
37258         int M = Mask[i];
37259         if (M == SM_SentinelUndef)
37260           continue;
37261         assert(((M / 2) == (i / 2)) && "Out of range shuffle mask index");
37262         PermuteImm |= (M & 1) << i;
37263       }
37264       return true;
37265     }
37266   }
37267 
37268   // We are checking for shuffle match or shift match. Loop twice so we can
37269   // order which we try and match first depending on target preference.
37270   for (unsigned Order = 0; Order < 2; ++Order) {
37271     if (Subtarget.preferLowerShuffleAsShift() ? (Order == 1) : (Order == 0)) {
37272       // Handle PSHUFD/VPERMILPI vXi32/vXf32 repeated patterns.
37273       // AVX introduced the VPERMILPD/VPERMILPS float permutes, before then we
37274       // had to use 2-input SHUFPD/SHUFPS shuffles (not handled here).
37275       if ((MaskScalarSizeInBits == 64 || MaskScalarSizeInBits == 32) &&
37276           !ContainsZeros && (AllowIntDomain || Subtarget.hasAVX())) {
37277         SmallVector<int, 4> RepeatedMask;
37278         if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
37279           // Narrow the repeated mask to create 32-bit element permutes.
37280           SmallVector<int, 4> WordMask = RepeatedMask;
37281           if (MaskScalarSizeInBits == 64)
37282             narrowShuffleMaskElts(2, RepeatedMask, WordMask);
37283 
37284           Shuffle = (AllowIntDomain ? X86ISD::PSHUFD : X86ISD::VPERMILPI);
37285           ShuffleVT = (AllowIntDomain ? MVT::i32 : MVT::f32);
37286           ShuffleVT = MVT::getVectorVT(ShuffleVT, InputSizeInBits / 32);
37287           PermuteImm = getV4X86ShuffleImm(WordMask);
37288           return true;
37289         }
37290       }
37291 
37292       // Handle PSHUFLW/PSHUFHW vXi16 repeated patterns.
37293       if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16 &&
37294           ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
37295            (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
37296            (MaskVT.is512BitVector() && Subtarget.hasBWI()))) {
37297         SmallVector<int, 4> RepeatedMask;
37298         if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
37299           ArrayRef<int> LoMask(RepeatedMask.data() + 0, 4);
37300           ArrayRef<int> HiMask(RepeatedMask.data() + 4, 4);
37301 
37302           // PSHUFLW: permute lower 4 elements only.
37303           if (isUndefOrInRange(LoMask, 0, 4) &&
37304               isSequentialOrUndefInRange(HiMask, 0, 4, 4)) {
37305             Shuffle = X86ISD::PSHUFLW;
37306             ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);
37307             PermuteImm = getV4X86ShuffleImm(LoMask);
37308             return true;
37309           }
37310 
37311           // PSHUFHW: permute upper 4 elements only.
37312           if (isUndefOrInRange(HiMask, 4, 8) &&
37313               isSequentialOrUndefInRange(LoMask, 0, 4, 0)) {
37314             // Offset the HiMask so that we can create the shuffle immediate.
37315             int OffsetHiMask[4];
37316             for (int i = 0; i != 4; ++i)
37317               OffsetHiMask[i] = (HiMask[i] < 0 ? HiMask[i] : HiMask[i] - 4);
37318 
37319             Shuffle = X86ISD::PSHUFHW;
37320             ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);
37321             PermuteImm = getV4X86ShuffleImm(OffsetHiMask);
37322             return true;
37323           }
37324         }
37325       }
37326     } else {
37327       // Attempt to match against bit rotates.
37328       if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits < 64 &&
37329           ((MaskVT.is128BitVector() && Subtarget.hasXOP()) ||
37330            Subtarget.hasAVX512())) {
37331         int RotateAmt = matchShuffleAsBitRotate(ShuffleVT, MaskScalarSizeInBits,
37332                                                 Subtarget, Mask);
37333         if (0 < RotateAmt) {
37334           Shuffle = X86ISD::VROTLI;
37335           PermuteImm = (unsigned)RotateAmt;
37336           return true;
37337         }
37338       }
37339     }
37340     // Attempt to match against byte/bit shifts.
37341     if (AllowIntDomain &&
37342         ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
37343          (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
37344          (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
37345       int ShiftAmt =
37346           matchShuffleAsShift(ShuffleVT, Shuffle, MaskScalarSizeInBits, Mask, 0,
37347                               Zeroable, Subtarget);
37348       if (0 < ShiftAmt && (!ShuffleVT.is512BitVector() || Subtarget.hasBWI() ||
37349                            32 <= ShuffleVT.getScalarSizeInBits())) {
37350         // Byte shifts can be slower so only match them on second attempt.
37351         if (Order == 0 &&
37352             (Shuffle == X86ISD::VSHLDQ || Shuffle == X86ISD::VSRLDQ))
37353           continue;
37354 
37355         PermuteImm = (unsigned)ShiftAmt;
37356         return true;
37357       }
37358 
37359     }
37360   }
37361 
37362   return false;
37363 }
37364 
37365 // Attempt to match a combined unary shuffle mask against supported binary
37366 // shuffle instructions.
37367 // TODO: Investigate sharing more of this with shuffle lowering.
37368 static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
37369                                bool AllowFloatDomain, bool AllowIntDomain,
37370                                SDValue &V1, SDValue &V2, const SDLoc &DL,
37371                                SelectionDAG &DAG, const X86Subtarget &Subtarget,
37372                                unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
37373                                bool IsUnary) {
37374   unsigned NumMaskElts = Mask.size();
37375   unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
37376   unsigned SizeInBits = MaskVT.getSizeInBits();
37377 
37378   if (MaskVT.is128BitVector()) {
37379     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG) &&
37380         AllowFloatDomain) {
37381       V2 = V1;
37382       V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1);
37383       Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKL : X86ISD::MOVLHPS;
37384       SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;
37385       return true;
37386     }
37387     if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1}, DAG) &&
37388         AllowFloatDomain) {
37389       V2 = V1;
37390       Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKH : X86ISD::MOVHLPS;
37391       SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;
37392       return true;
37393     }
37394     if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 3}, DAG) &&
37395         Subtarget.hasSSE2() && (AllowFloatDomain || !Subtarget.hasSSE41())) {
37396       std::swap(V1, V2);
37397       Shuffle = X86ISD::MOVSD;
37398       SrcVT = DstVT = MVT::v2f64;
37399       return true;
37400     }
37401     if (isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG) &&
37402         (AllowFloatDomain || !Subtarget.hasSSE41())) {
37403       Shuffle = X86ISD::MOVSS;
37404       SrcVT = DstVT = MVT::v4f32;
37405       return true;
37406     }
37407     if (isTargetShuffleEquivalent(MaskVT, Mask, {8, 1, 2, 3, 4, 5, 6, 7},
37408                                   DAG) &&
37409         Subtarget.hasFP16()) {
37410       Shuffle = X86ISD::MOVSH;
37411       SrcVT = DstVT = MVT::v8f16;
37412       return true;
37413     }
37414   }
37415 
37416   // Attempt to match against either an unary or binary PACKSS/PACKUS shuffle.
37417   if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) ||
37418       ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) ||
37419       ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) {
37420     if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG,
37421                              Subtarget)) {
37422       DstVT = MaskVT;
37423       return true;
37424     }
37425   }
37426   // TODO: Can we handle this inside matchShuffleWithPACK?
37427   if (MaskVT == MVT::v4i32 && Subtarget.hasSSE2() &&
37428       isTargetShuffleEquivalent(MaskVT, Mask, {0, 2, 4, 6}, DAG) &&
37429       V1.getScalarValueSizeInBits() == 64 &&
37430       V2.getScalarValueSizeInBits() == 64) {
37431     // Use (SSE41) PACKUSWD if the leading zerobits goto the lowest 16-bits.
37432     unsigned MinLZV1 = DAG.computeKnownBits(V1).countMinLeadingZeros();
37433     unsigned MinLZV2 = DAG.computeKnownBits(V2).countMinLeadingZeros();
37434     if (Subtarget.hasSSE41() && MinLZV1 >= 48 && MinLZV2 >= 48) {
37435       SrcVT = MVT::v4i32;
37436       DstVT = MVT::v8i16;
37437       Shuffle = X86ISD::PACKUS;
37438       return true;
37439     }
37440     // Use PACKUSBW if the leading zerobits goto the lowest 8-bits.
37441     if (MinLZV1 >= 56 && MinLZV2 >= 56) {
37442       SrcVT = MVT::v8i16;
37443       DstVT = MVT::v16i8;
37444       Shuffle = X86ISD::PACKUS;
37445       return true;
37446     }
37447     // Use PACKSSWD if the signbits extend to the lowest 16-bits.
37448     if (DAG.ComputeNumSignBits(V1) > 48 && DAG.ComputeNumSignBits(V2) > 48) {
37449       SrcVT = MVT::v4i32;
37450       DstVT = MVT::v8i16;
37451       Shuffle = X86ISD::PACKSS;
37452       return true;
37453     }
37454   }
37455 
37456   // Attempt to match against either a unary or binary UNPCKL/UNPCKH shuffle.
37457   if ((MaskVT == MVT::v4f32 && Subtarget.hasSSE1()) ||
37458       (MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
37459       (MaskVT.is256BitVector() && 32 <= EltSizeInBits && Subtarget.hasAVX()) ||
37460       (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
37461       (MaskVT.is512BitVector() && Subtarget.hasAVX512() &&
37462        (32 <= EltSizeInBits || Subtarget.hasBWI()))) {
37463     if (matchShuffleWithUNPCK(MaskVT, V1, V2, Shuffle, IsUnary, Mask, DL, DAG,
37464                               Subtarget)) {
37465       SrcVT = DstVT = MaskVT;
37466       if (MaskVT.is256BitVector() && !Subtarget.hasAVX2())
37467         SrcVT = DstVT = (32 == EltSizeInBits ? MVT::v8f32 : MVT::v4f64);
37468       return true;
37469     }
37470   }
37471 
37472   // Attempt to match against a OR if we're performing a blend shuffle and the
37473   // non-blended source element is zero in each case.
37474   // TODO: Handle cases where V1/V2 sizes doesn't match SizeInBits.
37475   if (SizeInBits == V1.getValueSizeInBits() &&
37476       SizeInBits == V2.getValueSizeInBits() &&
37477       (EltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
37478       (EltSizeInBits % V2.getScalarValueSizeInBits()) == 0) {
37479     bool IsBlend = true;
37480     unsigned NumV1Elts = V1.getValueType().getVectorNumElements();
37481     unsigned NumV2Elts = V2.getValueType().getVectorNumElements();
37482     unsigned Scale1 = NumV1Elts / NumMaskElts;
37483     unsigned Scale2 = NumV2Elts / NumMaskElts;
37484     APInt DemandedZeroV1 = APInt::getZero(NumV1Elts);
37485     APInt DemandedZeroV2 = APInt::getZero(NumV2Elts);
37486     for (unsigned i = 0; i != NumMaskElts; ++i) {
37487       int M = Mask[i];
37488       if (M == SM_SentinelUndef)
37489         continue;
37490       if (M == SM_SentinelZero) {
37491         DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
37492         DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
37493         continue;
37494       }
37495       if (M == (int)i) {
37496         DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
37497         continue;
37498       }
37499       if (M == (int)(i + NumMaskElts)) {
37500         DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
37501         continue;
37502       }
37503       IsBlend = false;
37504       break;
37505     }
37506     if (IsBlend) {
37507       if (DAG.MaskedVectorIsZero(V1, DemandedZeroV1) &&
37508           DAG.MaskedVectorIsZero(V2, DemandedZeroV2)) {
37509         Shuffle = ISD::OR;
37510         SrcVT = DstVT = MaskVT.changeTypeToInteger();
37511         return true;
37512       }
37513       if (NumV1Elts == NumV2Elts && NumV1Elts == NumMaskElts) {
37514         // FIXME: handle mismatched sizes?
37515         // TODO: investigate if `ISD::OR` handling in
37516         // `TargetLowering::SimplifyDemandedVectorElts` can be improved instead.
37517         auto computeKnownBitsElementWise = [&DAG](SDValue V) {
37518           unsigned NumElts = V.getValueType().getVectorNumElements();
37519           KnownBits Known(NumElts);
37520           for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
37521             APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
37522             KnownBits PeepholeKnown = DAG.computeKnownBits(V, Mask);
37523             if (PeepholeKnown.isZero())
37524               Known.Zero.setBit(EltIdx);
37525             if (PeepholeKnown.isAllOnes())
37526               Known.One.setBit(EltIdx);
37527           }
37528           return Known;
37529         };
37530 
37531         KnownBits V1Known = computeKnownBitsElementWise(V1);
37532         KnownBits V2Known = computeKnownBitsElementWise(V2);
37533 
37534         for (unsigned i = 0; i != NumMaskElts && IsBlend; ++i) {
37535           int M = Mask[i];
37536           if (M == SM_SentinelUndef)
37537             continue;
37538           if (M == SM_SentinelZero) {
37539             IsBlend &= V1Known.Zero[i] && V2Known.Zero[i];
37540             continue;
37541           }
37542           if (M == (int)i) {
37543             IsBlend &= V2Known.Zero[i] || V1Known.One[i];
37544             continue;
37545           }
37546           if (M == (int)(i + NumMaskElts)) {
37547             IsBlend &= V1Known.Zero[i] || V2Known.One[i];
37548             continue;
37549           }
37550           llvm_unreachable("will not get here.");
37551         }
37552         if (IsBlend) {
37553           Shuffle = ISD::OR;
37554           SrcVT = DstVT = MaskVT.changeTypeToInteger();
37555           return true;
37556         }
37557       }
37558     }
37559   }
37560 
37561   return false;
37562 }
37563 
37564 static bool matchBinaryPermuteShuffle(
37565     MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable,
37566     bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
37567     const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget,
37568     unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) {
37569   unsigned NumMaskElts = Mask.size();
37570   unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
37571 
37572   // Attempt to match against VALIGND/VALIGNQ rotate.
37573   if (AllowIntDomain && (EltSizeInBits == 64 || EltSizeInBits == 32) &&
37574       ((MaskVT.is128BitVector() && Subtarget.hasVLX()) ||
37575        (MaskVT.is256BitVector() && Subtarget.hasVLX()) ||
37576        (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
37577     if (!isAnyZero(Mask)) {
37578       int Rotation = matchShuffleAsElementRotate(V1, V2, Mask);
37579       if (0 < Rotation) {
37580         Shuffle = X86ISD::VALIGN;
37581         if (EltSizeInBits == 64)
37582           ShuffleVT = MVT::getVectorVT(MVT::i64, MaskVT.getSizeInBits() / 64);
37583         else
37584           ShuffleVT = MVT::getVectorVT(MVT::i32, MaskVT.getSizeInBits() / 32);
37585         PermuteImm = Rotation;
37586         return true;
37587       }
37588     }
37589   }
37590 
37591   // Attempt to match against PALIGNR byte rotate.
37592   if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSSE3()) ||
37593                          (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
37594                          (MaskVT.is512BitVector() && Subtarget.hasBWI()))) {
37595     int ByteRotation = matchShuffleAsByteRotate(MaskVT, V1, V2, Mask);
37596     if (0 < ByteRotation) {
37597       Shuffle = X86ISD::PALIGNR;
37598       ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8);
37599       PermuteImm = ByteRotation;
37600       return true;
37601     }
37602   }
37603 
37604   // Attempt to combine to X86ISD::BLENDI.
37605   if ((NumMaskElts <= 8 && ((Subtarget.hasSSE41() && MaskVT.is128BitVector()) ||
37606                             (Subtarget.hasAVX() && MaskVT.is256BitVector()))) ||
37607       (MaskVT == MVT::v16i16 && Subtarget.hasAVX2())) {
37608     uint64_t BlendMask = 0;
37609     bool ForceV1Zero = false, ForceV2Zero = false;
37610     SmallVector<int, 8> TargetMask(Mask);
37611     if (matchShuffleAsBlend(MaskVT, V1, V2, TargetMask, Zeroable, ForceV1Zero,
37612                             ForceV2Zero, BlendMask)) {
37613       if (MaskVT == MVT::v16i16) {
37614         // We can only use v16i16 PBLENDW if the lanes are repeated.
37615         SmallVector<int, 8> RepeatedMask;
37616         if (isRepeatedTargetShuffleMask(128, MaskVT, TargetMask,
37617                                         RepeatedMask)) {
37618           assert(RepeatedMask.size() == 8 &&
37619                  "Repeated mask size doesn't match!");
37620           PermuteImm = 0;
37621           for (int i = 0; i < 8; ++i)
37622             if (RepeatedMask[i] >= 8)
37623               PermuteImm |= 1 << i;
37624           V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
37625           V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
37626           Shuffle = X86ISD::BLENDI;
37627           ShuffleVT = MaskVT;
37628           return true;
37629         }
37630       } else {
37631         V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
37632         V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
37633         PermuteImm = (unsigned)BlendMask;
37634         Shuffle = X86ISD::BLENDI;
37635         ShuffleVT = MaskVT;
37636         return true;
37637       }
37638     }
37639   }
37640 
37641   // Attempt to combine to INSERTPS, but only if it has elements that need to
37642   // be set to zero.
37643   if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
37644       MaskVT.is128BitVector() && isAnyZero(Mask) &&
37645       matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
37646     Shuffle = X86ISD::INSERTPS;
37647     ShuffleVT = MVT::v4f32;
37648     return true;
37649   }
37650 
37651   // Attempt to combine to SHUFPD.
37652   if (AllowFloatDomain && EltSizeInBits == 64 &&
37653       ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
37654        (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
37655        (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
37656     bool ForceV1Zero = false, ForceV2Zero = false;
37657     if (matchShuffleWithSHUFPD(MaskVT, V1, V2, ForceV1Zero, ForceV2Zero,
37658                                PermuteImm, Mask, Zeroable)) {
37659       V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
37660       V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
37661       Shuffle = X86ISD::SHUFP;
37662       ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64);
37663       return true;
37664     }
37665   }
37666 
37667   // Attempt to combine to SHUFPS.
37668   if (AllowFloatDomain && EltSizeInBits == 32 &&
37669       ((MaskVT.is128BitVector() && Subtarget.hasSSE1()) ||
37670        (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
37671        (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
37672     SmallVector<int, 4> RepeatedMask;
37673     if (isRepeatedTargetShuffleMask(128, MaskVT, Mask, RepeatedMask)) {
37674       // Match each half of the repeated mask, to determine if its just
37675       // referencing one of the vectors, is zeroable or entirely undef.
37676       auto MatchHalf = [&](unsigned Offset, int &S0, int &S1) {
37677         int M0 = RepeatedMask[Offset];
37678         int M1 = RepeatedMask[Offset + 1];
37679 
37680         if (isUndefInRange(RepeatedMask, Offset, 2)) {
37681           return DAG.getUNDEF(MaskVT);
37682         } else if (isUndefOrZeroInRange(RepeatedMask, Offset, 2)) {
37683           S0 = (SM_SentinelUndef == M0 ? -1 : 0);
37684           S1 = (SM_SentinelUndef == M1 ? -1 : 1);
37685           return getZeroVector(MaskVT, Subtarget, DAG, DL);
37686         } else if (isUndefOrInRange(M0, 0, 4) && isUndefOrInRange(M1, 0, 4)) {
37687           S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3);
37688           S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3);
37689           return V1;
37690         } else if (isUndefOrInRange(M0, 4, 8) && isUndefOrInRange(M1, 4, 8)) {
37691           S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3);
37692           S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3);
37693           return V2;
37694         }
37695 
37696         return SDValue();
37697       };
37698 
37699       int ShufMask[4] = {-1, -1, -1, -1};
37700       SDValue Lo = MatchHalf(0, ShufMask[0], ShufMask[1]);
37701       SDValue Hi = MatchHalf(2, ShufMask[2], ShufMask[3]);
37702 
37703       if (Lo && Hi) {
37704         V1 = Lo;
37705         V2 = Hi;
37706         Shuffle = X86ISD::SHUFP;
37707         ShuffleVT = MVT::getVectorVT(MVT::f32, MaskVT.getSizeInBits() / 32);
37708         PermuteImm = getV4X86ShuffleImm(ShufMask);
37709         return true;
37710       }
37711     }
37712   }
37713 
37714   // Attempt to combine to INSERTPS more generally if X86ISD::SHUFP failed.
37715   if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
37716       MaskVT.is128BitVector() &&
37717       matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
37718     Shuffle = X86ISD::INSERTPS;
37719     ShuffleVT = MVT::v4f32;
37720     return true;
37721   }
37722 
37723   return false;
37724 }
37725 
37726 static SDValue combineX86ShuffleChainWithExtract(
37727     ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
37728     bool HasVariableMask, bool AllowVariableCrossLaneMask,
37729     bool AllowVariablePerLaneMask, SelectionDAG &DAG,
37730     const X86Subtarget &Subtarget);
37731 
37732 /// Combine an arbitrary chain of shuffles into a single instruction if
37733 /// possible.
37734 ///
37735 /// This is the leaf of the recursive combine below. When we have found some
37736 /// chain of single-use x86 shuffle instructions and accumulated the combined
37737 /// shuffle mask represented by them, this will try to pattern match that mask
37738 /// into either a single instruction if there is a special purpose instruction
37739 /// for this operation, or into a PSHUFB instruction which is a fully general
37740 /// instruction but should only be used to replace chains over a certain depth.
37741 static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
37742                                       ArrayRef<int> BaseMask, int Depth,
37743                                       bool HasVariableMask,
37744                                       bool AllowVariableCrossLaneMask,
37745                                       bool AllowVariablePerLaneMask,
37746                                       SelectionDAG &DAG,
37747                                       const X86Subtarget &Subtarget) {
37748   assert(!BaseMask.empty() && "Cannot combine an empty shuffle mask!");
37749   assert((Inputs.size() == 1 || Inputs.size() == 2) &&
37750          "Unexpected number of shuffle inputs!");
37751 
37752   SDLoc DL(Root);
37753   MVT RootVT = Root.getSimpleValueType();
37754   unsigned RootSizeInBits = RootVT.getSizeInBits();
37755   unsigned NumRootElts = RootVT.getVectorNumElements();
37756 
37757   // Canonicalize shuffle input op to the requested type.
37758   auto CanonicalizeShuffleInput = [&](MVT VT, SDValue Op) {
37759     if (VT.getSizeInBits() > Op.getValueSizeInBits())
37760       Op = widenSubVector(Op, false, Subtarget, DAG, DL, VT.getSizeInBits());
37761     else if (VT.getSizeInBits() < Op.getValueSizeInBits())
37762       Op = extractSubVector(Op, 0, DAG, DL, VT.getSizeInBits());
37763     return DAG.getBitcast(VT, Op);
37764   };
37765 
37766   // Find the inputs that enter the chain. Note that multiple uses are OK
37767   // here, we're not going to remove the operands we find.
37768   bool UnaryShuffle = (Inputs.size() == 1);
37769   SDValue V1 = peekThroughBitcasts(Inputs[0]);
37770   SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType())
37771                              : peekThroughBitcasts(Inputs[1]));
37772 
37773   MVT VT1 = V1.getSimpleValueType();
37774   MVT VT2 = V2.getSimpleValueType();
37775   assert((RootSizeInBits % VT1.getSizeInBits()) == 0 &&
37776          (RootSizeInBits % VT2.getSizeInBits()) == 0 && "Vector size mismatch");
37777 
37778   SDValue Res;
37779 
37780   unsigned NumBaseMaskElts = BaseMask.size();
37781   if (NumBaseMaskElts == 1) {
37782     assert(BaseMask[0] == 0 && "Invalid shuffle index found!");
37783     return CanonicalizeShuffleInput(RootVT, V1);
37784   }
37785 
37786   bool OptForSize = DAG.shouldOptForSize();
37787   unsigned BaseMaskEltSizeInBits = RootSizeInBits / NumBaseMaskElts;
37788   bool FloatDomain = VT1.isFloatingPoint() || VT2.isFloatingPoint() ||
37789                      (RootVT.isFloatingPoint() && Depth >= 1) ||
37790                      (RootVT.is256BitVector() && !Subtarget.hasAVX2());
37791 
37792   // Don't combine if we are a AVX512/EVEX target and the mask element size
37793   // is different from the root element size - this would prevent writemasks
37794   // from being reused.
37795   bool IsMaskedShuffle = false;
37796   if (RootSizeInBits == 512 || (Subtarget.hasVLX() && RootSizeInBits >= 128)) {
37797     if (Root.hasOneUse() && Root->use_begin()->getOpcode() == ISD::VSELECT &&
37798         Root->use_begin()->getOperand(0).getScalarValueSizeInBits() == 1) {
37799       IsMaskedShuffle = true;
37800     }
37801   }
37802 
37803   // If we are shuffling a splat (and not introducing zeros) then we can just
37804   // use it directly. This works for smaller elements as well as they already
37805   // repeat across each mask element.
37806   if (UnaryShuffle && !isAnyZero(BaseMask) &&
37807       V1.getValueSizeInBits() >= RootSizeInBits &&
37808       (BaseMaskEltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
37809       DAG.isSplatValue(V1, /*AllowUndefs*/ false)) {
37810     return CanonicalizeShuffleInput(RootVT, V1);
37811   }
37812 
37813   SmallVector<int, 64> Mask(BaseMask);
37814 
37815   // See if the shuffle is a hidden identity shuffle - repeated args in HOPs
37816   // etc. can be simplified.
37817   if (VT1 == VT2 && VT1.getSizeInBits() == RootSizeInBits && VT1.isVector()) {
37818     SmallVector<int> ScaledMask, IdentityMask;
37819     unsigned NumElts = VT1.getVectorNumElements();
37820     if (Mask.size() <= NumElts &&
37821         scaleShuffleElements(Mask, NumElts, ScaledMask)) {
37822       for (unsigned i = 0; i != NumElts; ++i)
37823         IdentityMask.push_back(i);
37824       if (isTargetShuffleEquivalent(RootVT, ScaledMask, IdentityMask, DAG, V1,
37825                                     V2))
37826         return CanonicalizeShuffleInput(RootVT, V1);
37827     }
37828   }
37829 
37830   // Handle 128/256-bit lane shuffles of 512-bit vectors.
37831   if (RootVT.is512BitVector() &&
37832       (NumBaseMaskElts == 2 || NumBaseMaskElts == 4)) {
37833     // If the upper subvectors are zeroable, then an extract+insert is more
37834     // optimal than using X86ISD::SHUF128. The insertion is free, even if it has
37835     // to zero the upper subvectors.
37836     if (isUndefOrZeroInRange(Mask, 1, NumBaseMaskElts - 1)) {
37837       if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
37838         return SDValue(); // Nothing to do!
37839       assert(isInRange(Mask[0], 0, NumBaseMaskElts) &&
37840              "Unexpected lane shuffle");
37841       Res = CanonicalizeShuffleInput(RootVT, V1);
37842       unsigned SubIdx = Mask[0] * (NumRootElts / NumBaseMaskElts);
37843       bool UseZero = isAnyZero(Mask);
37844       Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits);
37845       return widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits);
37846     }
37847 
37848     // Narrow shuffle mask to v4x128.
37849     SmallVector<int, 4> ScaledMask;
37850     assert((BaseMaskEltSizeInBits % 128) == 0 && "Illegal mask size");
37851     narrowShuffleMaskElts(BaseMaskEltSizeInBits / 128, Mask, ScaledMask);
37852 
37853     // Try to lower to vshuf64x2/vshuf32x4.
37854     auto MatchSHUF128 = [&](MVT ShuffleVT, const SDLoc &DL,
37855                             ArrayRef<int> ScaledMask, SDValue V1, SDValue V2,
37856                             SelectionDAG &DAG) {
37857       int PermMask[4] = {-1, -1, -1, -1};
37858       // Ensure elements came from the same Op.
37859       SDValue Ops[2] = {DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT)};
37860       for (int i = 0; i < 4; ++i) {
37861         assert(ScaledMask[i] >= -1 && "Illegal shuffle sentinel value");
37862         if (ScaledMask[i] < 0)
37863           continue;
37864 
37865         SDValue Op = ScaledMask[i] >= 4 ? V2 : V1;
37866         unsigned OpIndex = i / 2;
37867         if (Ops[OpIndex].isUndef())
37868           Ops[OpIndex] = Op;
37869         else if (Ops[OpIndex] != Op)
37870           return SDValue();
37871 
37872         PermMask[i] = ScaledMask[i] % 4;
37873       }
37874 
37875       return DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
37876                          CanonicalizeShuffleInput(ShuffleVT, Ops[0]),
37877                          CanonicalizeShuffleInput(ShuffleVT, Ops[1]),
37878                          getV4X86ShuffleImm8ForMask(PermMask, DL, DAG));
37879     };
37880 
37881     // FIXME: Is there a better way to do this? is256BitLaneRepeatedShuffleMask
37882     // doesn't work because our mask is for 128 bits and we don't have an MVT
37883     // to match that.
37884     bool PreferPERMQ = UnaryShuffle && isUndefOrInRange(ScaledMask[0], 0, 2) &&
37885                        isUndefOrInRange(ScaledMask[1], 0, 2) &&
37886                        isUndefOrInRange(ScaledMask[2], 2, 4) &&
37887                        isUndefOrInRange(ScaledMask[3], 2, 4) &&
37888                        (ScaledMask[0] < 0 || ScaledMask[2] < 0 ||
37889                         ScaledMask[0] == (ScaledMask[2] % 2)) &&
37890                        (ScaledMask[1] < 0 || ScaledMask[3] < 0 ||
37891                         ScaledMask[1] == (ScaledMask[3] % 2));
37892 
37893     if (!isAnyZero(ScaledMask) && !PreferPERMQ) {
37894       if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128)
37895         return SDValue(); // Nothing to do!
37896       MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64);
37897       if (SDValue V = MatchSHUF128(ShuffleVT, DL, ScaledMask, V1, V2, DAG))
37898         return DAG.getBitcast(RootVT, V);
37899     }
37900   }
37901 
37902   // Handle 128-bit lane shuffles of 256-bit vectors.
37903   if (RootVT.is256BitVector() && NumBaseMaskElts == 2) {
37904     // If the upper half is zeroable, then an extract+insert is more optimal
37905     // than using X86ISD::VPERM2X128. The insertion is free, even if it has to
37906     // zero the upper half.
37907     if (isUndefOrZero(Mask[1])) {
37908       if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
37909         return SDValue(); // Nothing to do!
37910       assert(isInRange(Mask[0], 0, 2) && "Unexpected lane shuffle");
37911       Res = CanonicalizeShuffleInput(RootVT, V1);
37912       Res = extract128BitVector(Res, Mask[0] * (NumRootElts / 2), DAG, DL);
37913       return widenSubVector(Res, Mask[1] == SM_SentinelZero, Subtarget, DAG, DL,
37914                             256);
37915     }
37916 
37917     // If we're inserting the low subvector, an insert-subvector 'concat'
37918     // pattern is quicker than VPERM2X128.
37919     // TODO: Add AVX2 support instead of VPERMQ/VPERMPD.
37920     if (BaseMask[0] == 0 && (BaseMask[1] == 0 || BaseMask[1] == 2) &&
37921         !Subtarget.hasAVX2()) {
37922       if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
37923         return SDValue(); // Nothing to do!
37924       SDValue Lo = CanonicalizeShuffleInput(RootVT, V1);
37925       SDValue Hi = CanonicalizeShuffleInput(RootVT, BaseMask[1] == 0 ? V1 : V2);
37926       Hi = extractSubVector(Hi, 0, DAG, DL, 128);
37927       return insertSubVector(Lo, Hi, NumRootElts / 2, DAG, DL, 128);
37928     }
37929 
37930     if (Depth == 0 && Root.getOpcode() == X86ISD::VPERM2X128)
37931       return SDValue(); // Nothing to do!
37932 
37933     // If we have AVX2, prefer to use VPERMQ/VPERMPD for unary shuffles unless
37934     // we need to use the zeroing feature.
37935     // Prefer blends for sequential shuffles unless we are optimizing for size.
37936     if (UnaryShuffle &&
37937         !(Subtarget.hasAVX2() && isUndefOrInRange(Mask, 0, 2)) &&
37938         (OptForSize || !isSequentialOrUndefOrZeroInRange(Mask, 0, 2, 0))) {
37939       unsigned PermMask = 0;
37940       PermMask |= ((Mask[0] < 0 ? 0x8 : (Mask[0] & 1)) << 0);
37941       PermMask |= ((Mask[1] < 0 ? 0x8 : (Mask[1] & 1)) << 4);
37942       return DAG.getNode(
37943           X86ISD::VPERM2X128, DL, RootVT, CanonicalizeShuffleInput(RootVT, V1),
37944           DAG.getUNDEF(RootVT), DAG.getTargetConstant(PermMask, DL, MVT::i8));
37945     }
37946 
37947     if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128)
37948       return SDValue(); // Nothing to do!
37949 
37950     // TODO - handle AVX512VL cases with X86ISD::SHUF128.
37951     if (!UnaryShuffle && !IsMaskedShuffle) {
37952       assert(llvm::all_of(Mask, [](int M) { return 0 <= M && M < 4; }) &&
37953              "Unexpected shuffle sentinel value");
37954       // Prefer blends to X86ISD::VPERM2X128.
37955       if (!((Mask[0] == 0 && Mask[1] == 3) || (Mask[0] == 2 && Mask[1] == 1))) {
37956         unsigned PermMask = 0;
37957         PermMask |= ((Mask[0] & 3) << 0);
37958         PermMask |= ((Mask[1] & 3) << 4);
37959         SDValue LHS = isInRange(Mask[0], 0, 2) ? V1 : V2;
37960         SDValue RHS = isInRange(Mask[1], 0, 2) ? V1 : V2;
37961         return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT,
37962                           CanonicalizeShuffleInput(RootVT, LHS),
37963                           CanonicalizeShuffleInput(RootVT, RHS),
37964                           DAG.getTargetConstant(PermMask, DL, MVT::i8));
37965       }
37966     }
37967   }
37968 
37969   // For masks that have been widened to 128-bit elements or more,
37970   // narrow back down to 64-bit elements.
37971   if (BaseMaskEltSizeInBits > 64) {
37972     assert((BaseMaskEltSizeInBits % 64) == 0 && "Illegal mask size");
37973     int MaskScale = BaseMaskEltSizeInBits / 64;
37974     SmallVector<int, 64> ScaledMask;
37975     narrowShuffleMaskElts(MaskScale, Mask, ScaledMask);
37976     Mask = std::move(ScaledMask);
37977   }
37978 
37979   // For masked shuffles, we're trying to match the root width for better
37980   // writemask folding, attempt to scale the mask.
37981   // TODO - variable shuffles might need this to be widened again.
37982   if (IsMaskedShuffle && NumRootElts > Mask.size()) {
37983     assert((NumRootElts % Mask.size()) == 0 && "Illegal mask size");
37984     int MaskScale = NumRootElts / Mask.size();
37985     SmallVector<int, 64> ScaledMask;
37986     narrowShuffleMaskElts(MaskScale, Mask, ScaledMask);
37987     Mask = std::move(ScaledMask);
37988   }
37989 
37990   unsigned NumMaskElts = Mask.size();
37991   unsigned MaskEltSizeInBits = RootSizeInBits / NumMaskElts;
37992 
37993   // Determine the effective mask value type.
37994   FloatDomain &= (32 <= MaskEltSizeInBits);
37995   MVT MaskVT = FloatDomain ? MVT::getFloatingPointVT(MaskEltSizeInBits)
37996                            : MVT::getIntegerVT(MaskEltSizeInBits);
37997   MaskVT = MVT::getVectorVT(MaskVT, NumMaskElts);
37998 
37999   // Only allow legal mask types.
38000   if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT))
38001     return SDValue();
38002 
38003   // Attempt to match the mask against known shuffle patterns.
38004   MVT ShuffleSrcVT, ShuffleVT;
38005   unsigned Shuffle, PermuteImm;
38006 
38007   // Which shuffle domains are permitted?
38008   // Permit domain crossing at higher combine depths.
38009   // TODO: Should we indicate which domain is preferred if both are allowed?
38010   bool AllowFloatDomain = FloatDomain || (Depth >= 3);
38011   bool AllowIntDomain = (!FloatDomain || (Depth >= 3)) && Subtarget.hasSSE2() &&
38012                         (!MaskVT.is256BitVector() || Subtarget.hasAVX2());
38013 
38014   // Determine zeroable mask elements.
38015   APInt KnownUndef, KnownZero;
38016   resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
38017   APInt Zeroable = KnownUndef | KnownZero;
38018 
38019   if (UnaryShuffle) {
38020     // Attempt to match against broadcast-from-vector.
38021     // Limit AVX1 to cases where we're loading+broadcasting a scalar element.
38022     if ((Subtarget.hasAVX2() ||
38023          (Subtarget.hasAVX() && 32 <= MaskEltSizeInBits)) &&
38024         (!IsMaskedShuffle || NumRootElts == NumMaskElts)) {
38025       if (isUndefOrEqual(Mask, 0)) {
38026         if (V1.getValueType() == MaskVT &&
38027             V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
38028             X86::mayFoldLoad(V1.getOperand(0), Subtarget)) {
38029           if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST)
38030             return SDValue(); // Nothing to do!
38031           Res = V1.getOperand(0);
38032           Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
38033           return DAG.getBitcast(RootVT, Res);
38034         }
38035         if (Subtarget.hasAVX2()) {
38036           if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST)
38037             return SDValue(); // Nothing to do!
38038           Res = CanonicalizeShuffleInput(MaskVT, V1);
38039           Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
38040           return DAG.getBitcast(RootVT, Res);
38041         }
38042       }
38043     }
38044 
38045     if (matchUnaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, V1,
38046                           DAG, Subtarget, Shuffle, ShuffleSrcVT, ShuffleVT) &&
38047         (!IsMaskedShuffle ||
38048          (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38049       if (Depth == 0 && Root.getOpcode() == Shuffle)
38050         return SDValue(); // Nothing to do!
38051       Res = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
38052       Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res);
38053       return DAG.getBitcast(RootVT, Res);
38054     }
38055 
38056     if (matchUnaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
38057                                  AllowIntDomain, DAG, Subtarget, Shuffle, ShuffleVT,
38058                                  PermuteImm) &&
38059         (!IsMaskedShuffle ||
38060          (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38061       if (Depth == 0 && Root.getOpcode() == Shuffle)
38062         return SDValue(); // Nothing to do!
38063       Res = CanonicalizeShuffleInput(ShuffleVT, V1);
38064       Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res,
38065                         DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38066       return DAG.getBitcast(RootVT, Res);
38067     }
38068   }
38069 
38070   // Attempt to combine to INSERTPS, but only if the inserted element has come
38071   // from a scalar.
38072   // TODO: Handle other insertions here as well?
38073   if (!UnaryShuffle && AllowFloatDomain && RootSizeInBits == 128 &&
38074       Subtarget.hasSSE41() &&
38075       !isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG)) {
38076     if (MaskEltSizeInBits == 32) {
38077       SDValue SrcV1 = V1, SrcV2 = V2;
38078       if (matchShuffleAsInsertPS(SrcV1, SrcV2, PermuteImm, Zeroable, Mask,
38079                                  DAG) &&
38080           SrcV2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
38081         if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTPS)
38082           return SDValue(); // Nothing to do!
38083         Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
38084                           CanonicalizeShuffleInput(MVT::v4f32, SrcV1),
38085                           CanonicalizeShuffleInput(MVT::v4f32, SrcV2),
38086                           DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38087         return DAG.getBitcast(RootVT, Res);
38088       }
38089     }
38090     if (MaskEltSizeInBits == 64 &&
38091         isTargetShuffleEquivalent(MaskVT, Mask, {0, 2}, DAG) &&
38092         V2.getOpcode() == ISD::SCALAR_TO_VECTOR &&
38093         V2.getScalarValueSizeInBits() <= 32) {
38094       if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTPS)
38095         return SDValue(); // Nothing to do!
38096       PermuteImm = (/*DstIdx*/ 2 << 4) | (/*SrcIdx*/ 0 << 0);
38097       Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
38098                         CanonicalizeShuffleInput(MVT::v4f32, V1),
38099                         CanonicalizeShuffleInput(MVT::v4f32, V2),
38100                         DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38101       return DAG.getBitcast(RootVT, Res);
38102     }
38103   }
38104 
38105   SDValue NewV1 = V1; // Save operands in case early exit happens.
38106   SDValue NewV2 = V2;
38107   if (matchBinaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, NewV1,
38108                          NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
38109                          ShuffleVT, UnaryShuffle) &&
38110       (!IsMaskedShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38111     if (Depth == 0 && Root.getOpcode() == Shuffle)
38112       return SDValue(); // Nothing to do!
38113     NewV1 = CanonicalizeShuffleInput(ShuffleSrcVT, NewV1);
38114     NewV2 = CanonicalizeShuffleInput(ShuffleSrcVT, NewV2);
38115     Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2);
38116     return DAG.getBitcast(RootVT, Res);
38117   }
38118 
38119   NewV1 = V1; // Save operands in case early exit happens.
38120   NewV2 = V2;
38121   if (matchBinaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
38122                                 AllowIntDomain, NewV1, NewV2, DL, DAG,
38123                                 Subtarget, Shuffle, ShuffleVT, PermuteImm) &&
38124       (!IsMaskedShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38125     if (Depth == 0 && Root.getOpcode() == Shuffle)
38126       return SDValue(); // Nothing to do!
38127     NewV1 = CanonicalizeShuffleInput(ShuffleVT, NewV1);
38128     NewV2 = CanonicalizeShuffleInput(ShuffleVT, NewV2);
38129     Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2,
38130                       DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38131     return DAG.getBitcast(RootVT, Res);
38132   }
38133 
38134   // Typically from here on, we need an integer version of MaskVT.
38135   MVT IntMaskVT = MVT::getIntegerVT(MaskEltSizeInBits);
38136   IntMaskVT = MVT::getVectorVT(IntMaskVT, NumMaskElts);
38137 
38138   // Annoyingly, SSE4A instructions don't map into the above match helpers.
38139   if (Subtarget.hasSSE4A() && AllowIntDomain && RootSizeInBits == 128) {
38140     uint64_t BitLen, BitIdx;
38141     if (matchShuffleAsEXTRQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx,
38142                             Zeroable)) {
38143       if (Depth == 0 && Root.getOpcode() == X86ISD::EXTRQI)
38144         return SDValue(); // Nothing to do!
38145       V1 = CanonicalizeShuffleInput(IntMaskVT, V1);
38146       Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1,
38147                         DAG.getTargetConstant(BitLen, DL, MVT::i8),
38148                         DAG.getTargetConstant(BitIdx, DL, MVT::i8));
38149       return DAG.getBitcast(RootVT, Res);
38150     }
38151 
38152     if (matchShuffleAsINSERTQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx)) {
38153       if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTQI)
38154         return SDValue(); // Nothing to do!
38155       V1 = CanonicalizeShuffleInput(IntMaskVT, V1);
38156       V2 = CanonicalizeShuffleInput(IntMaskVT, V2);
38157       Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2,
38158                         DAG.getTargetConstant(BitLen, DL, MVT::i8),
38159                         DAG.getTargetConstant(BitIdx, DL, MVT::i8));
38160       return DAG.getBitcast(RootVT, Res);
38161     }
38162   }
38163 
38164   // Match shuffle against TRUNCATE patterns.
38165   if (AllowIntDomain && MaskEltSizeInBits < 64 && Subtarget.hasAVX512()) {
38166     // Match against a VTRUNC instruction, accounting for src/dst sizes.
38167     if (matchShuffleAsVTRUNC(ShuffleSrcVT, ShuffleVT, IntMaskVT, Mask, Zeroable,
38168                              Subtarget)) {
38169       bool IsTRUNCATE = ShuffleVT.getVectorNumElements() ==
38170                         ShuffleSrcVT.getVectorNumElements();
38171       unsigned Opc =
38172           IsTRUNCATE ? (unsigned)ISD::TRUNCATE : (unsigned)X86ISD::VTRUNC;
38173       if (Depth == 0 && Root.getOpcode() == Opc)
38174         return SDValue(); // Nothing to do!
38175       V1 = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
38176       Res = DAG.getNode(Opc, DL, ShuffleVT, V1);
38177       if (ShuffleVT.getSizeInBits() < RootSizeInBits)
38178         Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits);
38179       return DAG.getBitcast(RootVT, Res);
38180     }
38181 
38182     // Do we need a more general binary truncation pattern?
38183     if (RootSizeInBits < 512 &&
38184         ((RootVT.is256BitVector() && Subtarget.useAVX512Regs()) ||
38185          (RootVT.is128BitVector() && Subtarget.hasVLX())) &&
38186         (MaskEltSizeInBits > 8 || Subtarget.hasBWI()) &&
38187         isSequentialOrUndefInRange(Mask, 0, NumMaskElts, 0, 2)) {
38188       // Bail if this was already a truncation or PACK node.
38189       // We sometimes fail to match PACK if we demand known undef elements.
38190       if (Depth == 0 && (Root.getOpcode() == ISD::TRUNCATE ||
38191                          Root.getOpcode() == X86ISD::PACKSS ||
38192                          Root.getOpcode() == X86ISD::PACKUS))
38193         return SDValue(); // Nothing to do!
38194       ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);
38195       ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts / 2);
38196       V1 = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
38197       V2 = CanonicalizeShuffleInput(ShuffleSrcVT, V2);
38198       ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);
38199       ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts);
38200       Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2);
38201       Res = DAG.getNode(ISD::TRUNCATE, DL, IntMaskVT, Res);
38202       return DAG.getBitcast(RootVT, Res);
38203     }
38204   }
38205 
38206   // Don't try to re-form single instruction chains under any circumstances now
38207   // that we've done encoding canonicalization for them.
38208   if (Depth < 1)
38209     return SDValue();
38210 
38211   // Depth threshold above which we can efficiently use variable mask shuffles.
38212   int VariableCrossLaneShuffleDepth =
38213       Subtarget.hasFastVariableCrossLaneShuffle() ? 1 : 2;
38214   int VariablePerLaneShuffleDepth =
38215       Subtarget.hasFastVariablePerLaneShuffle() ? 1 : 2;
38216   AllowVariableCrossLaneMask &=
38217       (Depth >= VariableCrossLaneShuffleDepth) || HasVariableMask;
38218   AllowVariablePerLaneMask &=
38219       (Depth >= VariablePerLaneShuffleDepth) || HasVariableMask;
38220   // VPERMI2W/VPERMI2B are 3 uops on Skylake and Icelake so we require a
38221   // higher depth before combining them.
38222   bool AllowBWIVPERMV3 =
38223       (Depth >= (VariableCrossLaneShuffleDepth + 2) || HasVariableMask);
38224 
38225   bool MaskContainsZeros = isAnyZero(Mask);
38226 
38227   if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) {
38228     // If we have a single input lane-crossing shuffle then lower to VPERMV.
38229     if (UnaryShuffle && AllowVariableCrossLaneMask && !MaskContainsZeros) {
38230       if (Subtarget.hasAVX2() &&
38231           (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) {
38232         SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
38233         Res = CanonicalizeShuffleInput(MaskVT, V1);
38234         Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res);
38235         return DAG.getBitcast(RootVT, Res);
38236       }
38237       // AVX512 variants (non-VLX will pad to 512-bit shuffles).
38238       if ((Subtarget.hasAVX512() &&
38239            (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
38240             MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
38241           (Subtarget.hasBWI() &&
38242            (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
38243           (Subtarget.hasVBMI() &&
38244            (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8))) {
38245         V1 = CanonicalizeShuffleInput(MaskVT, V1);
38246         V2 = DAG.getUNDEF(MaskVT);
38247         Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
38248         return DAG.getBitcast(RootVT, Res);
38249       }
38250     }
38251 
38252     // Lower a unary+zero lane-crossing shuffle as VPERMV3 with a zero
38253     // vector as the second source (non-VLX will pad to 512-bit shuffles).
38254     if (UnaryShuffle && AllowVariableCrossLaneMask &&
38255         ((Subtarget.hasAVX512() &&
38256           (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
38257            MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||
38258            MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32 ||
38259            MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
38260          (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
38261           (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
38262          (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
38263           (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) {
38264       // Adjust shuffle mask - replace SM_SentinelZero with second source index.
38265       for (unsigned i = 0; i != NumMaskElts; ++i)
38266         if (Mask[i] == SM_SentinelZero)
38267           Mask[i] = NumMaskElts + i;
38268       V1 = CanonicalizeShuffleInput(MaskVT, V1);
38269       V2 = getZeroVector(MaskVT, Subtarget, DAG, DL);
38270       Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
38271       return DAG.getBitcast(RootVT, Res);
38272     }
38273 
38274     // If that failed and either input is extracted then try to combine as a
38275     // shuffle with the larger type.
38276     if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
38277             Inputs, Root, BaseMask, Depth, HasVariableMask,
38278             AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG,
38279             Subtarget))
38280       return WideShuffle;
38281 
38282     // If we have a dual input lane-crossing shuffle then lower to VPERMV3,
38283     // (non-VLX will pad to 512-bit shuffles).
38284     if (AllowVariableCrossLaneMask && !MaskContainsZeros &&
38285         ((Subtarget.hasAVX512() &&
38286           (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
38287            MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||
38288            MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32 ||
38289            MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) ||
38290          (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
38291           (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
38292          (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
38293           (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) {
38294       V1 = CanonicalizeShuffleInput(MaskVT, V1);
38295       V2 = CanonicalizeShuffleInput(MaskVT, V2);
38296       Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
38297       return DAG.getBitcast(RootVT, Res);
38298     }
38299     return SDValue();
38300   }
38301 
38302   // See if we can combine a single input shuffle with zeros to a bit-mask,
38303   // which is much simpler than any shuffle.
38304   if (UnaryShuffle && MaskContainsZeros && AllowVariablePerLaneMask &&
38305       isSequentialOrUndefOrZeroInRange(Mask, 0, NumMaskElts, 0) &&
38306       DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) {
38307     APInt Zero = APInt::getZero(MaskEltSizeInBits);
38308     APInt AllOnes = APInt::getAllOnes(MaskEltSizeInBits);
38309     APInt UndefElts(NumMaskElts, 0);
38310     SmallVector<APInt, 64> EltBits(NumMaskElts, Zero);
38311     for (unsigned i = 0; i != NumMaskElts; ++i) {
38312       int M = Mask[i];
38313       if (M == SM_SentinelUndef) {
38314         UndefElts.setBit(i);
38315         continue;
38316       }
38317       if (M == SM_SentinelZero)
38318         continue;
38319       EltBits[i] = AllOnes;
38320     }
38321     SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
38322     Res = CanonicalizeShuffleInput(MaskVT, V1);
38323     unsigned AndOpcode =
38324         MaskVT.isFloatingPoint() ? unsigned(X86ISD::FAND) : unsigned(ISD::AND);
38325     Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask);
38326     return DAG.getBitcast(RootVT, Res);
38327   }
38328 
38329   // If we have a single input shuffle with different shuffle patterns in the
38330   // the 128-bit lanes use the variable mask to VPERMILPS.
38331   // TODO Combine other mask types at higher depths.
38332   if (UnaryShuffle && AllowVariablePerLaneMask && !MaskContainsZeros &&
38333       ((MaskVT == MVT::v8f32 && Subtarget.hasAVX()) ||
38334        (MaskVT == MVT::v16f32 && Subtarget.hasAVX512()))) {
38335     SmallVector<SDValue, 16> VPermIdx;
38336     for (int M : Mask) {
38337       SDValue Idx =
38338           M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32);
38339       VPermIdx.push_back(Idx);
38340     }
38341     SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
38342     Res = CanonicalizeShuffleInput(MaskVT, V1);
38343     Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask);
38344     return DAG.getBitcast(RootVT, Res);
38345   }
38346 
38347   // With XOP, binary shuffles of 128/256-bit floating point vectors can combine
38348   // to VPERMIL2PD/VPERMIL2PS.
38349   if (AllowVariablePerLaneMask && Subtarget.hasXOP() &&
38350       (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v4f32 ||
38351        MaskVT == MVT::v8f32)) {
38352     // VPERMIL2 Operation.
38353     // Bits[3] - Match Bit.
38354     // Bits[2:1] - (Per Lane) PD Shuffle Mask.
38355     // Bits[2:0] - (Per Lane) PS Shuffle Mask.
38356     unsigned NumLanes = MaskVT.getSizeInBits() / 128;
38357     unsigned NumEltsPerLane = NumMaskElts / NumLanes;
38358     SmallVector<int, 8> VPerm2Idx;
38359     unsigned M2ZImm = 0;
38360     for (int M : Mask) {
38361       if (M == SM_SentinelUndef) {
38362         VPerm2Idx.push_back(-1);
38363         continue;
38364       }
38365       if (M == SM_SentinelZero) {
38366         M2ZImm = 2;
38367         VPerm2Idx.push_back(8);
38368         continue;
38369       }
38370       int Index = (M % NumEltsPerLane) + ((M / NumMaskElts) * NumEltsPerLane);
38371       Index = (MaskVT.getScalarSizeInBits() == 64 ? Index << 1 : Index);
38372       VPerm2Idx.push_back(Index);
38373     }
38374     V1 = CanonicalizeShuffleInput(MaskVT, V1);
38375     V2 = CanonicalizeShuffleInput(MaskVT, V2);
38376     SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
38377     Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp,
38378                       DAG.getTargetConstant(M2ZImm, DL, MVT::i8));
38379     return DAG.getBitcast(RootVT, Res);
38380   }
38381 
38382   // If we have 3 or more shuffle instructions or a chain involving a variable
38383   // mask, we can replace them with a single PSHUFB instruction profitably.
38384   // Intel's manuals suggest only using PSHUFB if doing so replacing 5
38385   // instructions, but in practice PSHUFB tends to be *very* fast so we're
38386   // more aggressive.
38387   if (UnaryShuffle && AllowVariablePerLaneMask &&
38388       ((RootVT.is128BitVector() && Subtarget.hasSSSE3()) ||
38389        (RootVT.is256BitVector() && Subtarget.hasAVX2()) ||
38390        (RootVT.is512BitVector() && Subtarget.hasBWI()))) {
38391     SmallVector<SDValue, 16> PSHUFBMask;
38392     int NumBytes = RootVT.getSizeInBits() / 8;
38393     int Ratio = NumBytes / NumMaskElts;
38394     for (int i = 0; i < NumBytes; ++i) {
38395       int M = Mask[i / Ratio];
38396       if (M == SM_SentinelUndef) {
38397         PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8));
38398         continue;
38399       }
38400       if (M == SM_SentinelZero) {
38401         PSHUFBMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
38402         continue;
38403       }
38404       M = Ratio * M + i % Ratio;
38405       assert((M / 16) == (i / 16) && "Lane crossing detected");
38406       PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8));
38407     }
38408     MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes);
38409     Res = CanonicalizeShuffleInput(ByteVT, V1);
38410     SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
38411     Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp);
38412     return DAG.getBitcast(RootVT, Res);
38413   }
38414 
38415   // With XOP, if we have a 128-bit binary input shuffle we can always combine
38416   // to VPPERM. We match the depth requirement of PSHUFB - VPPERM is never
38417   // slower than PSHUFB on targets that support both.
38418   if (AllowVariablePerLaneMask && RootVT.is128BitVector() &&
38419       Subtarget.hasXOP()) {
38420     // VPPERM Mask Operation
38421     // Bits[4:0] - Byte Index (0 - 31)
38422     // Bits[7:5] - Permute Operation (0 - Source byte, 4 - ZERO)
38423     SmallVector<SDValue, 16> VPPERMMask;
38424     int NumBytes = 16;
38425     int Ratio = NumBytes / NumMaskElts;
38426     for (int i = 0; i < NumBytes; ++i) {
38427       int M = Mask[i / Ratio];
38428       if (M == SM_SentinelUndef) {
38429         VPPERMMask.push_back(DAG.getUNDEF(MVT::i8));
38430         continue;
38431       }
38432       if (M == SM_SentinelZero) {
38433         VPPERMMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
38434         continue;
38435       }
38436       M = Ratio * M + i % Ratio;
38437       VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8));
38438     }
38439     MVT ByteVT = MVT::v16i8;
38440     V1 = CanonicalizeShuffleInput(ByteVT, V1);
38441     V2 = CanonicalizeShuffleInput(ByteVT, V2);
38442     SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
38443     Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp);
38444     return DAG.getBitcast(RootVT, Res);
38445   }
38446 
38447   // If that failed and either input is extracted then try to combine as a
38448   // shuffle with the larger type.
38449   if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
38450           Inputs, Root, BaseMask, Depth, HasVariableMask,
38451           AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG, Subtarget))
38452     return WideShuffle;
38453 
38454   // If we have a dual input shuffle then lower to VPERMV3,
38455   // (non-VLX will pad to 512-bit shuffles)
38456   if (!UnaryShuffle && AllowVariablePerLaneMask && !MaskContainsZeros &&
38457       ((Subtarget.hasAVX512() &&
38458         (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v8f64 ||
38459          MaskVT == MVT::v2i64 || MaskVT == MVT::v4i64 || MaskVT == MVT::v8i64 ||
38460          MaskVT == MVT::v4f32 || MaskVT == MVT::v4i32 || MaskVT == MVT::v8f32 ||
38461          MaskVT == MVT::v8i32 || MaskVT == MVT::v16f32 ||
38462          MaskVT == MVT::v16i32)) ||
38463        (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
38464         (MaskVT == MVT::v8i16 || MaskVT == MVT::v16i16 ||
38465          MaskVT == MVT::v32i16)) ||
38466        (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
38467         (MaskVT == MVT::v16i8 || MaskVT == MVT::v32i8 ||
38468          MaskVT == MVT::v64i8)))) {
38469     V1 = CanonicalizeShuffleInput(MaskVT, V1);
38470     V2 = CanonicalizeShuffleInput(MaskVT, V2);
38471     Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
38472     return DAG.getBitcast(RootVT, Res);
38473   }
38474 
38475   // Failed to find any combines.
38476   return SDValue();
38477 }
38478 
38479 // Combine an arbitrary chain of shuffles + extract_subvectors into a single
38480 // instruction if possible.
38481 //
38482 // Wrapper for combineX86ShuffleChain that extends the shuffle mask to a larger
38483 // type size to attempt to combine:
38484 // shuffle(extract_subvector(x,c1),extract_subvector(y,c2),m1)
38485 // -->
38486 // extract_subvector(shuffle(x,y,m2),0)
38487 static SDValue combineX86ShuffleChainWithExtract(
38488     ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
38489     bool HasVariableMask, bool AllowVariableCrossLaneMask,
38490     bool AllowVariablePerLaneMask, SelectionDAG &DAG,
38491     const X86Subtarget &Subtarget) {
38492   unsigned NumMaskElts = BaseMask.size();
38493   unsigned NumInputs = Inputs.size();
38494   if (NumInputs == 0)
38495     return SDValue();
38496 
38497   EVT RootVT = Root.getValueType();
38498   unsigned RootSizeInBits = RootVT.getSizeInBits();
38499   unsigned RootEltSizeInBits = RootSizeInBits / NumMaskElts;
38500   assert((RootSizeInBits % NumMaskElts) == 0 && "Unexpected root shuffle mask");
38501 
38502   // Peek through extract_subvector to find widest legal vector.
38503   // TODO: Handle ISD::TRUNCATE
38504   unsigned WideSizeInBits = RootSizeInBits;
38505   for (unsigned I = 0; I != NumInputs; ++I) {
38506     SDValue Input = peekThroughBitcasts(Inputs[I]);
38507     while (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR)
38508       Input = peekThroughBitcasts(Input.getOperand(0));
38509     if (DAG.getTargetLoweringInfo().isTypeLegal(Input.getValueType()) &&
38510         WideSizeInBits < Input.getValueSizeInBits())
38511       WideSizeInBits = Input.getValueSizeInBits();
38512   }
38513 
38514   // Bail if we fail to find a source larger than the existing root.
38515   unsigned Scale = WideSizeInBits / RootSizeInBits;
38516   if (WideSizeInBits <= RootSizeInBits ||
38517       (WideSizeInBits % RootSizeInBits) != 0)
38518     return SDValue();
38519 
38520   // Create new mask for larger type.
38521   SmallVector<int, 64> WideMask(BaseMask);
38522   for (int &M : WideMask) {
38523     if (M < 0)
38524       continue;
38525     M = (M % NumMaskElts) + ((M / NumMaskElts) * Scale * NumMaskElts);
38526   }
38527   WideMask.append((Scale - 1) * NumMaskElts, SM_SentinelUndef);
38528 
38529   // Attempt to peek through inputs and adjust mask when we extract from an
38530   // upper subvector.
38531   int AdjustedMasks = 0;
38532   SmallVector<SDValue, 4> WideInputs(Inputs.begin(), Inputs.end());
38533   for (unsigned I = 0; I != NumInputs; ++I) {
38534     SDValue &Input = WideInputs[I];
38535     Input = peekThroughBitcasts(Input);
38536     while (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
38537            Input.getOperand(0).getValueSizeInBits() <= WideSizeInBits) {
38538       uint64_t Idx = Input.getConstantOperandVal(1);
38539       if (Idx != 0) {
38540         ++AdjustedMasks;
38541         unsigned InputEltSizeInBits = Input.getScalarValueSizeInBits();
38542         Idx = (Idx * InputEltSizeInBits) / RootEltSizeInBits;
38543 
38544         int lo = I * WideMask.size();
38545         int hi = (I + 1) * WideMask.size();
38546         for (int &M : WideMask)
38547           if (lo <= M && M < hi)
38548             M += Idx;
38549       }
38550       Input = peekThroughBitcasts(Input.getOperand(0));
38551     }
38552   }
38553 
38554   // Remove unused/repeated shuffle source ops.
38555   resolveTargetShuffleInputsAndMask(WideInputs, WideMask);
38556   assert(!WideInputs.empty() && "Shuffle with no inputs detected");
38557 
38558   // Bail if we're always extracting from the lowest subvectors,
38559   // combineX86ShuffleChain should match this for the current width, or the
38560   // shuffle still references too many inputs.
38561   if (AdjustedMasks == 0 || WideInputs.size() > 2)
38562     return SDValue();
38563 
38564   // Minor canonicalization of the accumulated shuffle mask to make it easier
38565   // to match below. All this does is detect masks with sequential pairs of
38566   // elements, and shrink them to the half-width mask. It does this in a loop
38567   // so it will reduce the size of the mask to the minimal width mask which
38568   // performs an equivalent shuffle.
38569   while (WideMask.size() > 1) {
38570     SmallVector<int, 64> WidenedMask;
38571     if (!canWidenShuffleElements(WideMask, WidenedMask))
38572       break;
38573     WideMask = std::move(WidenedMask);
38574   }
38575 
38576   // Canonicalization of binary shuffle masks to improve pattern matching by
38577   // commuting the inputs.
38578   if (WideInputs.size() == 2 && canonicalizeShuffleMaskWithCommute(WideMask)) {
38579     ShuffleVectorSDNode::commuteMask(WideMask);
38580     std::swap(WideInputs[0], WideInputs[1]);
38581   }
38582 
38583   // Increase depth for every upper subvector we've peeked through.
38584   Depth += AdjustedMasks;
38585 
38586   // Attempt to combine wider chain.
38587   // TODO: Can we use a better Root?
38588   SDValue WideRoot = WideInputs.front().getValueSizeInBits() >
38589                              WideInputs.back().getValueSizeInBits()
38590                          ? WideInputs.front()
38591                          : WideInputs.back();
38592   assert(WideRoot.getValueSizeInBits() == WideSizeInBits &&
38593          "WideRootSize mismatch");
38594 
38595   if (SDValue WideShuffle =
38596           combineX86ShuffleChain(WideInputs, WideRoot, WideMask, Depth,
38597                                  HasVariableMask, AllowVariableCrossLaneMask,
38598                                  AllowVariablePerLaneMask, DAG, Subtarget)) {
38599     WideShuffle =
38600         extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits);
38601     return DAG.getBitcast(RootVT, WideShuffle);
38602   }
38603 
38604   return SDValue();
38605 }
38606 
38607 // Canonicalize the combined shuffle mask chain with horizontal ops.
38608 // NOTE: This may update the Ops and Mask.
38609 static SDValue canonicalizeShuffleMaskWithHorizOp(
38610     MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask,
38611     unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
38612     const X86Subtarget &Subtarget) {
38613   if (Mask.empty() || Ops.empty())
38614     return SDValue();
38615 
38616   SmallVector<SDValue> BC;
38617   for (SDValue Op : Ops)
38618     BC.push_back(peekThroughBitcasts(Op));
38619 
38620   // All ops must be the same horizop + type.
38621   SDValue BC0 = BC[0];
38622   EVT VT0 = BC0.getValueType();
38623   unsigned Opcode0 = BC0.getOpcode();
38624   if (VT0.getSizeInBits() != RootSizeInBits || llvm::any_of(BC, [&](SDValue V) {
38625         return V.getOpcode() != Opcode0 || V.getValueType() != VT0;
38626       }))
38627     return SDValue();
38628 
38629   bool isHoriz = (Opcode0 == X86ISD::FHADD || Opcode0 == X86ISD::HADD ||
38630                   Opcode0 == X86ISD::FHSUB || Opcode0 == X86ISD::HSUB);
38631   bool isPack = (Opcode0 == X86ISD::PACKSS || Opcode0 == X86ISD::PACKUS);
38632   if (!isHoriz && !isPack)
38633     return SDValue();
38634 
38635   // Do all ops have a single use?
38636   bool OneUseOps = llvm::all_of(Ops, [](SDValue Op) {
38637     return Op.hasOneUse() &&
38638            peekThroughBitcasts(Op) == peekThroughOneUseBitcasts(Op);
38639   });
38640 
38641   int NumElts = VT0.getVectorNumElements();
38642   int NumLanes = VT0.getSizeInBits() / 128;
38643   int NumEltsPerLane = NumElts / NumLanes;
38644   int NumHalfEltsPerLane = NumEltsPerLane / 2;
38645   MVT SrcVT = BC0.getOperand(0).getSimpleValueType();
38646   unsigned EltSizeInBits = RootSizeInBits / Mask.size();
38647 
38648   if (NumEltsPerLane >= 4 &&
38649       (isPack || shouldUseHorizontalOp(Ops.size() == 1, DAG, Subtarget))) {
38650     SmallVector<int> LaneMask, ScaledMask;
38651     if (isRepeatedTargetShuffleMask(128, EltSizeInBits, Mask, LaneMask) &&
38652         scaleShuffleElements(LaneMask, 4, ScaledMask)) {
38653       // See if we can remove the shuffle by resorting the HOP chain so that
38654       // the HOP args are pre-shuffled.
38655       // TODO: Generalize to any sized/depth chain.
38656       // TODO: Add support for PACKSS/PACKUS.
38657       if (isHoriz) {
38658         // Attempt to find a HOP(HOP(X,Y),HOP(Z,W)) source operand.
38659         auto GetHOpSrc = [&](int M) {
38660           if (M == SM_SentinelUndef)
38661             return DAG.getUNDEF(VT0);
38662           if (M == SM_SentinelZero)
38663             return getZeroVector(VT0.getSimpleVT(), Subtarget, DAG, DL);
38664           SDValue Src0 = BC[M / 4];
38665           SDValue Src1 = Src0.getOperand((M % 4) >= 2);
38666           if (Src1.getOpcode() == Opcode0 && Src0->isOnlyUserOf(Src1.getNode()))
38667             return Src1.getOperand(M % 2);
38668           return SDValue();
38669         };
38670         SDValue M0 = GetHOpSrc(ScaledMask[0]);
38671         SDValue M1 = GetHOpSrc(ScaledMask[1]);
38672         SDValue M2 = GetHOpSrc(ScaledMask[2]);
38673         SDValue M3 = GetHOpSrc(ScaledMask[3]);
38674         if (M0 && M1 && M2 && M3) {
38675           SDValue LHS = DAG.getNode(Opcode0, DL, SrcVT, M0, M1);
38676           SDValue RHS = DAG.getNode(Opcode0, DL, SrcVT, M2, M3);
38677           return DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
38678         }
38679       }
38680       // shuffle(hop(x,y),hop(z,w)) -> permute(hop(x,z)) etc.
38681       if (Ops.size() >= 2) {
38682         SDValue LHS, RHS;
38683         auto GetHOpSrc = [&](int M, int &OutM) {
38684           // TODO: Support SM_SentinelZero
38685           if (M < 0)
38686             return M == SM_SentinelUndef;
38687           SDValue Src = BC[M / 4].getOperand((M % 4) >= 2);
38688           if (!LHS || LHS == Src) {
38689             LHS = Src;
38690             OutM = (M % 2);
38691             return true;
38692           }
38693           if (!RHS || RHS == Src) {
38694             RHS = Src;
38695             OutM = (M % 2) + 2;
38696             return true;
38697           }
38698           return false;
38699         };
38700         int PostMask[4] = {-1, -1, -1, -1};
38701         if (GetHOpSrc(ScaledMask[0], PostMask[0]) &&
38702             GetHOpSrc(ScaledMask[1], PostMask[1]) &&
38703             GetHOpSrc(ScaledMask[2], PostMask[2]) &&
38704             GetHOpSrc(ScaledMask[3], PostMask[3])) {
38705           LHS = DAG.getBitcast(SrcVT, LHS);
38706           RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS);
38707           SDValue Res = DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
38708           // Use SHUFPS for the permute so this will work on SSE2 targets,
38709           // shuffle combining and domain handling will simplify this later on.
38710           MVT ShuffleVT = MVT::getVectorVT(MVT::f32, RootSizeInBits / 32);
38711           Res = DAG.getBitcast(ShuffleVT, Res);
38712           return DAG.getNode(X86ISD::SHUFP, DL, ShuffleVT, Res, Res,
38713                              getV4X86ShuffleImm8ForMask(PostMask, DL, DAG));
38714         }
38715       }
38716     }
38717   }
38718 
38719   if (2 < Ops.size())
38720     return SDValue();
38721 
38722   SDValue BC1 = BC[BC.size() - 1];
38723   if (Mask.size() == VT0.getVectorNumElements()) {
38724     // Canonicalize binary shuffles of horizontal ops that use the
38725     // same sources to an unary shuffle.
38726     // TODO: Try to perform this fold even if the shuffle remains.
38727     if (Ops.size() == 2) {
38728       auto ContainsOps = [](SDValue HOp, SDValue Op) {
38729         return Op == HOp.getOperand(0) || Op == HOp.getOperand(1);
38730       };
38731       // Commute if all BC0's ops are contained in BC1.
38732       if (ContainsOps(BC1, BC0.getOperand(0)) &&
38733           ContainsOps(BC1, BC0.getOperand(1))) {
38734         ShuffleVectorSDNode::commuteMask(Mask);
38735         std::swap(Ops[0], Ops[1]);
38736         std::swap(BC0, BC1);
38737       }
38738 
38739       // If BC1 can be represented by BC0, then convert to unary shuffle.
38740       if (ContainsOps(BC0, BC1.getOperand(0)) &&
38741           ContainsOps(BC0, BC1.getOperand(1))) {
38742         for (int &M : Mask) {
38743           if (M < NumElts) // BC0 element or UNDEF/Zero sentinel.
38744             continue;
38745           int SubLane = ((M % NumEltsPerLane) >= NumHalfEltsPerLane) ? 1 : 0;
38746           M -= NumElts + (SubLane * NumHalfEltsPerLane);
38747           if (BC1.getOperand(SubLane) != BC0.getOperand(0))
38748             M += NumHalfEltsPerLane;
38749         }
38750       }
38751     }
38752 
38753     // Canonicalize unary horizontal ops to only refer to lower halves.
38754     for (int i = 0; i != NumElts; ++i) {
38755       int &M = Mask[i];
38756       if (isUndefOrZero(M))
38757         continue;
38758       if (M < NumElts && BC0.getOperand(0) == BC0.getOperand(1) &&
38759           (M % NumEltsPerLane) >= NumHalfEltsPerLane)
38760         M -= NumHalfEltsPerLane;
38761       if (NumElts <= M && BC1.getOperand(0) == BC1.getOperand(1) &&
38762           (M % NumEltsPerLane) >= NumHalfEltsPerLane)
38763         M -= NumHalfEltsPerLane;
38764     }
38765   }
38766 
38767   // Combine binary shuffle of 2 similar 'Horizontal' instructions into a
38768   // single instruction. Attempt to match a v2X64 repeating shuffle pattern that
38769   // represents the LHS/RHS inputs for the lower/upper halves.
38770   SmallVector<int, 16> TargetMask128, WideMask128;
38771   if (isRepeatedTargetShuffleMask(128, EltSizeInBits, Mask, TargetMask128) &&
38772       scaleShuffleElements(TargetMask128, 2, WideMask128)) {
38773     assert(isUndefOrZeroOrInRange(WideMask128, 0, 4) && "Illegal shuffle");
38774     bool SingleOp = (Ops.size() == 1);
38775     if (isPack || OneUseOps ||
38776         shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) {
38777       SDValue Lo = isInRange(WideMask128[0], 0, 2) ? BC0 : BC1;
38778       SDValue Hi = isInRange(WideMask128[1], 0, 2) ? BC0 : BC1;
38779       Lo = Lo.getOperand(WideMask128[0] & 1);
38780       Hi = Hi.getOperand(WideMask128[1] & 1);
38781       if (SingleOp) {
38782         SDValue Undef = DAG.getUNDEF(SrcVT);
38783         SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL);
38784         Lo = (WideMask128[0] == SM_SentinelZero ? Zero : Lo);
38785         Hi = (WideMask128[1] == SM_SentinelZero ? Zero : Hi);
38786         Lo = (WideMask128[0] == SM_SentinelUndef ? Undef : Lo);
38787         Hi = (WideMask128[1] == SM_SentinelUndef ? Undef : Hi);
38788       }
38789       return DAG.getNode(Opcode0, DL, VT0, Lo, Hi);
38790     }
38791   }
38792 
38793   // If we are post-shuffling a 256-bit hop and not requiring the upper
38794   // elements, then try to narrow to a 128-bit hop directly.
38795   SmallVector<int, 16> WideMask64;
38796   if (Ops.size() == 1 && NumLanes == 2 &&
38797       scaleShuffleElements(Mask, 4, WideMask64) &&
38798       isUndefInRange(WideMask64, 2, 2)) {
38799     int M0 = WideMask64[0];
38800     int M1 = WideMask64[1];
38801     if (isInRange(M0, 0, 4) && isInRange(M1, 0, 4)) {
38802       MVT HalfVT = VT0.getSimpleVT().getHalfNumVectorElementsVT();
38803       unsigned Idx0 = (M0 & 2) ? (SrcVT.getVectorNumElements() / 2) : 0;
38804       unsigned Idx1 = (M1 & 2) ? (SrcVT.getVectorNumElements() / 2) : 0;
38805       SDValue V0 = extract128BitVector(BC[0].getOperand(M0 & 1), Idx0, DAG, DL);
38806       SDValue V1 = extract128BitVector(BC[0].getOperand(M1 & 1), Idx1, DAG, DL);
38807       SDValue Res = DAG.getNode(Opcode0, DL, HalfVT, V0, V1);
38808       return widenSubVector(Res, false, Subtarget, DAG, DL, 256);
38809     }
38810   }
38811 
38812   return SDValue();
38813 }
38814 
38815 // Attempt to constant fold all of the constant source ops.
38816 // Returns true if the entire shuffle is folded to a constant.
38817 // TODO: Extend this to merge multiple constant Ops and update the mask.
38818 static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops,
38819                                            ArrayRef<int> Mask, SDValue Root,
38820                                            bool HasVariableMask,
38821                                            SelectionDAG &DAG,
38822                                            const X86Subtarget &Subtarget) {
38823   MVT VT = Root.getSimpleValueType();
38824 
38825   unsigned SizeInBits = VT.getSizeInBits();
38826   unsigned NumMaskElts = Mask.size();
38827   unsigned MaskSizeInBits = SizeInBits / NumMaskElts;
38828   unsigned NumOps = Ops.size();
38829 
38830   // Extract constant bits from each source op.
38831   SmallVector<APInt, 16> UndefEltsOps(NumOps);
38832   SmallVector<SmallVector<APInt, 16>, 16> RawBitsOps(NumOps);
38833   for (unsigned I = 0; I != NumOps; ++I)
38834     if (!getTargetConstantBitsFromNode(Ops[I], MaskSizeInBits, UndefEltsOps[I],
38835                                        RawBitsOps[I]))
38836       return SDValue();
38837 
38838   // If we're optimizing for size, only fold if at least one of the constants is
38839   // only used once or the combined shuffle has included a variable mask
38840   // shuffle, this is to avoid constant pool bloat.
38841   bool IsOptimizingSize = DAG.shouldOptForSize();
38842   if (IsOptimizingSize && !HasVariableMask &&
38843       llvm::none_of(Ops, [](SDValue SrcOp) { return SrcOp->hasOneUse(); }))
38844     return SDValue();
38845 
38846   // Shuffle the constant bits according to the mask.
38847   SDLoc DL(Root);
38848   APInt UndefElts(NumMaskElts, 0);
38849   APInt ZeroElts(NumMaskElts, 0);
38850   APInt ConstantElts(NumMaskElts, 0);
38851   SmallVector<APInt, 8> ConstantBitData(NumMaskElts,
38852                                         APInt::getZero(MaskSizeInBits));
38853   for (unsigned i = 0; i != NumMaskElts; ++i) {
38854     int M = Mask[i];
38855     if (M == SM_SentinelUndef) {
38856       UndefElts.setBit(i);
38857       continue;
38858     } else if (M == SM_SentinelZero) {
38859       ZeroElts.setBit(i);
38860       continue;
38861     }
38862     assert(0 <= M && M < (int)(NumMaskElts * NumOps));
38863 
38864     unsigned SrcOpIdx = (unsigned)M / NumMaskElts;
38865     unsigned SrcMaskIdx = (unsigned)M % NumMaskElts;
38866 
38867     auto &SrcUndefElts = UndefEltsOps[SrcOpIdx];
38868     if (SrcUndefElts[SrcMaskIdx]) {
38869       UndefElts.setBit(i);
38870       continue;
38871     }
38872 
38873     auto &SrcEltBits = RawBitsOps[SrcOpIdx];
38874     APInt &Bits = SrcEltBits[SrcMaskIdx];
38875     if (!Bits) {
38876       ZeroElts.setBit(i);
38877       continue;
38878     }
38879 
38880     ConstantElts.setBit(i);
38881     ConstantBitData[i] = Bits;
38882   }
38883   assert((UndefElts | ZeroElts | ConstantElts).isAllOnes());
38884 
38885   // Attempt to create a zero vector.
38886   if ((UndefElts | ZeroElts).isAllOnes())
38887     return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG, DL);
38888 
38889   // Create the constant data.
38890   MVT MaskSVT;
38891   if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64))
38892     MaskSVT = MVT::getFloatingPointVT(MaskSizeInBits);
38893   else
38894     MaskSVT = MVT::getIntegerVT(MaskSizeInBits);
38895 
38896   MVT MaskVT = MVT::getVectorVT(MaskSVT, NumMaskElts);
38897   if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT))
38898     return SDValue();
38899 
38900   SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
38901   return DAG.getBitcast(VT, CstOp);
38902 }
38903 
38904 namespace llvm {
38905   namespace X86 {
38906     enum {
38907       MaxShuffleCombineDepth = 8
38908     };
38909   } // namespace X86
38910 } // namespace llvm
38911 
38912 /// Fully generic combining of x86 shuffle instructions.
38913 ///
38914 /// This should be the last combine run over the x86 shuffle instructions. Once
38915 /// they have been fully optimized, this will recursively consider all chains
38916 /// of single-use shuffle instructions, build a generic model of the cumulative
38917 /// shuffle operation, and check for simpler instructions which implement this
38918 /// operation. We use this primarily for two purposes:
38919 ///
38920 /// 1) Collapse generic shuffles to specialized single instructions when
38921 ///    equivalent. In most cases, this is just an encoding size win, but
38922 ///    sometimes we will collapse multiple generic shuffles into a single
38923 ///    special-purpose shuffle.
38924 /// 2) Look for sequences of shuffle instructions with 3 or more total
38925 ///    instructions, and replace them with the slightly more expensive SSSE3
38926 ///    PSHUFB instruction if available. We do this as the last combining step
38927 ///    to ensure we avoid using PSHUFB if we can implement the shuffle with
38928 ///    a suitable short sequence of other instructions. The PSHUFB will either
38929 ///    use a register or have to read from memory and so is slightly (but only
38930 ///    slightly) more expensive than the other shuffle instructions.
38931 ///
38932 /// Because this is inherently a quadratic operation (for each shuffle in
38933 /// a chain, we recurse up the chain), the depth is limited to 8 instructions.
38934 /// This should never be an issue in practice as the shuffle lowering doesn't
38935 /// produce sequences of more than 8 instructions.
38936 ///
38937 /// FIXME: We will currently miss some cases where the redundant shuffling
38938 /// would simplify under the threshold for PSHUFB formation because of
38939 /// combine-ordering. To fix this, we should do the redundant instruction
38940 /// combining in this recursive walk.
38941 static SDValue combineX86ShufflesRecursively(
38942     ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
38943     ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth,
38944     unsigned MaxDepth, bool HasVariableMask, bool AllowVariableCrossLaneMask,
38945     bool AllowVariablePerLaneMask, SelectionDAG &DAG,
38946     const X86Subtarget &Subtarget) {
38947   assert(!RootMask.empty() &&
38948          (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
38949          "Illegal shuffle root mask");
38950   MVT RootVT = Root.getSimpleValueType();
38951   assert(RootVT.isVector() && "Shuffles operate on vector types!");
38952   unsigned RootSizeInBits = RootVT.getSizeInBits();
38953 
38954   // Bound the depth of our recursive combine because this is ultimately
38955   // quadratic in nature.
38956   if (Depth >= MaxDepth)
38957     return SDValue();
38958 
38959   // Directly rip through bitcasts to find the underlying operand.
38960   SDValue Op = SrcOps[SrcOpIndex];
38961   Op = peekThroughOneUseBitcasts(Op);
38962 
38963   EVT VT = Op.getValueType();
38964   if (!VT.isVector() || !VT.isSimple())
38965     return SDValue(); // Bail if we hit a non-simple non-vector.
38966 
38967   // FIXME: Just bail on f16 for now.
38968   if (VT.getVectorElementType() == MVT::f16)
38969     return SDValue();
38970 
38971   assert((RootSizeInBits % VT.getSizeInBits()) == 0 &&
38972          "Can only combine shuffles upto size of the root op.");
38973 
38974   // Create a demanded elts mask from the referenced elements of Op.
38975   APInt OpDemandedElts = APInt::getZero(RootMask.size());
38976   for (int M : RootMask) {
38977     int BaseIdx = RootMask.size() * SrcOpIndex;
38978     if (isInRange(M, BaseIdx, BaseIdx + RootMask.size()))
38979       OpDemandedElts.setBit(M - BaseIdx);
38980   }
38981   if (RootSizeInBits != VT.getSizeInBits()) {
38982     // Op is smaller than Root - extract the demanded elts for the subvector.
38983     unsigned Scale = RootSizeInBits / VT.getSizeInBits();
38984     unsigned NumOpMaskElts = RootMask.size() / Scale;
38985     assert((RootMask.size() % Scale) == 0 && "Root mask size mismatch");
38986     assert(OpDemandedElts
38987                .extractBits(RootMask.size() - NumOpMaskElts, NumOpMaskElts)
38988                .isZero() &&
38989            "Out of range elements referenced in root mask");
38990     OpDemandedElts = OpDemandedElts.extractBits(NumOpMaskElts, 0);
38991   }
38992   OpDemandedElts =
38993       APIntOps::ScaleBitMask(OpDemandedElts, VT.getVectorNumElements());
38994 
38995   // Extract target shuffle mask and resolve sentinels and inputs.
38996   SmallVector<int, 64> OpMask;
38997   SmallVector<SDValue, 2> OpInputs;
38998   APInt OpUndef, OpZero;
38999   bool IsOpVariableMask = isTargetShuffleVariableMask(Op.getOpcode());
39000   if (getTargetShuffleInputs(Op, OpDemandedElts, OpInputs, OpMask, OpUndef,
39001                              OpZero, DAG, Depth, false)) {
39002     // Shuffle inputs must not be larger than the shuffle result.
39003     // TODO: Relax this for single input faux shuffles (e.g. trunc).
39004     if (llvm::any_of(OpInputs, [VT](SDValue OpInput) {
39005           return OpInput.getValueSizeInBits() > VT.getSizeInBits();
39006         }))
39007       return SDValue();
39008   } else if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
39009              (RootSizeInBits % Op.getOperand(0).getValueSizeInBits()) == 0 &&
39010              !isNullConstant(Op.getOperand(1))) {
39011     SDValue SrcVec = Op.getOperand(0);
39012     int ExtractIdx = Op.getConstantOperandVal(1);
39013     unsigned NumElts = VT.getVectorNumElements();
39014     OpInputs.assign({SrcVec});
39015     OpMask.assign(NumElts, SM_SentinelUndef);
39016     std::iota(OpMask.begin(), OpMask.end(), ExtractIdx);
39017     OpZero = OpUndef = APInt::getZero(NumElts);
39018   } else {
39019     return SDValue();
39020   }
39021 
39022   // If the shuffle result was smaller than the root, we need to adjust the
39023   // mask indices and pad the mask with undefs.
39024   if (RootSizeInBits > VT.getSizeInBits()) {
39025     unsigned NumSubVecs = RootSizeInBits / VT.getSizeInBits();
39026     unsigned OpMaskSize = OpMask.size();
39027     if (OpInputs.size() > 1) {
39028       unsigned PaddedMaskSize = NumSubVecs * OpMaskSize;
39029       for (int &M : OpMask) {
39030         if (M < 0)
39031           continue;
39032         int EltIdx = M % OpMaskSize;
39033         int OpIdx = M / OpMaskSize;
39034         M = (PaddedMaskSize * OpIdx) + EltIdx;
39035       }
39036     }
39037     OpZero = OpZero.zext(NumSubVecs * OpMaskSize);
39038     OpUndef = OpUndef.zext(NumSubVecs * OpMaskSize);
39039     OpMask.append((NumSubVecs - 1) * OpMaskSize, SM_SentinelUndef);
39040   }
39041 
39042   SmallVector<int, 64> Mask;
39043   SmallVector<SDValue, 16> Ops;
39044 
39045   // We don't need to merge masks if the root is empty.
39046   bool EmptyRoot = (Depth == 0) && (RootMask.size() == 1);
39047   if (EmptyRoot) {
39048     // Only resolve zeros if it will remove an input, otherwise we might end
39049     // up in an infinite loop.
39050     bool ResolveKnownZeros = true;
39051     if (!OpZero.isZero()) {
39052       APInt UsedInputs = APInt::getZero(OpInputs.size());
39053       for (int i = 0, e = OpMask.size(); i != e; ++i) {
39054         int M = OpMask[i];
39055         if (OpUndef[i] || OpZero[i] || isUndefOrZero(M))
39056           continue;
39057         UsedInputs.setBit(M / OpMask.size());
39058         if (UsedInputs.isAllOnes()) {
39059           ResolveKnownZeros = false;
39060           break;
39061         }
39062       }
39063     }
39064     resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero,
39065                                       ResolveKnownZeros);
39066 
39067     Mask = OpMask;
39068     Ops.append(OpInputs.begin(), OpInputs.end());
39069   } else {
39070     resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero);
39071 
39072     // Add the inputs to the Ops list, avoiding duplicates.
39073     Ops.append(SrcOps.begin(), SrcOps.end());
39074 
39075     auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int {
39076       // Attempt to find an existing match.
39077       SDValue InputBC = peekThroughBitcasts(Input);
39078       for (int i = 0, e = Ops.size(); i < e; ++i)
39079         if (InputBC == peekThroughBitcasts(Ops[i]))
39080           return i;
39081       // Match failed - should we replace an existing Op?
39082       if (InsertionPoint >= 0) {
39083         Ops[InsertionPoint] = Input;
39084         return InsertionPoint;
39085       }
39086       // Add to the end of the Ops list.
39087       Ops.push_back(Input);
39088       return Ops.size() - 1;
39089     };
39090 
39091     SmallVector<int, 2> OpInputIdx;
39092     for (SDValue OpInput : OpInputs)
39093       OpInputIdx.push_back(
39094           AddOp(OpInput, OpInputIdx.empty() ? SrcOpIndex : -1));
39095 
39096     assert(((RootMask.size() > OpMask.size() &&
39097              RootMask.size() % OpMask.size() == 0) ||
39098             (OpMask.size() > RootMask.size() &&
39099              OpMask.size() % RootMask.size() == 0) ||
39100             OpMask.size() == RootMask.size()) &&
39101            "The smaller number of elements must divide the larger.");
39102 
39103     // This function can be performance-critical, so we rely on the power-of-2
39104     // knowledge that we have about the mask sizes to replace div/rem ops with
39105     // bit-masks and shifts.
39106     assert(llvm::has_single_bit<uint32_t>(RootMask.size()) &&
39107            "Non-power-of-2 shuffle mask sizes");
39108     assert(llvm::has_single_bit<uint32_t>(OpMask.size()) &&
39109            "Non-power-of-2 shuffle mask sizes");
39110     unsigned RootMaskSizeLog2 = llvm::countr_zero(RootMask.size());
39111     unsigned OpMaskSizeLog2 = llvm::countr_zero(OpMask.size());
39112 
39113     unsigned MaskWidth = std::max<unsigned>(OpMask.size(), RootMask.size());
39114     unsigned RootRatio =
39115         std::max<unsigned>(1, OpMask.size() >> RootMaskSizeLog2);
39116     unsigned OpRatio = std::max<unsigned>(1, RootMask.size() >> OpMaskSizeLog2);
39117     assert((RootRatio == 1 || OpRatio == 1) &&
39118            "Must not have a ratio for both incoming and op masks!");
39119 
39120     assert(isPowerOf2_32(MaskWidth) && "Non-power-of-2 shuffle mask sizes");
39121     assert(isPowerOf2_32(RootRatio) && "Non-power-of-2 shuffle mask sizes");
39122     assert(isPowerOf2_32(OpRatio) && "Non-power-of-2 shuffle mask sizes");
39123     unsigned RootRatioLog2 = llvm::countr_zero(RootRatio);
39124     unsigned OpRatioLog2 = llvm::countr_zero(OpRatio);
39125 
39126     Mask.resize(MaskWidth, SM_SentinelUndef);
39127 
39128     // Merge this shuffle operation's mask into our accumulated mask. Note that
39129     // this shuffle's mask will be the first applied to the input, followed by
39130     // the root mask to get us all the way to the root value arrangement. The
39131     // reason for this order is that we are recursing up the operation chain.
39132     for (unsigned i = 0; i < MaskWidth; ++i) {
39133       unsigned RootIdx = i >> RootRatioLog2;
39134       if (RootMask[RootIdx] < 0) {
39135         // This is a zero or undef lane, we're done.
39136         Mask[i] = RootMask[RootIdx];
39137         continue;
39138       }
39139 
39140       unsigned RootMaskedIdx =
39141           RootRatio == 1
39142               ? RootMask[RootIdx]
39143               : (RootMask[RootIdx] << RootRatioLog2) + (i & (RootRatio - 1));
39144 
39145       // Just insert the scaled root mask value if it references an input other
39146       // than the SrcOp we're currently inserting.
39147       if ((RootMaskedIdx < (SrcOpIndex * MaskWidth)) ||
39148           (((SrcOpIndex + 1) * MaskWidth) <= RootMaskedIdx)) {
39149         Mask[i] = RootMaskedIdx;
39150         continue;
39151       }
39152 
39153       RootMaskedIdx = RootMaskedIdx & (MaskWidth - 1);
39154       unsigned OpIdx = RootMaskedIdx >> OpRatioLog2;
39155       if (OpMask[OpIdx] < 0) {
39156         // The incoming lanes are zero or undef, it doesn't matter which ones we
39157         // are using.
39158         Mask[i] = OpMask[OpIdx];
39159         continue;
39160       }
39161 
39162       // Ok, we have non-zero lanes, map them through to one of the Op's inputs.
39163       unsigned OpMaskedIdx = OpRatio == 1 ? OpMask[OpIdx]
39164                                           : (OpMask[OpIdx] << OpRatioLog2) +
39165                                                 (RootMaskedIdx & (OpRatio - 1));
39166 
39167       OpMaskedIdx = OpMaskedIdx & (MaskWidth - 1);
39168       int InputIdx = OpMask[OpIdx] / (int)OpMask.size();
39169       assert(0 <= OpInputIdx[InputIdx] && "Unknown target shuffle input");
39170       OpMaskedIdx += OpInputIdx[InputIdx] * MaskWidth;
39171 
39172       Mask[i] = OpMaskedIdx;
39173     }
39174   }
39175 
39176   // Peek through vector widenings and set out of bounds mask indices to undef.
39177   // TODO: Can resolveTargetShuffleInputsAndMask do some of this?
39178   for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
39179     SDValue &Op = Ops[I];
39180     if (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op.getOperand(0).isUndef() &&
39181         isNullConstant(Op.getOperand(2))) {
39182       Op = Op.getOperand(1);
39183       unsigned Scale = RootSizeInBits / Op.getValueSizeInBits();
39184       int Lo = I * Mask.size();
39185       int Hi = (I + 1) * Mask.size();
39186       int NewHi = Lo + (Mask.size() / Scale);
39187       for (int &M : Mask) {
39188         if (Lo <= M && NewHi <= M && M < Hi)
39189           M = SM_SentinelUndef;
39190       }
39191     }
39192   }
39193 
39194   // Peek through any free extract_subvector nodes back to root size.
39195   for (SDValue &Op : Ops)
39196     while (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
39197            (RootSizeInBits % Op.getOperand(0).getValueSizeInBits()) == 0 &&
39198            isNullConstant(Op.getOperand(1)))
39199       Op = Op.getOperand(0);
39200 
39201   // Remove unused/repeated shuffle source ops.
39202   resolveTargetShuffleInputsAndMask(Ops, Mask);
39203 
39204   // Handle the all undef/zero/ones cases early.
39205   if (all_of(Mask, [](int Idx) { return Idx == SM_SentinelUndef; }))
39206     return DAG.getUNDEF(RootVT);
39207   if (all_of(Mask, [](int Idx) { return Idx < 0; }))
39208     return getZeroVector(RootVT, Subtarget, DAG, SDLoc(Root));
39209   if (Ops.size() == 1 && ISD::isBuildVectorAllOnes(Ops[0].getNode()) &&
39210       !llvm::is_contained(Mask, SM_SentinelZero))
39211     return getOnesVector(RootVT, DAG, SDLoc(Root));
39212 
39213   assert(!Ops.empty() && "Shuffle with no inputs detected");
39214   HasVariableMask |= IsOpVariableMask;
39215 
39216   // Update the list of shuffle nodes that have been combined so far.
39217   SmallVector<const SDNode *, 16> CombinedNodes(SrcNodes.begin(),
39218                                                 SrcNodes.end());
39219   CombinedNodes.push_back(Op.getNode());
39220 
39221   // See if we can recurse into each shuffle source op (if it's a target
39222   // shuffle). The source op should only be generally combined if it either has
39223   // a single use (i.e. current Op) or all its users have already been combined,
39224   // if not then we can still combine but should prevent generation of variable
39225   // shuffles to avoid constant pool bloat.
39226   // Don't recurse if we already have more source ops than we can combine in
39227   // the remaining recursion depth.
39228   if (Ops.size() < (MaxDepth - Depth)) {
39229     for (int i = 0, e = Ops.size(); i < e; ++i) {
39230       // For empty roots, we need to resolve zeroable elements before combining
39231       // them with other shuffles.
39232       SmallVector<int, 64> ResolvedMask = Mask;
39233       if (EmptyRoot)
39234         resolveTargetShuffleFromZeroables(ResolvedMask, OpUndef, OpZero);
39235       bool AllowCrossLaneVar = false;
39236       bool AllowPerLaneVar = false;
39237       if (Ops[i].getNode()->hasOneUse() ||
39238           SDNode::areOnlyUsersOf(CombinedNodes, Ops[i].getNode())) {
39239         AllowCrossLaneVar = AllowVariableCrossLaneMask;
39240         AllowPerLaneVar = AllowVariablePerLaneMask;
39241       }
39242       if (SDValue Res = combineX86ShufflesRecursively(
39243               Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1, MaxDepth,
39244               HasVariableMask, AllowCrossLaneVar, AllowPerLaneVar, DAG,
39245               Subtarget))
39246         return Res;
39247     }
39248   }
39249 
39250   // Attempt to constant fold all of the constant source ops.
39251   if (SDValue Cst = combineX86ShufflesConstants(
39252           Ops, Mask, Root, HasVariableMask, DAG, Subtarget))
39253     return Cst;
39254 
39255   // If constant fold failed and we only have constants - then we have
39256   // multiple uses by a single non-variable shuffle - just bail.
39257   if (Depth == 0 && llvm::all_of(Ops, [&](SDValue Op) {
39258         APInt UndefElts;
39259         SmallVector<APInt> RawBits;
39260         unsigned EltSizeInBits = RootSizeInBits / Mask.size();
39261         return getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
39262                                              RawBits);
39263       })) {
39264     return SDValue();
39265   }
39266 
39267   // Canonicalize the combined shuffle mask chain with horizontal ops.
39268   // NOTE: This will update the Ops and Mask.
39269   if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp(
39270           Ops, Mask, RootSizeInBits, SDLoc(Root), DAG, Subtarget))
39271     return DAG.getBitcast(RootVT, HOp);
39272 
39273   // Try to refine our inputs given our knowledge of target shuffle mask.
39274   for (auto I : enumerate(Ops)) {
39275     int OpIdx = I.index();
39276     SDValue &Op = I.value();
39277 
39278     // What range of shuffle mask element values results in picking from Op?
39279     int Lo = OpIdx * Mask.size();
39280     int Hi = Lo + Mask.size();
39281 
39282     // Which elements of Op do we demand, given the mask's granularity?
39283     APInt OpDemandedElts(Mask.size(), 0);
39284     for (int MaskElt : Mask) {
39285       if (isInRange(MaskElt, Lo, Hi)) { // Picks from Op?
39286         int OpEltIdx = MaskElt - Lo;
39287         OpDemandedElts.setBit(OpEltIdx);
39288       }
39289     }
39290 
39291     // Is the shuffle result smaller than the root?
39292     if (Op.getValueSizeInBits() < RootSizeInBits) {
39293       // We padded the mask with undefs. But we now need to undo that.
39294       unsigned NumExpectedVectorElts = Mask.size();
39295       unsigned EltSizeInBits = RootSizeInBits / NumExpectedVectorElts;
39296       unsigned NumOpVectorElts = Op.getValueSizeInBits() / EltSizeInBits;
39297       assert(!OpDemandedElts.extractBits(
39298                  NumExpectedVectorElts - NumOpVectorElts, NumOpVectorElts) &&
39299              "Demanding the virtual undef widening padding?");
39300       OpDemandedElts = OpDemandedElts.trunc(NumOpVectorElts); // NUW
39301     }
39302 
39303     // The Op itself may be of different VT, so we need to scale the mask.
39304     unsigned NumOpElts = Op.getValueType().getVectorNumElements();
39305     APInt OpScaledDemandedElts = APIntOps::ScaleBitMask(OpDemandedElts, NumOpElts);
39306 
39307     // Can this operand be simplified any further, given it's demanded elements?
39308     if (SDValue NewOp =
39309             DAG.getTargetLoweringInfo().SimplifyMultipleUseDemandedVectorElts(
39310                 Op, OpScaledDemandedElts, DAG))
39311       Op = NewOp;
39312   }
39313   // FIXME: should we rerun resolveTargetShuffleInputsAndMask() now?
39314 
39315   // Widen any subvector shuffle inputs we've collected.
39316   // TODO: Remove this to avoid generating temporary nodes, we should only
39317   // widen once combineX86ShuffleChain has found a match.
39318   if (any_of(Ops, [RootSizeInBits](SDValue Op) {
39319         return Op.getValueSizeInBits() < RootSizeInBits;
39320       })) {
39321     for (SDValue &Op : Ops)
39322       if (Op.getValueSizeInBits() < RootSizeInBits)
39323         Op = widenSubVector(Op, false, Subtarget, DAG, SDLoc(Op),
39324                             RootSizeInBits);
39325     // Reresolve - we might have repeated subvector sources.
39326     resolveTargetShuffleInputsAndMask(Ops, Mask);
39327   }
39328 
39329   // We can only combine unary and binary shuffle mask cases.
39330   if (Ops.size() <= 2) {
39331     // Minor canonicalization of the accumulated shuffle mask to make it easier
39332     // to match below. All this does is detect masks with sequential pairs of
39333     // elements, and shrink them to the half-width mask. It does this in a loop
39334     // so it will reduce the size of the mask to the minimal width mask which
39335     // performs an equivalent shuffle.
39336     while (Mask.size() > 1) {
39337       SmallVector<int, 64> WidenedMask;
39338       if (!canWidenShuffleElements(Mask, WidenedMask))
39339         break;
39340       Mask = std::move(WidenedMask);
39341     }
39342 
39343     // Canonicalization of binary shuffle masks to improve pattern matching by
39344     // commuting the inputs.
39345     if (Ops.size() == 2 && canonicalizeShuffleMaskWithCommute(Mask)) {
39346       ShuffleVectorSDNode::commuteMask(Mask);
39347       std::swap(Ops[0], Ops[1]);
39348     }
39349 
39350     // Try to combine into a single shuffle instruction.
39351     if (SDValue Shuffle = combineX86ShuffleChain(
39352             Ops, Root, Mask, Depth, HasVariableMask, AllowVariableCrossLaneMask,
39353             AllowVariablePerLaneMask, DAG, Subtarget))
39354       return Shuffle;
39355 
39356     // If all the operands come from the same larger vector, fallthrough and try
39357     // to use combineX86ShuffleChainWithExtract.
39358     SDValue LHS = peekThroughBitcasts(Ops.front());
39359     SDValue RHS = peekThroughBitcasts(Ops.back());
39360     if (Ops.size() != 2 || !Subtarget.hasAVX2() || RootSizeInBits != 128 ||
39361         (RootSizeInBits / Mask.size()) != 64 ||
39362         LHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
39363         RHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
39364         LHS.getOperand(0) != RHS.getOperand(0))
39365       return SDValue();
39366   }
39367 
39368   // If that failed and any input is extracted then try to combine as a
39369   // shuffle with the larger type.
39370   return combineX86ShuffleChainWithExtract(
39371       Ops, Root, Mask, Depth, HasVariableMask, AllowVariableCrossLaneMask,
39372       AllowVariablePerLaneMask, DAG, Subtarget);
39373 }
39374 
39375 /// Helper entry wrapper to combineX86ShufflesRecursively.
39376 static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
39377                                              const X86Subtarget &Subtarget) {
39378   return combineX86ShufflesRecursively(
39379       {Op}, 0, Op, {0}, {}, /*Depth*/ 0, X86::MaxShuffleCombineDepth,
39380       /*HasVarMask*/ false,
39381       /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, DAG,
39382       Subtarget);
39383 }
39384 
39385 /// Get the PSHUF-style mask from PSHUF node.
39386 ///
39387 /// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
39388 /// PSHUF-style masks that can be reused with such instructions.
39389 static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
39390   MVT VT = N.getSimpleValueType();
39391   SmallVector<int, 4> Mask;
39392   SmallVector<SDValue, 2> Ops;
39393   bool HaveMask =
39394       getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask);
39395   (void)HaveMask;
39396   assert(HaveMask);
39397 
39398   // If we have more than 128-bits, only the low 128-bits of shuffle mask
39399   // matter. Check that the upper masks are repeats and remove them.
39400   if (VT.getSizeInBits() > 128) {
39401     int LaneElts = 128 / VT.getScalarSizeInBits();
39402 #ifndef NDEBUG
39403     for (int i = 1, NumLanes = VT.getSizeInBits() / 128; i < NumLanes; ++i)
39404       for (int j = 0; j < LaneElts; ++j)
39405         assert(Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) &&
39406                "Mask doesn't repeat in high 128-bit lanes!");
39407 #endif
39408     Mask.resize(LaneElts);
39409   }
39410 
39411   switch (N.getOpcode()) {
39412   case X86ISD::PSHUFD:
39413     return Mask;
39414   case X86ISD::PSHUFLW:
39415     Mask.resize(4);
39416     return Mask;
39417   case X86ISD::PSHUFHW:
39418     Mask.erase(Mask.begin(), Mask.begin() + 4);
39419     for (int &M : Mask)
39420       M -= 4;
39421     return Mask;
39422   default:
39423     llvm_unreachable("No valid shuffle instruction found!");
39424   }
39425 }
39426 
39427 /// Search for a combinable shuffle across a chain ending in pshufd.
39428 ///
39429 /// We walk up the chain and look for a combinable shuffle, skipping over
39430 /// shuffles that we could hoist this shuffle's transformation past without
39431 /// altering anything.
39432 static SDValue
39433 combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
39434                              SelectionDAG &DAG) {
39435   assert(N.getOpcode() == X86ISD::PSHUFD &&
39436          "Called with something other than an x86 128-bit half shuffle!");
39437   SDLoc DL(N);
39438 
39439   // Walk up a single-use chain looking for a combinable shuffle. Keep a stack
39440   // of the shuffles in the chain so that we can form a fresh chain to replace
39441   // this one.
39442   SmallVector<SDValue, 8> Chain;
39443   SDValue V = N.getOperand(0);
39444   for (; V.hasOneUse(); V = V.getOperand(0)) {
39445     switch (V.getOpcode()) {
39446     default:
39447       return SDValue(); // Nothing combined!
39448 
39449     case ISD::BITCAST:
39450       // Skip bitcasts as we always know the type for the target specific
39451       // instructions.
39452       continue;
39453 
39454     case X86ISD::PSHUFD:
39455       // Found another dword shuffle.
39456       break;
39457 
39458     case X86ISD::PSHUFLW:
39459       // Check that the low words (being shuffled) are the identity in the
39460       // dword shuffle, and the high words are self-contained.
39461       if (Mask[0] != 0 || Mask[1] != 1 ||
39462           !(Mask[2] >= 2 && Mask[2] < 4 && Mask[3] >= 2 && Mask[3] < 4))
39463         return SDValue();
39464 
39465       Chain.push_back(V);
39466       continue;
39467 
39468     case X86ISD::PSHUFHW:
39469       // Check that the high words (being shuffled) are the identity in the
39470       // dword shuffle, and the low words are self-contained.
39471       if (Mask[2] != 2 || Mask[3] != 3 ||
39472           !(Mask[0] >= 0 && Mask[0] < 2 && Mask[1] >= 0 && Mask[1] < 2))
39473         return SDValue();
39474 
39475       Chain.push_back(V);
39476       continue;
39477 
39478     case X86ISD::UNPCKL:
39479     case X86ISD::UNPCKH:
39480       // For either i8 -> i16 or i16 -> i32 unpacks, we can combine a dword
39481       // shuffle into a preceding word shuffle.
39482       if (V.getSimpleValueType().getVectorElementType() != MVT::i8 &&
39483           V.getSimpleValueType().getVectorElementType() != MVT::i16)
39484         return SDValue();
39485 
39486       // Search for a half-shuffle which we can combine with.
39487       unsigned CombineOp =
39488           V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
39489       if (V.getOperand(0) != V.getOperand(1) ||
39490           !V->isOnlyUserOf(V.getOperand(0).getNode()))
39491         return SDValue();
39492       Chain.push_back(V);
39493       V = V.getOperand(0);
39494       do {
39495         switch (V.getOpcode()) {
39496         default:
39497           return SDValue(); // Nothing to combine.
39498 
39499         case X86ISD::PSHUFLW:
39500         case X86ISD::PSHUFHW:
39501           if (V.getOpcode() == CombineOp)
39502             break;
39503 
39504           Chain.push_back(V);
39505 
39506           [[fallthrough]];
39507         case ISD::BITCAST:
39508           V = V.getOperand(0);
39509           continue;
39510         }
39511         break;
39512       } while (V.hasOneUse());
39513       break;
39514     }
39515     // Break out of the loop if we break out of the switch.
39516     break;
39517   }
39518 
39519   if (!V.hasOneUse())
39520     // We fell out of the loop without finding a viable combining instruction.
39521     return SDValue();
39522 
39523   // Merge this node's mask and our incoming mask.
39524   SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
39525   for (int &M : Mask)
39526     M = VMask[M];
39527   V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
39528                   getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
39529 
39530   // Rebuild the chain around this new shuffle.
39531   while (!Chain.empty()) {
39532     SDValue W = Chain.pop_back_val();
39533 
39534     if (V.getValueType() != W.getOperand(0).getValueType())
39535       V = DAG.getBitcast(W.getOperand(0).getValueType(), V);
39536 
39537     switch (W.getOpcode()) {
39538     default:
39539       llvm_unreachable("Only PSHUF and UNPCK instructions get here!");
39540 
39541     case X86ISD::UNPCKL:
39542     case X86ISD::UNPCKH:
39543       V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V);
39544       break;
39545 
39546     case X86ISD::PSHUFD:
39547     case X86ISD::PSHUFLW:
39548     case X86ISD::PSHUFHW:
39549       V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1));
39550       break;
39551     }
39552   }
39553   if (V.getValueType() != N.getValueType())
39554     V = DAG.getBitcast(N.getValueType(), V);
39555 
39556   // Return the new chain to replace N.
39557   return V;
39558 }
39559 
39560 // Attempt to commute shufps LHS loads:
39561 // permilps(shufps(load(),x)) --> permilps(shufps(x,load()))
39562 static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
39563                                       SelectionDAG &DAG) {
39564   // TODO: Add vXf64 support.
39565   if (VT != MVT::v4f32 && VT != MVT::v8f32 && VT != MVT::v16f32)
39566     return SDValue();
39567 
39568   // SHUFP(LHS, RHS) -> SHUFP(RHS, LHS) iff LHS is foldable + RHS is not.
39569   auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) {
39570     if (V.getOpcode() != X86ISD::SHUFP || !Parent->isOnlyUserOf(V.getNode()))
39571       return SDValue();
39572     SDValue N0 = V.getOperand(0);
39573     SDValue N1 = V.getOperand(1);
39574     unsigned Imm = V.getConstantOperandVal(2);
39575     const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
39576     if (!X86::mayFoldLoad(peekThroughOneUseBitcasts(N0), Subtarget) ||
39577         X86::mayFoldLoad(peekThroughOneUseBitcasts(N1), Subtarget))
39578       return SDValue();
39579     Imm = ((Imm & 0x0F) << 4) | ((Imm & 0xF0) >> 4);
39580     return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0,
39581                        DAG.getTargetConstant(Imm, DL, MVT::i8));
39582   };
39583 
39584   switch (N.getOpcode()) {
39585   case X86ISD::VPERMILPI:
39586     if (SDValue NewSHUFP = commuteSHUFP(N, N.getOperand(0))) {
39587       unsigned Imm = N.getConstantOperandVal(1);
39588       return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP,
39589                          DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
39590     }
39591     break;
39592   case X86ISD::SHUFP: {
39593     SDValue N0 = N.getOperand(0);
39594     SDValue N1 = N.getOperand(1);
39595     unsigned Imm = N.getConstantOperandVal(2);
39596     if (N0 == N1) {
39597       if (SDValue NewSHUFP = commuteSHUFP(N, N0))
39598         return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP,
39599                            DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
39600     } else if (SDValue NewSHUFP = commuteSHUFP(N, N0)) {
39601       return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1,
39602                          DAG.getTargetConstant(Imm ^ 0x0A, DL, MVT::i8));
39603     } else if (SDValue NewSHUFP = commuteSHUFP(N, N1)) {
39604       return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP,
39605                          DAG.getTargetConstant(Imm ^ 0xA0, DL, MVT::i8));
39606     }
39607     break;
39608   }
39609   }
39610 
39611   return SDValue();
39612 }
39613 
39614 // TODO - move this to TLI like isBinOp?
39615 static bool isUnaryOp(unsigned Opcode) {
39616   switch (Opcode) {
39617   case ISD::CTLZ:
39618   case ISD::CTTZ:
39619   case ISD::CTPOP:
39620     return true;
39621   }
39622   return false;
39623 }
39624 
39625 // Canonicalize SHUFFLE(UNARYOP(X)) -> UNARYOP(SHUFFLE(X)).
39626 // Canonicalize SHUFFLE(BINOP(X,Y)) -> BINOP(SHUFFLE(X),SHUFFLE(Y)).
39627 static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
39628                                          const SDLoc &DL) {
39629   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
39630   EVT ShuffleVT = N.getValueType();
39631 
39632   auto IsMergeableWithShuffle = [&DAG](SDValue Op, bool FoldLoad = false) {
39633     // AllZeros/AllOnes constants are freely shuffled and will peek through
39634     // bitcasts. Other constant build vectors do not peek through bitcasts. Only
39635     // merge with target shuffles if it has one use so shuffle combining is
39636     // likely to kick in. Shuffles of splats are expected to be removed.
39637     return ISD::isBuildVectorAllOnes(Op.getNode()) ||
39638            ISD::isBuildVectorAllZeros(Op.getNode()) ||
39639            ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
39640            ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()) ||
39641            getTargetConstantFromNode(dyn_cast<LoadSDNode>(Op)) ||
39642            (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op->hasOneUse()) ||
39643            (isTargetShuffle(Op.getOpcode()) && Op->hasOneUse()) ||
39644            (FoldLoad && isShuffleFoldableLoad(Op)) ||
39645            DAG.isSplatValue(Op, /*AllowUndefs*/ false);
39646   };
39647   auto IsSafeToMoveShuffle = [ShuffleVT](SDValue Op, unsigned BinOp) {
39648     // Ensure we only shuffle whole vector src elements, unless its a logical
39649     // binops where we can more aggressively move shuffles from dst to src.
39650     return BinOp == ISD::AND || BinOp == ISD::OR || BinOp == ISD::XOR ||
39651            BinOp == X86ISD::ANDNP ||
39652            (Op.getScalarValueSizeInBits() <= ShuffleVT.getScalarSizeInBits());
39653   };
39654 
39655   unsigned Opc = N.getOpcode();
39656   switch (Opc) {
39657   // Unary and Unary+Permute Shuffles.
39658   case X86ISD::PSHUFB: {
39659     // Don't merge PSHUFB if it contains zero'd elements.
39660     SmallVector<int> Mask;
39661     SmallVector<SDValue> Ops;
39662     if (!getTargetShuffleMask(N.getNode(), ShuffleVT.getSimpleVT(), false, Ops,
39663                               Mask))
39664       break;
39665     [[fallthrough]];
39666   }
39667   case X86ISD::VBROADCAST:
39668   case X86ISD::MOVDDUP:
39669   case X86ISD::PSHUFD:
39670   case X86ISD::PSHUFHW:
39671   case X86ISD::PSHUFLW:
39672   case X86ISD::VPERMI:
39673   case X86ISD::VPERMILPI: {
39674     if (N.getOperand(0).getValueType() == ShuffleVT &&
39675         N->isOnlyUserOf(N.getOperand(0).getNode())) {
39676       SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
39677       unsigned SrcOpcode = N0.getOpcode();
39678       if (TLI.isBinOp(SrcOpcode) && IsSafeToMoveShuffle(N0, SrcOpcode)) {
39679         SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
39680         SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
39681         if (IsMergeableWithShuffle(Op00, Opc != X86ISD::PSHUFB) ||
39682             IsMergeableWithShuffle(Op01, Opc != X86ISD::PSHUFB)) {
39683           SDValue LHS, RHS;
39684           Op00 = DAG.getBitcast(ShuffleVT, Op00);
39685           Op01 = DAG.getBitcast(ShuffleVT, Op01);
39686           if (N.getNumOperands() == 2) {
39687             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, N.getOperand(1));
39688             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, N.getOperand(1));
39689           } else {
39690             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00);
39691             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01);
39692           }
39693           EVT OpVT = N0.getValueType();
39694           return DAG.getBitcast(ShuffleVT,
39695                                 DAG.getNode(SrcOpcode, DL, OpVT,
39696                                             DAG.getBitcast(OpVT, LHS),
39697                                             DAG.getBitcast(OpVT, RHS)));
39698         }
39699       }
39700     }
39701     break;
39702   }
39703   // Binary and Binary+Permute Shuffles.
39704   case X86ISD::INSERTPS: {
39705     // Don't merge INSERTPS if it contains zero'd elements.
39706     unsigned InsertPSMask = N.getConstantOperandVal(2);
39707     unsigned ZeroMask = InsertPSMask & 0xF;
39708     if (ZeroMask != 0)
39709       break;
39710     [[fallthrough]];
39711   }
39712   case X86ISD::MOVSD:
39713   case X86ISD::MOVSS:
39714   case X86ISD::BLENDI:
39715   case X86ISD::SHUFP:
39716   case X86ISD::UNPCKH:
39717   case X86ISD::UNPCKL: {
39718     if (N->isOnlyUserOf(N.getOperand(0).getNode()) &&
39719         N->isOnlyUserOf(N.getOperand(1).getNode())) {
39720       SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
39721       SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
39722       unsigned SrcOpcode = N0.getOpcode();
39723       if (TLI.isBinOp(SrcOpcode) && N1.getOpcode() == SrcOpcode &&
39724           N0.getValueType() == N1.getValueType() &&
39725           IsSafeToMoveShuffle(N0, SrcOpcode) &&
39726           IsSafeToMoveShuffle(N1, SrcOpcode)) {
39727         SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
39728         SDValue Op10 = peekThroughOneUseBitcasts(N1.getOperand(0));
39729         SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
39730         SDValue Op11 = peekThroughOneUseBitcasts(N1.getOperand(1));
39731         // Ensure the total number of shuffles doesn't increase by folding this
39732         // shuffle through to the source ops.
39733         if (((IsMergeableWithShuffle(Op00) && IsMergeableWithShuffle(Op10)) ||
39734              (IsMergeableWithShuffle(Op01) && IsMergeableWithShuffle(Op11))) ||
39735             ((IsMergeableWithShuffle(Op00) || IsMergeableWithShuffle(Op10)) &&
39736              (IsMergeableWithShuffle(Op01) || IsMergeableWithShuffle(Op11)))) {
39737           SDValue LHS, RHS;
39738           Op00 = DAG.getBitcast(ShuffleVT, Op00);
39739           Op10 = DAG.getBitcast(ShuffleVT, Op10);
39740           Op01 = DAG.getBitcast(ShuffleVT, Op01);
39741           Op11 = DAG.getBitcast(ShuffleVT, Op11);
39742           if (N.getNumOperands() == 3) {
39743             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2));
39744             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11, N.getOperand(2));
39745           } else {
39746             LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10);
39747             RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11);
39748           }
39749           EVT OpVT = N0.getValueType();
39750           return DAG.getBitcast(ShuffleVT,
39751                                 DAG.getNode(SrcOpcode, DL, OpVT,
39752                                             DAG.getBitcast(OpVT, LHS),
39753                                             DAG.getBitcast(OpVT, RHS)));
39754         }
39755       }
39756       if (isUnaryOp(SrcOpcode) && N1.getOpcode() == SrcOpcode &&
39757           N0.getValueType() == N1.getValueType() &&
39758           IsSafeToMoveShuffle(N0, SrcOpcode) &&
39759           IsSafeToMoveShuffle(N1, SrcOpcode)) {
39760         SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
39761         SDValue Op10 = peekThroughOneUseBitcasts(N1.getOperand(0));
39762         SDValue Res;
39763         Op00 = DAG.getBitcast(ShuffleVT, Op00);
39764         Op10 = DAG.getBitcast(ShuffleVT, Op10);
39765         if (N.getNumOperands() == 3) {
39766           Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2));
39767         } else {
39768           Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10);
39769         }
39770         EVT OpVT = N0.getValueType();
39771         return DAG.getBitcast(
39772             ShuffleVT,
39773             DAG.getNode(SrcOpcode, DL, OpVT, DAG.getBitcast(OpVT, Res)));
39774       }
39775     }
39776     break;
39777   }
39778   }
39779   return SDValue();
39780 }
39781 
39782 /// Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
39783 static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V,
39784                                                       SelectionDAG &DAG,
39785                                                       const SDLoc &DL) {
39786   assert(V.getOpcode() == X86ISD::VPERM2X128 && "Unknown lane shuffle");
39787 
39788   MVT VT = V.getSimpleValueType();
39789   SDValue Src0 = peekThroughBitcasts(V.getOperand(0));
39790   SDValue Src1 = peekThroughBitcasts(V.getOperand(1));
39791   unsigned SrcOpc0 = Src0.getOpcode();
39792   unsigned SrcOpc1 = Src1.getOpcode();
39793   EVT SrcVT0 = Src0.getValueType();
39794   EVT SrcVT1 = Src1.getValueType();
39795 
39796   if (!Src1.isUndef() && (SrcVT0 != SrcVT1 || SrcOpc0 != SrcOpc1))
39797     return SDValue();
39798 
39799   switch (SrcOpc0) {
39800   case X86ISD::MOVDDUP: {
39801     SDValue LHS = Src0.getOperand(0);
39802     SDValue RHS = Src1.isUndef() ? Src1 : Src1.getOperand(0);
39803     SDValue Res =
39804         DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, V.getOperand(2));
39805     Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res);
39806     return DAG.getBitcast(VT, Res);
39807   }
39808   case X86ISD::VPERMILPI:
39809     // TODO: Handle v4f64 permutes with different low/high lane masks.
39810     if (SrcVT0 == MVT::v4f64) {
39811       uint64_t Mask = Src0.getConstantOperandVal(1);
39812       if ((Mask & 0x3) != ((Mask >> 2) & 0x3))
39813         break;
39814     }
39815     [[fallthrough]];
39816   case X86ISD::VSHLI:
39817   case X86ISD::VSRLI:
39818   case X86ISD::VSRAI:
39819   case X86ISD::PSHUFD:
39820     if (Src1.isUndef() || Src0.getOperand(1) == Src1.getOperand(1)) {
39821       SDValue LHS = Src0.getOperand(0);
39822       SDValue RHS = Src1.isUndef() ? Src1 : Src1.getOperand(0);
39823       SDValue Res = DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS,
39824                                 V.getOperand(2));
39825       Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res, Src0.getOperand(1));
39826       return DAG.getBitcast(VT, Res);
39827     }
39828     break;
39829   }
39830 
39831   return SDValue();
39832 }
39833 
39834 /// Try to combine x86 target specific shuffles.
39835 static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
39836                                     TargetLowering::DAGCombinerInfo &DCI,
39837                                     const X86Subtarget &Subtarget) {
39838   SDLoc DL(N);
39839   MVT VT = N.getSimpleValueType();
39840   SmallVector<int, 4> Mask;
39841   unsigned Opcode = N.getOpcode();
39842 
39843   if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG))
39844     return R;
39845 
39846   // Handle specific target shuffles.
39847   switch (Opcode) {
39848   case X86ISD::MOVDDUP: {
39849     SDValue Src = N.getOperand(0);
39850     // Turn a 128-bit MOVDDUP of a full vector load into movddup+vzload.
39851     if (VT == MVT::v2f64 && Src.hasOneUse() &&
39852         ISD::isNormalLoad(Src.getNode())) {
39853       LoadSDNode *LN = cast<LoadSDNode>(Src);
39854       if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::f64, MVT::v2f64, DAG)) {
39855         SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);
39856         DCI.CombineTo(N.getNode(), Movddup);
39857         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
39858         DCI.recursivelyDeleteUnusedNodes(LN);
39859         return N; // Return N so it doesn't get rechecked!
39860       }
39861     }
39862 
39863     return SDValue();
39864   }
39865   case X86ISD::VBROADCAST: {
39866     SDValue Src = N.getOperand(0);
39867     SDValue BC = peekThroughBitcasts(Src);
39868     EVT SrcVT = Src.getValueType();
39869     EVT BCVT = BC.getValueType();
39870 
39871     // If broadcasting from another shuffle, attempt to simplify it.
39872     // TODO - we really need a general SimplifyDemandedVectorElts mechanism.
39873     if (isTargetShuffle(BC.getOpcode()) &&
39874         VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits() == 0) {
39875       unsigned Scale = VT.getScalarSizeInBits() / BCVT.getScalarSizeInBits();
39876       SmallVector<int, 16> DemandedMask(BCVT.getVectorNumElements(),
39877                                         SM_SentinelUndef);
39878       for (unsigned i = 0; i != Scale; ++i)
39879         DemandedMask[i] = i;
39880       if (SDValue Res = combineX86ShufflesRecursively(
39881               {BC}, 0, BC, DemandedMask, {}, /*Depth*/ 0,
39882               X86::MaxShuffleCombineDepth,
39883               /*HasVarMask*/ false, /*AllowCrossLaneVarMask*/ true,
39884               /*AllowPerLaneVarMask*/ true, DAG, Subtarget))
39885         return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
39886                            DAG.getBitcast(SrcVT, Res));
39887     }
39888 
39889     // broadcast(bitcast(src)) -> bitcast(broadcast(src))
39890     // 32-bit targets have to bitcast i64 to f64, so better to bitcast upward.
39891     if (Src.getOpcode() == ISD::BITCAST &&
39892         SrcVT.getScalarSizeInBits() == BCVT.getScalarSizeInBits() &&
39893         DAG.getTargetLoweringInfo().isTypeLegal(BCVT) &&
39894         FixedVectorType::isValidElementType(
39895             BCVT.getScalarType().getTypeForEVT(*DAG.getContext()))) {
39896       EVT NewVT = EVT::getVectorVT(*DAG.getContext(), BCVT.getScalarType(),
39897                                    VT.getVectorNumElements());
39898       return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
39899     }
39900 
39901     // vbroadcast(bitcast(vbroadcast(src))) -> bitcast(vbroadcast(src))
39902     // If we're re-broadcasting a smaller type then broadcast with that type and
39903     // bitcast.
39904     // TODO: Do this for any splat?
39905     if (Src.getOpcode() == ISD::BITCAST &&
39906         (BC.getOpcode() == X86ISD::VBROADCAST ||
39907          BC.getOpcode() == X86ISD::VBROADCAST_LOAD) &&
39908         (VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits()) == 0 &&
39909         (VT.getSizeInBits() % BCVT.getSizeInBits()) == 0) {
39910       MVT NewVT =
39911           MVT::getVectorVT(BCVT.getSimpleVT().getScalarType(),
39912                            VT.getSizeInBits() / BCVT.getScalarSizeInBits());
39913       return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
39914     }
39915 
39916     // Reduce broadcast source vector to lowest 128-bits.
39917     if (SrcVT.getSizeInBits() > 128)
39918       return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
39919                          extract128BitVector(Src, 0, DAG, DL));
39920 
39921     // broadcast(scalar_to_vector(x)) -> broadcast(x).
39922     if (Src.getOpcode() == ISD::SCALAR_TO_VECTOR &&
39923         Src.getValueType().getScalarType() == Src.getOperand(0).getValueType())
39924       return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
39925 
39926     // broadcast(extract_vector_elt(x, 0)) -> broadcast(x).
39927     if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
39928         isNullConstant(Src.getOperand(1)) &&
39929         Src.getValueType() ==
39930             Src.getOperand(0).getValueType().getScalarType() &&
39931         DAG.getTargetLoweringInfo().isTypeLegal(
39932             Src.getOperand(0).getValueType()))
39933       return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
39934 
39935     // Share broadcast with the longest vector and extract low subvector (free).
39936     // Ensure the same SDValue from the SDNode use is being used.
39937     for (SDNode *User : Src->uses())
39938       if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST &&
39939           Src == User->getOperand(0) &&
39940           User->getValueSizeInBits(0).getFixedValue() >
39941               VT.getFixedSizeInBits()) {
39942         return extractSubVector(SDValue(User, 0), 0, DAG, DL,
39943                                 VT.getSizeInBits());
39944       }
39945 
39946     // vbroadcast(scalarload X) -> vbroadcast_load X
39947     // For float loads, extract other uses of the scalar from the broadcast.
39948     if (!SrcVT.isVector() && (Src.hasOneUse() || VT.isFloatingPoint()) &&
39949         ISD::isNormalLoad(Src.getNode())) {
39950       LoadSDNode *LN = cast<LoadSDNode>(Src);
39951       SDVTList Tys = DAG.getVTList(VT, MVT::Other);
39952       SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
39953       SDValue BcastLd =
39954           DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
39955                                   LN->getMemoryVT(), LN->getMemOperand());
39956       // If the load value is used only by N, replace it via CombineTo N.
39957       bool NoReplaceExtract = Src.hasOneUse();
39958       DCI.CombineTo(N.getNode(), BcastLd);
39959       if (NoReplaceExtract) {
39960         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
39961         DCI.recursivelyDeleteUnusedNodes(LN);
39962       } else {
39963         SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
39964                                   DAG.getIntPtrConstant(0, DL));
39965         DCI.CombineTo(LN, Scl, BcastLd.getValue(1));
39966       }
39967       return N; // Return N so it doesn't get rechecked!
39968     }
39969 
39970     // Due to isTypeDesirableForOp, we won't always shrink a load truncated to
39971     // i16. So shrink it ourselves if we can make a broadcast_load.
39972     if (SrcVT == MVT::i16 && Src.getOpcode() == ISD::TRUNCATE &&
39973         Src.hasOneUse() && Src.getOperand(0).hasOneUse()) {
39974       assert(Subtarget.hasAVX2() && "Expected AVX2");
39975       SDValue TruncIn = Src.getOperand(0);
39976 
39977       // If this is a truncate of a non extending load we can just narrow it to
39978       // use a broadcast_load.
39979       if (ISD::isNormalLoad(TruncIn.getNode())) {
39980         LoadSDNode *LN = cast<LoadSDNode>(TruncIn);
39981         // Unless its volatile or atomic.
39982         if (LN->isSimple()) {
39983           SDVTList Tys = DAG.getVTList(VT, MVT::Other);
39984           SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
39985           SDValue BcastLd = DAG.getMemIntrinsicNode(
39986               X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
39987               LN->getPointerInfo(), LN->getOriginalAlign(),
39988               LN->getMemOperand()->getFlags());
39989           DCI.CombineTo(N.getNode(), BcastLd);
39990           DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
39991           DCI.recursivelyDeleteUnusedNodes(Src.getNode());
39992           return N; // Return N so it doesn't get rechecked!
39993         }
39994       }
39995 
39996       // If this is a truncate of an i16 extload, we can directly replace it.
39997       if (ISD::isUNINDEXEDLoad(Src.getOperand(0).getNode()) &&
39998           ISD::isEXTLoad(Src.getOperand(0).getNode())) {
39999         LoadSDNode *LN = cast<LoadSDNode>(Src.getOperand(0));
40000         if (LN->getMemoryVT().getSizeInBits() == 16) {
40001           SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40002           SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
40003           SDValue BcastLd =
40004               DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
40005                                       LN->getMemoryVT(), LN->getMemOperand());
40006           DCI.CombineTo(N.getNode(), BcastLd);
40007           DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40008           DCI.recursivelyDeleteUnusedNodes(Src.getNode());
40009           return N; // Return N so it doesn't get rechecked!
40010         }
40011       }
40012 
40013       // If this is a truncate of load that has been shifted right, we can
40014       // offset the pointer and use a narrower load.
40015       if (TruncIn.getOpcode() == ISD::SRL &&
40016           TruncIn.getOperand(0).hasOneUse() &&
40017           isa<ConstantSDNode>(TruncIn.getOperand(1)) &&
40018           ISD::isNormalLoad(TruncIn.getOperand(0).getNode())) {
40019         LoadSDNode *LN = cast<LoadSDNode>(TruncIn.getOperand(0));
40020         unsigned ShiftAmt = TruncIn.getConstantOperandVal(1);
40021         // Make sure the shift amount and the load size are divisible by 16.
40022         // Don't do this if the load is volatile or atomic.
40023         if (ShiftAmt % 16 == 0 && TruncIn.getValueSizeInBits() % 16 == 0 &&
40024             LN->isSimple()) {
40025           unsigned Offset = ShiftAmt / 8;
40026           SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40027           SDValue Ptr = DAG.getMemBasePlusOffset(
40028               LN->getBasePtr(), TypeSize::getFixed(Offset), DL);
40029           SDValue Ops[] = { LN->getChain(), Ptr };
40030           SDValue BcastLd = DAG.getMemIntrinsicNode(
40031               X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
40032               LN->getPointerInfo().getWithOffset(Offset),
40033               LN->getOriginalAlign(),
40034               LN->getMemOperand()->getFlags());
40035           DCI.CombineTo(N.getNode(), BcastLd);
40036           DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40037           DCI.recursivelyDeleteUnusedNodes(Src.getNode());
40038           return N; // Return N so it doesn't get rechecked!
40039         }
40040       }
40041     }
40042 
40043     // vbroadcast(vzload X) -> vbroadcast_load X
40044     if (Src.getOpcode() == X86ISD::VZEXT_LOAD && Src.hasOneUse()) {
40045       MemSDNode *LN = cast<MemIntrinsicSDNode>(Src);
40046       if (LN->getMemoryVT().getSizeInBits() == VT.getScalarSizeInBits()) {
40047         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40048         SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
40049         SDValue BcastLd =
40050             DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
40051                                     LN->getMemoryVT(), LN->getMemOperand());
40052         DCI.CombineTo(N.getNode(), BcastLd);
40053         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40054         DCI.recursivelyDeleteUnusedNodes(LN);
40055         return N; // Return N so it doesn't get rechecked!
40056       }
40057     }
40058 
40059     // vbroadcast(vector load X) -> vbroadcast_load
40060     if ((SrcVT == MVT::v2f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v2i64 ||
40061          SrcVT == MVT::v4i32) &&
40062         Src.hasOneUse() && ISD::isNormalLoad(Src.getNode())) {
40063       LoadSDNode *LN = cast<LoadSDNode>(Src);
40064       // Unless the load is volatile or atomic.
40065       if (LN->isSimple()) {
40066         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40067         SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
40068         SDValue BcastLd = DAG.getMemIntrinsicNode(
40069             X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SrcVT.getScalarType(),
40070             LN->getPointerInfo(), LN->getOriginalAlign(),
40071             LN->getMemOperand()->getFlags());
40072         DCI.CombineTo(N.getNode(), BcastLd);
40073         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40074         DCI.recursivelyDeleteUnusedNodes(LN);
40075         return N; // Return N so it doesn't get rechecked!
40076       }
40077     }
40078 
40079     return SDValue();
40080   }
40081   case X86ISD::VZEXT_MOVL: {
40082     SDValue N0 = N.getOperand(0);
40083 
40084     // If this a vzmovl of a full vector load, replace it with a vzload, unless
40085     // the load is volatile.
40086     if (N0.hasOneUse() && ISD::isNormalLoad(N0.getNode())) {
40087       auto *LN = cast<LoadSDNode>(N0);
40088       if (SDValue VZLoad =
40089               narrowLoadToVZLoad(LN, VT.getVectorElementType(), VT, DAG)) {
40090         DCI.CombineTo(N.getNode(), VZLoad);
40091         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
40092         DCI.recursivelyDeleteUnusedNodes(LN);
40093         return N;
40094       }
40095     }
40096 
40097     // If this a VZEXT_MOVL of a VBROADCAST_LOAD, we don't need the broadcast
40098     // and can just use a VZEXT_LOAD.
40099     // FIXME: Is there some way to do this with SimplifyDemandedVectorElts?
40100     if (N0.hasOneUse() && N0.getOpcode() == X86ISD::VBROADCAST_LOAD) {
40101       auto *LN = cast<MemSDNode>(N0);
40102       if (VT.getScalarSizeInBits() == LN->getMemoryVT().getSizeInBits()) {
40103         SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40104         SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
40105         SDValue VZLoad =
40106             DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops,
40107                                     LN->getMemoryVT(), LN->getMemOperand());
40108         DCI.CombineTo(N.getNode(), VZLoad);
40109         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
40110         DCI.recursivelyDeleteUnusedNodes(LN);
40111         return N;
40112       }
40113     }
40114 
40115     // Turn (v2i64 (vzext_movl (scalar_to_vector (i64 X)))) into
40116     // (v2i64 (bitcast (v4i32 (vzext_movl (scalar_to_vector (i32 (trunc X)))))))
40117     // if the upper bits of the i64 are zero.
40118     if (N0.hasOneUse() && N0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
40119         N0.getOperand(0).hasOneUse() &&
40120         N0.getOperand(0).getValueType() == MVT::i64) {
40121       SDValue In = N0.getOperand(0);
40122       APInt Mask = APInt::getHighBitsSet(64, 32);
40123       if (DAG.MaskedValueIsZero(In, Mask)) {
40124         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In);
40125         MVT VecVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);
40126         SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc);
40127         SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec);
40128         return DAG.getBitcast(VT, Movl);
40129       }
40130     }
40131 
40132     // Load a scalar integer constant directly to XMM instead of transferring an
40133     // immediate value from GPR.
40134     // vzext_movl (scalar_to_vector C) --> load [C,0...]
40135     if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR) {
40136       if (auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
40137         // Create a vector constant - scalar constant followed by zeros.
40138         EVT ScalarVT = N0.getOperand(0).getValueType();
40139         Type *ScalarTy = ScalarVT.getTypeForEVT(*DAG.getContext());
40140         unsigned NumElts = VT.getVectorNumElements();
40141         Constant *Zero = ConstantInt::getNullValue(ScalarTy);
40142         SmallVector<Constant *, 32> ConstantVec(NumElts, Zero);
40143         ConstantVec[0] = const_cast<ConstantInt *>(C->getConstantIntValue());
40144 
40145         // Load the vector constant from constant pool.
40146         MVT PVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
40147         SDValue CP = DAG.getConstantPool(ConstantVector::get(ConstantVec), PVT);
40148         MachinePointerInfo MPI =
40149             MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
40150         Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
40151         return DAG.getLoad(VT, DL, DAG.getEntryNode(), CP, MPI, Alignment,
40152                            MachineMemOperand::MOLoad);
40153       }
40154     }
40155 
40156     // Pull subvector inserts into undef through VZEXT_MOVL by making it an
40157     // insert into a zero vector. This helps get VZEXT_MOVL closer to
40158     // scalar_to_vectors where 256/512 are canonicalized to an insert and a
40159     // 128-bit scalar_to_vector. This reduces the number of isel patterns.
40160     if (!DCI.isBeforeLegalizeOps() && N0.hasOneUse()) {
40161       SDValue V = peekThroughOneUseBitcasts(N0);
40162 
40163       if (V.getOpcode() == ISD::INSERT_SUBVECTOR && V.getOperand(0).isUndef() &&
40164           isNullConstant(V.getOperand(2))) {
40165         SDValue In = V.getOperand(1);
40166         MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(),
40167                                      In.getValueSizeInBits() /
40168                                          VT.getScalarSizeInBits());
40169         In = DAG.getBitcast(SubVT, In);
40170         SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, SubVT, In);
40171         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
40172                            getZeroVector(VT, Subtarget, DAG, DL), Movl,
40173                            V.getOperand(2));
40174       }
40175     }
40176 
40177     return SDValue();
40178   }
40179   case X86ISD::BLENDI: {
40180     SDValue N0 = N.getOperand(0);
40181     SDValue N1 = N.getOperand(1);
40182 
40183     // blend(bitcast(x),bitcast(y)) -> bitcast(blend(x,y)) to narrower types.
40184     // TODO: Handle MVT::v16i16 repeated blend mask.
40185     if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST &&
40186         N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType()) {
40187       MVT SrcVT = N0.getOperand(0).getSimpleValueType();
40188       if ((VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
40189           SrcVT.getScalarSizeInBits() >= 32) {
40190         unsigned BlendMask = N.getConstantOperandVal(2);
40191         unsigned Size = VT.getVectorNumElements();
40192         unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
40193         BlendMask = scaleVectorShuffleBlendMask(BlendMask, Size, Scale);
40194         return DAG.getBitcast(
40195             VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
40196                             N1.getOperand(0),
40197                             DAG.getTargetConstant(BlendMask, DL, MVT::i8)));
40198       }
40199     }
40200     return SDValue();
40201   }
40202   case X86ISD::SHUFP: {
40203     // Fold shufps(shuffle(x),shuffle(y)) -> shufps(x,y).
40204     // This is a more relaxed shuffle combiner that can ignore oneuse limits.
40205     // TODO: Support types other than v4f32.
40206     if (VT == MVT::v4f32) {
40207       bool Updated = false;
40208       SmallVector<int> Mask;
40209       SmallVector<SDValue> Ops;
40210       if (getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask) &&
40211           Ops.size() == 2) {
40212         for (int i = 0; i != 2; ++i) {
40213           SmallVector<SDValue> SubOps;
40214           SmallVector<int> SubMask, SubScaledMask;
40215           SDValue Sub = peekThroughBitcasts(Ops[i]);
40216           // TODO: Scaling might be easier if we specify the demanded elts.
40217           if (getTargetShuffleInputs(Sub, SubOps, SubMask, DAG, 0, false) &&
40218               scaleShuffleElements(SubMask, 4, SubScaledMask) &&
40219               SubOps.size() == 1 && isUndefOrInRange(SubScaledMask, 0, 4)) {
40220             int Ofs = i * 2;
40221             Mask[Ofs + 0] = SubScaledMask[Mask[Ofs + 0] % 4] + (i * 4);
40222             Mask[Ofs + 1] = SubScaledMask[Mask[Ofs + 1] % 4] + (i * 4);
40223             Ops[i] = DAG.getBitcast(VT, SubOps[0]);
40224             Updated = true;
40225           }
40226         }
40227       }
40228       if (Updated) {
40229         for (int &M : Mask)
40230           M %= 4;
40231         Ops.push_back(getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
40232         return DAG.getNode(X86ISD::SHUFP, DL, VT, Ops);
40233       }
40234     }
40235     return SDValue();
40236   }
40237   case X86ISD::VPERMI: {
40238     // vpermi(bitcast(x)) -> bitcast(vpermi(x)) for same number of elements.
40239     // TODO: Remove when we have preferred domains in combineX86ShuffleChain.
40240     SDValue N0 = N.getOperand(0);
40241     SDValue N1 = N.getOperand(1);
40242     unsigned EltSizeInBits = VT.getScalarSizeInBits();
40243     if (N0.getOpcode() == ISD::BITCAST &&
40244         N0.getOperand(0).getScalarValueSizeInBits() == EltSizeInBits) {
40245       SDValue Src = N0.getOperand(0);
40246       EVT SrcVT = Src.getValueType();
40247       SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
40248       return DAG.getBitcast(VT, Res);
40249     }
40250     return SDValue();
40251   }
40252   case X86ISD::SHUF128: {
40253     // If we're permuting the upper 256-bits subvectors of a concatenation, then
40254     // see if we can peek through and access the subvector directly.
40255     if (VT.is512BitVector()) {
40256       // 512-bit mask uses 4 x i2 indices - if the msb is always set then only the
40257       // upper subvector is used.
40258       SDValue LHS = N->getOperand(0);
40259       SDValue RHS = N->getOperand(1);
40260       uint64_t Mask = N->getConstantOperandVal(2);
40261       SmallVector<SDValue> LHSOps, RHSOps;
40262       SDValue NewLHS, NewRHS;
40263       if ((Mask & 0x0A) == 0x0A &&
40264           collectConcatOps(LHS.getNode(), LHSOps, DAG) && LHSOps.size() == 2) {
40265         NewLHS = widenSubVector(LHSOps[1], false, Subtarget, DAG, DL, 512);
40266         Mask &= ~0x0A;
40267       }
40268       if ((Mask & 0xA0) == 0xA0 &&
40269           collectConcatOps(RHS.getNode(), RHSOps, DAG) && RHSOps.size() == 2) {
40270         NewRHS = widenSubVector(RHSOps[1], false, Subtarget, DAG, DL, 512);
40271         Mask &= ~0xA0;
40272       }
40273       if (NewLHS || NewRHS)
40274         return DAG.getNode(X86ISD::SHUF128, DL, VT, NewLHS ? NewLHS : LHS,
40275                            NewRHS ? NewRHS : RHS,
40276                            DAG.getTargetConstant(Mask, DL, MVT::i8));
40277     }
40278     return SDValue();
40279   }
40280   case X86ISD::VPERM2X128: {
40281     // Fold vperm2x128(bitcast(x),bitcast(y),c) -> bitcast(vperm2x128(x,y,c)).
40282     SDValue LHS = N->getOperand(0);
40283     SDValue RHS = N->getOperand(1);
40284     if (LHS.getOpcode() == ISD::BITCAST &&
40285         (RHS.getOpcode() == ISD::BITCAST || RHS.isUndef())) {
40286       EVT SrcVT = LHS.getOperand(0).getValueType();
40287       if (RHS.isUndef() || SrcVT == RHS.getOperand(0).getValueType()) {
40288         return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT,
40289                                               DAG.getBitcast(SrcVT, LHS),
40290                                               DAG.getBitcast(SrcVT, RHS),
40291                                               N->getOperand(2)));
40292       }
40293     }
40294 
40295     // Fold vperm2x128(op(),op()) -> op(vperm2x128(),vperm2x128()).
40296     if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL))
40297       return Res;
40298 
40299     // Fold vperm2x128 subvector shuffle with an inner concat pattern.
40300     // vperm2x128(concat(X,Y),concat(Z,W)) --> concat X,Y etc.
40301     auto FindSubVector128 = [&](unsigned Idx) {
40302       if (Idx > 3)
40303         return SDValue();
40304       SDValue Src = peekThroughBitcasts(N.getOperand(Idx < 2 ? 0 : 1));
40305       SmallVector<SDValue> SubOps;
40306       if (collectConcatOps(Src.getNode(), SubOps, DAG) && SubOps.size() == 2)
40307         return SubOps[Idx & 1];
40308       unsigned NumElts = Src.getValueType().getVectorNumElements();
40309       if ((Idx & 1) == 1 && Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
40310           Src.getOperand(1).getValueSizeInBits() == 128 &&
40311           Src.getConstantOperandAPInt(2) == (NumElts / 2)) {
40312         return Src.getOperand(1);
40313       }
40314       return SDValue();
40315     };
40316     unsigned Imm = N.getConstantOperandVal(2);
40317     if (SDValue SubLo = FindSubVector128(Imm & 0x0F)) {
40318       if (SDValue SubHi = FindSubVector128((Imm & 0xF0) >> 4)) {
40319         MVT SubVT = VT.getHalfNumVectorElementsVT();
40320         SubLo = DAG.getBitcast(SubVT, SubLo);
40321         SubHi = DAG.getBitcast(SubVT, SubHi);
40322         return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi);
40323       }
40324     }
40325     return SDValue();
40326   }
40327   case X86ISD::PSHUFD:
40328   case X86ISD::PSHUFLW:
40329   case X86ISD::PSHUFHW: {
40330     SDValue N0 = N.getOperand(0);
40331     SDValue N1 = N.getOperand(1);
40332     if (N0->hasOneUse()) {
40333       SDValue V = peekThroughOneUseBitcasts(N0);
40334       switch (V.getOpcode()) {
40335       case X86ISD::VSHL:
40336       case X86ISD::VSRL:
40337       case X86ISD::VSRA:
40338       case X86ISD::VSHLI:
40339       case X86ISD::VSRLI:
40340       case X86ISD::VSRAI:
40341       case X86ISD::VROTLI:
40342       case X86ISD::VROTRI: {
40343         MVT InnerVT = V.getSimpleValueType();
40344         if (InnerVT.getScalarSizeInBits() <= VT.getScalarSizeInBits()) {
40345           SDValue Res = DAG.getNode(Opcode, DL, VT,
40346                                     DAG.getBitcast(VT, V.getOperand(0)), N1);
40347           Res = DAG.getBitcast(InnerVT, Res);
40348           Res = DAG.getNode(V.getOpcode(), DL, InnerVT, Res, V.getOperand(1));
40349           return DAG.getBitcast(VT, Res);
40350         }
40351         break;
40352       }
40353       }
40354     }
40355 
40356     Mask = getPSHUFShuffleMask(N);
40357     assert(Mask.size() == 4);
40358     break;
40359   }
40360   case X86ISD::MOVSD:
40361   case X86ISD::MOVSH:
40362   case X86ISD::MOVSS: {
40363     SDValue N0 = N.getOperand(0);
40364     SDValue N1 = N.getOperand(1);
40365 
40366     // Canonicalize scalar FPOps:
40367     // MOVS*(N0, OP(N0, N1)) --> MOVS*(N0, SCALAR_TO_VECTOR(OP(N0[0], N1[0])))
40368     // If commutable, allow OP(N1[0], N0[0]).
40369     unsigned Opcode1 = N1.getOpcode();
40370     if (Opcode1 == ISD::FADD || Opcode1 == ISD::FMUL || Opcode1 == ISD::FSUB ||
40371         Opcode1 == ISD::FDIV) {
40372       SDValue N10 = N1.getOperand(0);
40373       SDValue N11 = N1.getOperand(1);
40374       if (N10 == N0 ||
40375           (N11 == N0 && (Opcode1 == ISD::FADD || Opcode1 == ISD::FMUL))) {
40376         if (N10 != N0)
40377           std::swap(N10, N11);
40378         MVT SVT = VT.getVectorElementType();
40379         SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
40380         N10 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N10, ZeroIdx);
40381         N11 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N11, ZeroIdx);
40382         SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
40383         SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
40384         return DAG.getNode(Opcode, DL, VT, N0, SclVec);
40385       }
40386     }
40387 
40388     return SDValue();
40389   }
40390   case X86ISD::INSERTPS: {
40391     assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32");
40392     SDValue Op0 = N.getOperand(0);
40393     SDValue Op1 = N.getOperand(1);
40394     unsigned InsertPSMask = N.getConstantOperandVal(2);
40395     unsigned SrcIdx = (InsertPSMask >> 6) & 0x3;
40396     unsigned DstIdx = (InsertPSMask >> 4) & 0x3;
40397     unsigned ZeroMask = InsertPSMask & 0xF;
40398 
40399     // If we zero out all elements from Op0 then we don't need to reference it.
40400     if (((ZeroMask | (1u << DstIdx)) == 0xF) && !Op0.isUndef())
40401       return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
40402                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40403 
40404     // If we zero out the element from Op1 then we don't need to reference it.
40405     if ((ZeroMask & (1u << DstIdx)) && !Op1.isUndef())
40406       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
40407                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40408 
40409     // Attempt to merge insertps Op1 with an inner target shuffle node.
40410     SmallVector<int, 8> TargetMask1;
40411     SmallVector<SDValue, 2> Ops1;
40412     APInt KnownUndef1, KnownZero1;
40413     if (getTargetShuffleAndZeroables(Op1, TargetMask1, Ops1, KnownUndef1,
40414                                      KnownZero1)) {
40415       if (KnownUndef1[SrcIdx] || KnownZero1[SrcIdx]) {
40416         // Zero/UNDEF insertion - zero out element and remove dependency.
40417         InsertPSMask |= (1u << DstIdx);
40418         return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
40419                            DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40420       }
40421       // Update insertps mask srcidx and reference the source input directly.
40422       int M = TargetMask1[SrcIdx];
40423       assert(0 <= M && M < 8 && "Shuffle index out of range");
40424       InsertPSMask = (InsertPSMask & 0x3f) | ((M & 0x3) << 6);
40425       Op1 = Ops1[M < 4 ? 0 : 1];
40426       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
40427                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40428     }
40429 
40430     // Attempt to merge insertps Op0 with an inner target shuffle node.
40431     SmallVector<int, 8> TargetMask0;
40432     SmallVector<SDValue, 2> Ops0;
40433     APInt KnownUndef0, KnownZero0;
40434     if (getTargetShuffleAndZeroables(Op0, TargetMask0, Ops0, KnownUndef0,
40435                                      KnownZero0)) {
40436       bool Updated = false;
40437       bool UseInput00 = false;
40438       bool UseInput01 = false;
40439       for (int i = 0; i != 4; ++i) {
40440         if ((InsertPSMask & (1u << i)) || (i == (int)DstIdx)) {
40441           // No change if element is already zero or the inserted element.
40442           continue;
40443         }
40444 
40445         if (KnownUndef0[i] || KnownZero0[i]) {
40446           // If the target mask is undef/zero then we must zero the element.
40447           InsertPSMask |= (1u << i);
40448           Updated = true;
40449           continue;
40450         }
40451 
40452         // The input vector element must be inline.
40453         int M = TargetMask0[i];
40454         if (M != i && M != (i + 4))
40455           return SDValue();
40456 
40457         // Determine which inputs of the target shuffle we're using.
40458         UseInput00 |= (0 <= M && M < 4);
40459         UseInput01 |= (4 <= M);
40460       }
40461 
40462       // If we're not using both inputs of the target shuffle then use the
40463       // referenced input directly.
40464       if (UseInput00 && !UseInput01) {
40465         Updated = true;
40466         Op0 = Ops0[0];
40467       } else if (!UseInput00 && UseInput01) {
40468         Updated = true;
40469         Op0 = Ops0[1];
40470       }
40471 
40472       if (Updated)
40473         return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
40474                            DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
40475     }
40476 
40477     // If we're inserting an element from a vbroadcast load, fold the
40478     // load into the X86insertps instruction. We need to convert the scalar
40479     // load to a vector and clear the source lane of the INSERTPS control.
40480     if (Op1.getOpcode() == X86ISD::VBROADCAST_LOAD && Op1.hasOneUse()) {
40481       auto *MemIntr = cast<MemIntrinsicSDNode>(Op1);
40482       if (MemIntr->getMemoryVT().getScalarSizeInBits() == 32) {
40483         SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
40484                                    MemIntr->getBasePtr(),
40485                                    MemIntr->getMemOperand());
40486         SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
40487                            DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
40488                                        Load),
40489                            DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
40490         DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
40491         return Insert;
40492       }
40493     }
40494 
40495     return SDValue();
40496   }
40497   default:
40498     return SDValue();
40499   }
40500 
40501   // Nuke no-op shuffles that show up after combining.
40502   if (isNoopShuffleMask(Mask))
40503     return N.getOperand(0);
40504 
40505   // Look for simplifications involving one or two shuffle instructions.
40506   SDValue V = N.getOperand(0);
40507   switch (N.getOpcode()) {
40508   default:
40509     break;
40510   case X86ISD::PSHUFLW:
40511   case X86ISD::PSHUFHW:
40512     assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!");
40513 
40514     // See if this reduces to a PSHUFD which is no more expensive and can
40515     // combine with more operations. Note that it has to at least flip the
40516     // dwords as otherwise it would have been removed as a no-op.
40517     if (ArrayRef<int>(Mask).equals({2, 3, 0, 1})) {
40518       int DMask[] = {0, 1, 2, 3};
40519       int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
40520       DMask[DOffset + 0] = DOffset + 1;
40521       DMask[DOffset + 1] = DOffset + 0;
40522       MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
40523       V = DAG.getBitcast(DVT, V);
40524       V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
40525                       getV4X86ShuffleImm8ForMask(DMask, DL, DAG));
40526       return DAG.getBitcast(VT, V);
40527     }
40528 
40529     // Look for shuffle patterns which can be implemented as a single unpack.
40530     // FIXME: This doesn't handle the location of the PSHUFD generically, and
40531     // only works when we have a PSHUFD followed by two half-shuffles.
40532     if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
40533         (V.getOpcode() == X86ISD::PSHUFLW ||
40534          V.getOpcode() == X86ISD::PSHUFHW) &&
40535         V.getOpcode() != N.getOpcode() &&
40536         V.hasOneUse() && V.getOperand(0).hasOneUse()) {
40537       SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
40538       if (D.getOpcode() == X86ISD::PSHUFD) {
40539         SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
40540         SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
40541         int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
40542         int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
40543         int WordMask[8];
40544         for (int i = 0; i < 4; ++i) {
40545           WordMask[i + NOffset] = Mask[i] + NOffset;
40546           WordMask[i + VOffset] = VMask[i] + VOffset;
40547         }
40548         // Map the word mask through the DWord mask.
40549         int MappedMask[8];
40550         for (int i = 0; i < 8; ++i)
40551           MappedMask[i] = 2 * DMask[WordMask[i] / 2] + WordMask[i] % 2;
40552         if (ArrayRef<int>(MappedMask).equals({0, 0, 1, 1, 2, 2, 3, 3}) ||
40553             ArrayRef<int>(MappedMask).equals({4, 4, 5, 5, 6, 6, 7, 7})) {
40554           // We can replace all three shuffles with an unpack.
40555           V = DAG.getBitcast(VT, D.getOperand(0));
40556           return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
40557                                                 : X86ISD::UNPCKH,
40558                              DL, VT, V, V);
40559         }
40560       }
40561     }
40562 
40563     break;
40564 
40565   case X86ISD::PSHUFD:
40566     if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DAG))
40567       return NewN;
40568 
40569     break;
40570   }
40571 
40572   return SDValue();
40573 }
40574 
40575 /// Checks if the shuffle mask takes subsequent elements
40576 /// alternately from two vectors.
40577 /// For example <0, 5, 2, 7> or <8, 1, 10, 3, 12, 5, 14, 7> are both correct.
40578 static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
40579 
40580   int ParitySrc[2] = {-1, -1};
40581   unsigned Size = Mask.size();
40582   for (unsigned i = 0; i != Size; ++i) {
40583     int M = Mask[i];
40584     if (M < 0)
40585       continue;
40586 
40587     // Make sure we are using the matching element from the input.
40588     if ((M % Size) != i)
40589       return false;
40590 
40591     // Make sure we use the same input for all elements of the same parity.
40592     int Src = M / Size;
40593     if (ParitySrc[i % 2] >= 0 && ParitySrc[i % 2] != Src)
40594       return false;
40595     ParitySrc[i % 2] = Src;
40596   }
40597 
40598   // Make sure each input is used.
40599   if (ParitySrc[0] < 0 || ParitySrc[1] < 0 || ParitySrc[0] == ParitySrc[1])
40600     return false;
40601 
40602   Op0Even = ParitySrc[0] == 0;
40603   return true;
40604 }
40605 
40606 /// Returns true iff the shuffle node \p N can be replaced with ADDSUB(SUBADD)
40607 /// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
40608 /// are written to the parameters \p Opnd0 and \p Opnd1.
40609 ///
40610 /// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
40611 /// so it is easier to generically match. We also insert dummy vector shuffle
40612 /// nodes for the operands which explicitly discard the lanes which are unused
40613 /// by this operation to try to flow through the rest of the combiner
40614 /// the fact that they're unused.
40615 static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
40616                              SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
40617                              bool &IsSubAdd) {
40618 
40619   EVT VT = N->getValueType(0);
40620   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40621   if (!Subtarget.hasSSE3() || !TLI.isTypeLegal(VT) ||
40622       !VT.getSimpleVT().isFloatingPoint())
40623     return false;
40624 
40625   // We only handle target-independent shuffles.
40626   // FIXME: It would be easy and harmless to use the target shuffle mask
40627   // extraction tool to support more.
40628   if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
40629     return false;
40630 
40631   SDValue V1 = N->getOperand(0);
40632   SDValue V2 = N->getOperand(1);
40633 
40634   // Make sure we have an FADD and an FSUB.
40635   if ((V1.getOpcode() != ISD::FADD && V1.getOpcode() != ISD::FSUB) ||
40636       (V2.getOpcode() != ISD::FADD && V2.getOpcode() != ISD::FSUB) ||
40637       V1.getOpcode() == V2.getOpcode())
40638     return false;
40639 
40640   // If there are other uses of these operations we can't fold them.
40641   if (!V1->hasOneUse() || !V2->hasOneUse())
40642     return false;
40643 
40644   // Ensure that both operations have the same operands. Note that we can
40645   // commute the FADD operands.
40646   SDValue LHS, RHS;
40647   if (V1.getOpcode() == ISD::FSUB) {
40648     LHS = V1->getOperand(0); RHS = V1->getOperand(1);
40649     if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
40650         (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
40651       return false;
40652   } else {
40653     assert(V2.getOpcode() == ISD::FSUB && "Unexpected opcode");
40654     LHS = V2->getOperand(0); RHS = V2->getOperand(1);
40655     if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) &&
40656         (V1->getOperand(0) != RHS || V1->getOperand(1) != LHS))
40657       return false;
40658   }
40659 
40660   ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
40661   bool Op0Even;
40662   if (!isAddSubOrSubAddMask(Mask, Op0Even))
40663     return false;
40664 
40665   // It's a subadd if the vector in the even parity is an FADD.
40666   IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
40667                      : V2->getOpcode() == ISD::FADD;
40668 
40669   Opnd0 = LHS;
40670   Opnd1 = RHS;
40671   return true;
40672 }
40673 
40674 /// Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
40675 static SDValue combineShuffleToFMAddSub(SDNode *N,
40676                                         const X86Subtarget &Subtarget,
40677                                         SelectionDAG &DAG) {
40678   // We only handle target-independent shuffles.
40679   // FIXME: It would be easy and harmless to use the target shuffle mask
40680   // extraction tool to support more.
40681   if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
40682     return SDValue();
40683 
40684   MVT VT = N->getSimpleValueType(0);
40685   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40686   if (!Subtarget.hasAnyFMA() || !TLI.isTypeLegal(VT))
40687     return SDValue();
40688 
40689   // We're trying to match (shuffle fma(a, b, c), X86Fmsub(a, b, c).
40690   SDValue Op0 = N->getOperand(0);
40691   SDValue Op1 = N->getOperand(1);
40692   SDValue FMAdd = Op0, FMSub = Op1;
40693   if (FMSub.getOpcode() != X86ISD::FMSUB)
40694     std::swap(FMAdd, FMSub);
40695 
40696   if (FMAdd.getOpcode() != ISD::FMA || FMSub.getOpcode() != X86ISD::FMSUB ||
40697       FMAdd.getOperand(0) != FMSub.getOperand(0) || !FMAdd.hasOneUse() ||
40698       FMAdd.getOperand(1) != FMSub.getOperand(1) || !FMSub.hasOneUse() ||
40699       FMAdd.getOperand(2) != FMSub.getOperand(2))
40700     return SDValue();
40701 
40702   // Check for correct shuffle mask.
40703   ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
40704   bool Op0Even;
40705   if (!isAddSubOrSubAddMask(Mask, Op0Even))
40706     return SDValue();
40707 
40708   // FMAddSub takes zeroth operand from FMSub node.
40709   SDLoc DL(N);
40710   bool IsSubAdd = Op0Even ? Op0 == FMAdd : Op1 == FMAdd;
40711   unsigned Opcode = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
40712   return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1),
40713                      FMAdd.getOperand(2));
40714 }
40715 
40716 /// Try to combine a shuffle into a target-specific add-sub or
40717 /// mul-add-sub node.
40718 static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N,
40719                                                 const X86Subtarget &Subtarget,
40720                                                 SelectionDAG &DAG) {
40721   if (SDValue V = combineShuffleToFMAddSub(N, Subtarget, DAG))
40722     return V;
40723 
40724   SDValue Opnd0, Opnd1;
40725   bool IsSubAdd;
40726   if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd))
40727     return SDValue();
40728 
40729   MVT VT = N->getSimpleValueType(0);
40730   SDLoc DL(N);
40731 
40732   // Try to generate X86ISD::FMADDSUB node here.
40733   SDValue Opnd2;
40734   if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) {
40735     unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
40736     return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
40737   }
40738 
40739   if (IsSubAdd)
40740     return SDValue();
40741 
40742   // Do not generate X86ISD::ADDSUB node for 512-bit types even though
40743   // the ADDSUB idiom has been successfully recognized. There are no known
40744   // X86 targets with 512-bit ADDSUB instructions!
40745   if (VT.is512BitVector())
40746     return SDValue();
40747 
40748   // Do not generate X86ISD::ADDSUB node for FP16's vector types even though
40749   // the ADDSUB idiom has been successfully recognized. There are no known
40750   // X86 targets with FP16 ADDSUB instructions!
40751   if (VT.getVectorElementType() == MVT::f16)
40752     return SDValue();
40753 
40754   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
40755 }
40756 
40757 // We are looking for a shuffle where both sources are concatenated with undef
40758 // and have a width that is half of the output's width. AVX2 has VPERMD/Q, so
40759 // if we can express this as a single-source shuffle, that's preferable.
40760 static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
40761                                            const X86Subtarget &Subtarget) {
40762   if (!Subtarget.hasAVX2() || !isa<ShuffleVectorSDNode>(N))
40763     return SDValue();
40764 
40765   EVT VT = N->getValueType(0);
40766 
40767   // We only care about shuffles of 128/256-bit vectors of 32/64-bit values.
40768   if (!VT.is128BitVector() && !VT.is256BitVector())
40769     return SDValue();
40770 
40771   if (VT.getVectorElementType() != MVT::i32 &&
40772       VT.getVectorElementType() != MVT::i64 &&
40773       VT.getVectorElementType() != MVT::f32 &&
40774       VT.getVectorElementType() != MVT::f64)
40775     return SDValue();
40776 
40777   SDValue N0 = N->getOperand(0);
40778   SDValue N1 = N->getOperand(1);
40779 
40780   // Check that both sources are concats with undef.
40781   if (N0.getOpcode() != ISD::CONCAT_VECTORS ||
40782       N1.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 ||
40783       N1.getNumOperands() != 2 || !N0.getOperand(1).isUndef() ||
40784       !N1.getOperand(1).isUndef())
40785     return SDValue();
40786 
40787   // Construct the new shuffle mask. Elements from the first source retain their
40788   // index, but elements from the second source no longer need to skip an undef.
40789   SmallVector<int, 8> Mask;
40790   int NumElts = VT.getVectorNumElements();
40791 
40792   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
40793   for (int Elt : SVOp->getMask())
40794     Mask.push_back(Elt < NumElts ? Elt : (Elt - NumElts / 2));
40795 
40796   SDLoc DL(N);
40797   SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
40798                                N1.getOperand(0));
40799   return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask);
40800 }
40801 
40802 /// If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the
40803 /// low half of each source vector and does not set any high half elements in
40804 /// the destination vector, narrow the shuffle to half its original size.
40805 static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
40806   EVT VT = Shuf->getValueType(0);
40807   if (!DAG.getTargetLoweringInfo().isTypeLegal(Shuf->getValueType(0)))
40808     return SDValue();
40809   if (!VT.is256BitVector() && !VT.is512BitVector())
40810     return SDValue();
40811 
40812   // See if we can ignore all of the high elements of the shuffle.
40813   ArrayRef<int> Mask = Shuf->getMask();
40814   if (!isUndefUpperHalf(Mask))
40815     return SDValue();
40816 
40817   // Check if the shuffle mask accesses only the low half of each input vector
40818   // (half-index output is 0 or 2).
40819   int HalfIdx1, HalfIdx2;
40820   SmallVector<int, 8> HalfMask(Mask.size() / 2);
40821   if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2) ||
40822       (HalfIdx1 % 2 == 1) || (HalfIdx2 % 2 == 1))
40823     return SDValue();
40824 
40825   // Create a half-width shuffle to replace the unnecessarily wide shuffle.
40826   // The trick is knowing that all of the insert/extract are actually free
40827   // subregister (zmm<->ymm or ymm<->xmm) ops. That leaves us with a shuffle
40828   // of narrow inputs into a narrow output, and that is always cheaper than
40829   // the wide shuffle that we started with.
40830   return getShuffleHalfVectors(SDLoc(Shuf), Shuf->getOperand(0),
40831                                Shuf->getOperand(1), HalfMask, HalfIdx1,
40832                                HalfIdx2, false, DAG, /*UseConcat*/ true);
40833 }
40834 
40835 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
40836                               TargetLowering::DAGCombinerInfo &DCI,
40837                               const X86Subtarget &Subtarget) {
40838   if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N))
40839     if (SDValue V = narrowShuffle(Shuf, DAG))
40840       return V;
40841 
40842   // If we have legalized the vector types, look for blends of FADD and FSUB
40843   // nodes that we can fuse into an ADDSUB, FMADDSUB, or FMSUBADD node.
40844   SDLoc dl(N);
40845   EVT VT = N->getValueType(0);
40846   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40847   if (TLI.isTypeLegal(VT) && !isSoftF16(VT, Subtarget))
40848     if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
40849       return AddSub;
40850 
40851   // Attempt to combine into a vector load/broadcast.
40852   if (SDValue LD = combineToConsecutiveLoads(
40853           VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true))
40854     return LD;
40855 
40856   // For AVX2, we sometimes want to combine
40857   // (vector_shuffle <mask> (concat_vectors t1, undef)
40858   //                        (concat_vectors t2, undef))
40859   // Into:
40860   // (vector_shuffle <mask> (concat_vectors t1, t2), undef)
40861   // Since the latter can be efficiently lowered with VPERMD/VPERMQ
40862   if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, DAG, Subtarget))
40863     return ShufConcat;
40864 
40865   if (isTargetShuffle(N->getOpcode())) {
40866     SDValue Op(N, 0);
40867     if (SDValue Shuffle = combineTargetShuffle(Op, DAG, DCI, Subtarget))
40868       return Shuffle;
40869 
40870     // Try recursively combining arbitrary sequences of x86 shuffle
40871     // instructions into higher-order shuffles. We do this after combining
40872     // specific PSHUF instruction sequences into their minimal form so that we
40873     // can evaluate how many specialized shuffle instructions are involved in
40874     // a particular chain.
40875     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
40876       return Res;
40877 
40878     // Simplify source operands based on shuffle mask.
40879     // TODO - merge this into combineX86ShufflesRecursively.
40880     APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
40881     if (TLI.SimplifyDemandedVectorElts(Op, DemandedElts, DCI))
40882       return SDValue(N, 0);
40883 
40884     // Canonicalize SHUFFLE(UNARYOP(X)) -> UNARYOP(SHUFFLE(X)).
40885     // Canonicalize SHUFFLE(BINOP(X,Y)) -> BINOP(SHUFFLE(X),SHUFFLE(Y)).
40886     // Perform this after other shuffle combines to allow inner shuffles to be
40887     // combined away first.
40888     if (SDValue BinOp = canonicalizeShuffleWithOp(Op, DAG, dl))
40889       return BinOp;
40890   }
40891 
40892   return SDValue();
40893 }
40894 
40895 // Simplify variable target shuffle masks based on the demanded elements.
40896 // TODO: Handle DemandedBits in mask indices as well?
40897 bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle(
40898     SDValue Op, const APInt &DemandedElts, unsigned MaskIndex,
40899     TargetLowering::TargetLoweringOpt &TLO, unsigned Depth) const {
40900   // If we're demanding all elements don't bother trying to simplify the mask.
40901   unsigned NumElts = DemandedElts.getBitWidth();
40902   if (DemandedElts.isAllOnes())
40903     return false;
40904 
40905   SDValue Mask = Op.getOperand(MaskIndex);
40906   if (!Mask.hasOneUse())
40907     return false;
40908 
40909   // Attempt to generically simplify the variable shuffle mask.
40910   APInt MaskUndef, MaskZero;
40911   if (SimplifyDemandedVectorElts(Mask, DemandedElts, MaskUndef, MaskZero, TLO,
40912                                  Depth + 1))
40913     return true;
40914 
40915   // Attempt to extract+simplify a (constant pool load) shuffle mask.
40916   // TODO: Support other types from getTargetShuffleMaskIndices?
40917   SDValue BC = peekThroughOneUseBitcasts(Mask);
40918   EVT BCVT = BC.getValueType();
40919   auto *Load = dyn_cast<LoadSDNode>(BC);
40920   if (!Load || !Load->getBasePtr().hasOneUse())
40921     return false;
40922 
40923   const Constant *C = getTargetConstantFromNode(Load);
40924   if (!C)
40925     return false;
40926 
40927   Type *CTy = C->getType();
40928   if (!CTy->isVectorTy() ||
40929       CTy->getPrimitiveSizeInBits() != Mask.getValueSizeInBits())
40930     return false;
40931 
40932   // Handle scaling for i64 elements on 32-bit targets.
40933   unsigned NumCstElts = cast<FixedVectorType>(CTy)->getNumElements();
40934   if (NumCstElts != NumElts && NumCstElts != (NumElts * 2))
40935     return false;
40936   unsigned Scale = NumCstElts / NumElts;
40937 
40938   // Simplify mask if we have an undemanded element that is not undef.
40939   bool Simplified = false;
40940   SmallVector<Constant *, 32> ConstVecOps;
40941   for (unsigned i = 0; i != NumCstElts; ++i) {
40942     Constant *Elt = C->getAggregateElement(i);
40943     if (!DemandedElts[i / Scale] && !isa<UndefValue>(Elt)) {
40944       ConstVecOps.push_back(UndefValue::get(Elt->getType()));
40945       Simplified = true;
40946       continue;
40947     }
40948     ConstVecOps.push_back(Elt);
40949   }
40950   if (!Simplified)
40951     return false;
40952 
40953   // Generate new constant pool entry + legalize immediately for the load.
40954   SDLoc DL(Op);
40955   SDValue CV = TLO.DAG.getConstantPool(ConstantVector::get(ConstVecOps), BCVT);
40956   SDValue LegalCV = LowerConstantPool(CV, TLO.DAG);
40957   SDValue NewMask = TLO.DAG.getLoad(
40958       BCVT, DL, TLO.DAG.getEntryNode(), LegalCV,
40959       MachinePointerInfo::getConstantPool(TLO.DAG.getMachineFunction()),
40960       Load->getAlign());
40961   return TLO.CombineTo(Mask, TLO.DAG.getBitcast(Mask.getValueType(), NewMask));
40962 }
40963 
40964 bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
40965     SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
40966     TargetLoweringOpt &TLO, unsigned Depth) const {
40967   int NumElts = DemandedElts.getBitWidth();
40968   unsigned Opc = Op.getOpcode();
40969   EVT VT = Op.getValueType();
40970 
40971   // Handle special case opcodes.
40972   switch (Opc) {
40973   case X86ISD::PMULDQ:
40974   case X86ISD::PMULUDQ: {
40975     APInt LHSUndef, LHSZero;
40976     APInt RHSUndef, RHSZero;
40977     SDValue LHS = Op.getOperand(0);
40978     SDValue RHS = Op.getOperand(1);
40979     if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO,
40980                                    Depth + 1))
40981       return true;
40982     if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO,
40983                                    Depth + 1))
40984       return true;
40985     // Multiply by zero.
40986     KnownZero = LHSZero | RHSZero;
40987     break;
40988   }
40989   case X86ISD::VPMADDWD: {
40990     APInt LHSUndef, LHSZero;
40991     APInt RHSUndef, RHSZero;
40992     SDValue LHS = Op.getOperand(0);
40993     SDValue RHS = Op.getOperand(1);
40994     APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, 2 * NumElts);
40995 
40996     if (SimplifyDemandedVectorElts(LHS, DemandedSrcElts, LHSUndef, LHSZero, TLO,
40997                                    Depth + 1))
40998       return true;
40999     if (SimplifyDemandedVectorElts(RHS, DemandedSrcElts, RHSUndef, RHSZero, TLO,
41000                                    Depth + 1))
41001       return true;
41002 
41003     // TODO: Multiply by zero.
41004 
41005     // If RHS/LHS elements are known zero then we don't need the LHS/RHS equivalent.
41006     APInt DemandedLHSElts = DemandedSrcElts & ~RHSZero;
41007     if (SimplifyDemandedVectorElts(LHS, DemandedLHSElts, LHSUndef, LHSZero, TLO,
41008                                    Depth + 1))
41009       return true;
41010     APInt DemandedRHSElts = DemandedSrcElts & ~LHSZero;
41011     if (SimplifyDemandedVectorElts(RHS, DemandedRHSElts, RHSUndef, RHSZero, TLO,
41012                                    Depth + 1))
41013       return true;
41014     break;
41015   }
41016   case X86ISD::PSADBW: {
41017     SDValue LHS = Op.getOperand(0);
41018     SDValue RHS = Op.getOperand(1);
41019     assert(VT.getScalarType() == MVT::i64 &&
41020            LHS.getValueType() == RHS.getValueType() &&
41021            LHS.getValueType().getScalarType() == MVT::i8 &&
41022            "Unexpected PSADBW types");
41023 
41024     // Aggressively peek through ops to get at the demanded elts.
41025     if (!DemandedElts.isAllOnes()) {
41026       unsigned NumSrcElts = LHS.getValueType().getVectorNumElements();
41027       APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
41028       SDValue NewLHS = SimplifyMultipleUseDemandedVectorElts(
41029           LHS, DemandedSrcElts, TLO.DAG, Depth + 1);
41030       SDValue NewRHS = SimplifyMultipleUseDemandedVectorElts(
41031           RHS, DemandedSrcElts, TLO.DAG, Depth + 1);
41032       if (NewLHS || NewRHS) {
41033         NewLHS = NewLHS ? NewLHS : LHS;
41034         NewRHS = NewRHS ? NewRHS : RHS;
41035         return TLO.CombineTo(
41036             Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS));
41037       }
41038     }
41039     break;
41040   }
41041   case X86ISD::VSHL:
41042   case X86ISD::VSRL:
41043   case X86ISD::VSRA: {
41044     // We only need the bottom 64-bits of the (128-bit) shift amount.
41045     SDValue Amt = Op.getOperand(1);
41046     MVT AmtVT = Amt.getSimpleValueType();
41047     assert(AmtVT.is128BitVector() && "Unexpected value type");
41048 
41049     // If we reuse the shift amount just for sse shift amounts then we know that
41050     // only the bottom 64-bits are only ever used.
41051     bool AssumeSingleUse = llvm::all_of(Amt->uses(), [&Amt](SDNode *Use) {
41052       unsigned UseOpc = Use->getOpcode();
41053       return (UseOpc == X86ISD::VSHL || UseOpc == X86ISD::VSRL ||
41054               UseOpc == X86ISD::VSRA) &&
41055              Use->getOperand(0) != Amt;
41056     });
41057 
41058     APInt AmtUndef, AmtZero;
41059     unsigned NumAmtElts = AmtVT.getVectorNumElements();
41060     APInt AmtElts = APInt::getLowBitsSet(NumAmtElts, NumAmtElts / 2);
41061     if (SimplifyDemandedVectorElts(Amt, AmtElts, AmtUndef, AmtZero, TLO,
41062                                    Depth + 1, AssumeSingleUse))
41063       return true;
41064     [[fallthrough]];
41065   }
41066   case X86ISD::VSHLI:
41067   case X86ISD::VSRLI:
41068   case X86ISD::VSRAI: {
41069     SDValue Src = Op.getOperand(0);
41070     APInt SrcUndef;
41071     if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO,
41072                                    Depth + 1))
41073       return true;
41074 
41075     // Fold shift(0,x) -> 0
41076     if (DemandedElts.isSubsetOf(KnownZero))
41077       return TLO.CombineTo(
41078           Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
41079 
41080     // Aggressively peek through ops to get at the demanded elts.
41081     if (!DemandedElts.isAllOnes())
41082       if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
41083               Src, DemandedElts, TLO.DAG, Depth + 1))
41084         return TLO.CombineTo(
41085             Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1)));
41086     break;
41087   }
41088   case X86ISD::VPSHA:
41089   case X86ISD::VPSHL:
41090   case X86ISD::VSHLV:
41091   case X86ISD::VSRLV:
41092   case X86ISD::VSRAV: {
41093     APInt LHSUndef, LHSZero;
41094     APInt RHSUndef, RHSZero;
41095     SDValue LHS = Op.getOperand(0);
41096     SDValue RHS = Op.getOperand(1);
41097     if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO,
41098                                    Depth + 1))
41099       return true;
41100 
41101     // Fold shift(0,x) -> 0
41102     if (DemandedElts.isSubsetOf(LHSZero))
41103       return TLO.CombineTo(
41104           Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
41105 
41106     if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO,
41107                                    Depth + 1))
41108       return true;
41109 
41110     KnownZero = LHSZero;
41111     break;
41112   }
41113   case X86ISD::KSHIFTL: {
41114     SDValue Src = Op.getOperand(0);
41115     auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
41116     assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount");
41117     unsigned ShiftAmt = Amt->getZExtValue();
41118 
41119     if (ShiftAmt == 0)
41120       return TLO.CombineTo(Op, Src);
41121 
41122     // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
41123     // single shift.  We can do this if the bottom bits (which are shifted
41124     // out) are never demanded.
41125     if (Src.getOpcode() == X86ISD::KSHIFTR) {
41126       if (!DemandedElts.intersects(APInt::getLowBitsSet(NumElts, ShiftAmt))) {
41127         unsigned C1 = Src.getConstantOperandVal(1);
41128         unsigned NewOpc = X86ISD::KSHIFTL;
41129         int Diff = ShiftAmt - C1;
41130         if (Diff < 0) {
41131           Diff = -Diff;
41132           NewOpc = X86ISD::KSHIFTR;
41133         }
41134 
41135         SDLoc dl(Op);
41136         SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
41137         return TLO.CombineTo(
41138             Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
41139       }
41140     }
41141 
41142     APInt DemandedSrc = DemandedElts.lshr(ShiftAmt);
41143     if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
41144                                    Depth + 1))
41145       return true;
41146 
41147     KnownUndef <<= ShiftAmt;
41148     KnownZero <<= ShiftAmt;
41149     KnownZero.setLowBits(ShiftAmt);
41150     break;
41151   }
41152   case X86ISD::KSHIFTR: {
41153     SDValue Src = Op.getOperand(0);
41154     auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
41155     assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount");
41156     unsigned ShiftAmt = Amt->getZExtValue();
41157 
41158     if (ShiftAmt == 0)
41159       return TLO.CombineTo(Op, Src);
41160 
41161     // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
41162     // single shift.  We can do this if the top bits (which are shifted
41163     // out) are never demanded.
41164     if (Src.getOpcode() == X86ISD::KSHIFTL) {
41165       if (!DemandedElts.intersects(APInt::getHighBitsSet(NumElts, ShiftAmt))) {
41166         unsigned C1 = Src.getConstantOperandVal(1);
41167         unsigned NewOpc = X86ISD::KSHIFTR;
41168         int Diff = ShiftAmt - C1;
41169         if (Diff < 0) {
41170           Diff = -Diff;
41171           NewOpc = X86ISD::KSHIFTL;
41172         }
41173 
41174         SDLoc dl(Op);
41175         SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
41176         return TLO.CombineTo(
41177             Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
41178       }
41179     }
41180 
41181     APInt DemandedSrc = DemandedElts.shl(ShiftAmt);
41182     if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
41183                                    Depth + 1))
41184       return true;
41185 
41186     KnownUndef.lshrInPlace(ShiftAmt);
41187     KnownZero.lshrInPlace(ShiftAmt);
41188     KnownZero.setHighBits(ShiftAmt);
41189     break;
41190   }
41191   case X86ISD::ANDNP: {
41192     // ANDNP = (~LHS & RHS);
41193     SDValue LHS = Op.getOperand(0);
41194     SDValue RHS = Op.getOperand(1);
41195 
41196     auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) {
41197       APInt UndefElts;
41198       SmallVector<APInt> EltBits;
41199       int NumElts = VT.getVectorNumElements();
41200       int EltSizeInBits = VT.getScalarSizeInBits();
41201       APInt OpBits = APInt::getAllOnes(EltSizeInBits);
41202       APInt OpElts = DemandedElts;
41203       if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
41204                                         EltBits)) {
41205         OpBits.clearAllBits();
41206         OpElts.clearAllBits();
41207         for (int I = 0; I != NumElts; ++I) {
41208           if (!DemandedElts[I])
41209             continue;
41210           if (UndefElts[I]) {
41211             // We can't assume an undef src element gives an undef dst - the
41212             // other src might be zero.
41213             OpBits.setAllBits();
41214             OpElts.setBit(I);
41215           } else if ((Invert && !EltBits[I].isAllOnes()) ||
41216                      (!Invert && !EltBits[I].isZero())) {
41217             OpBits |= Invert ? ~EltBits[I] : EltBits[I];
41218             OpElts.setBit(I);
41219           }
41220         }
41221       }
41222       return std::make_pair(OpBits, OpElts);
41223     };
41224     APInt BitsLHS, EltsLHS;
41225     APInt BitsRHS, EltsRHS;
41226     std::tie(BitsLHS, EltsLHS) = GetDemandedMasks(RHS);
41227     std::tie(BitsRHS, EltsRHS) = GetDemandedMasks(LHS, true);
41228 
41229     APInt LHSUndef, LHSZero;
41230     APInt RHSUndef, RHSZero;
41231     if (SimplifyDemandedVectorElts(LHS, EltsLHS, LHSUndef, LHSZero, TLO,
41232                                    Depth + 1))
41233       return true;
41234     if (SimplifyDemandedVectorElts(RHS, EltsRHS, RHSUndef, RHSZero, TLO,
41235                                    Depth + 1))
41236       return true;
41237 
41238     if (!DemandedElts.isAllOnes()) {
41239       SDValue NewLHS = SimplifyMultipleUseDemandedBits(LHS, BitsLHS, EltsLHS,
41240                                                        TLO.DAG, Depth + 1);
41241       SDValue NewRHS = SimplifyMultipleUseDemandedBits(RHS, BitsRHS, EltsRHS,
41242                                                        TLO.DAG, Depth + 1);
41243       if (NewLHS || NewRHS) {
41244         NewLHS = NewLHS ? NewLHS : LHS;
41245         NewRHS = NewRHS ? NewRHS : RHS;
41246         return TLO.CombineTo(
41247             Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS));
41248       }
41249     }
41250     break;
41251   }
41252   case X86ISD::CVTSI2P:
41253   case X86ISD::CVTUI2P: {
41254     SDValue Src = Op.getOperand(0);
41255     MVT SrcVT = Src.getSimpleValueType();
41256     APInt SrcUndef, SrcZero;
41257     APInt SrcElts = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
41258     if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
41259                                    Depth + 1))
41260       return true;
41261     break;
41262   }
41263   case X86ISD::PACKSS:
41264   case X86ISD::PACKUS: {
41265     SDValue N0 = Op.getOperand(0);
41266     SDValue N1 = Op.getOperand(1);
41267 
41268     APInt DemandedLHS, DemandedRHS;
41269     getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
41270 
41271     APInt LHSUndef, LHSZero;
41272     if (SimplifyDemandedVectorElts(N0, DemandedLHS, LHSUndef, LHSZero, TLO,
41273                                    Depth + 1))
41274       return true;
41275     APInt RHSUndef, RHSZero;
41276     if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO,
41277                                    Depth + 1))
41278       return true;
41279 
41280     // TODO - pass on known zero/undef.
41281 
41282     // Aggressively peek through ops to get at the demanded elts.
41283     // TODO - we should do this for all target/faux shuffles ops.
41284     if (!DemandedElts.isAllOnes()) {
41285       SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
41286                                                             TLO.DAG, Depth + 1);
41287       SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
41288                                                             TLO.DAG, Depth + 1);
41289       if (NewN0 || NewN1) {
41290         NewN0 = NewN0 ? NewN0 : N0;
41291         NewN1 = NewN1 ? NewN1 : N1;
41292         return TLO.CombineTo(Op,
41293                              TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
41294       }
41295     }
41296     break;
41297   }
41298   case X86ISD::HADD:
41299   case X86ISD::HSUB:
41300   case X86ISD::FHADD:
41301   case X86ISD::FHSUB: {
41302     SDValue N0 = Op.getOperand(0);
41303     SDValue N1 = Op.getOperand(1);
41304 
41305     APInt DemandedLHS, DemandedRHS;
41306     getHorizDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
41307 
41308     APInt LHSUndef, LHSZero;
41309     if (SimplifyDemandedVectorElts(N0, DemandedLHS, LHSUndef, LHSZero, TLO,
41310                                    Depth + 1))
41311       return true;
41312     APInt RHSUndef, RHSZero;
41313     if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO,
41314                                    Depth + 1))
41315       return true;
41316 
41317     // TODO - pass on known zero/undef.
41318 
41319     // Aggressively peek through ops to get at the demanded elts.
41320     // TODO: Handle repeated operands.
41321     if (N0 != N1 && !DemandedElts.isAllOnes()) {
41322       SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
41323                                                             TLO.DAG, Depth + 1);
41324       SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
41325                                                             TLO.DAG, Depth + 1);
41326       if (NewN0 || NewN1) {
41327         NewN0 = NewN0 ? NewN0 : N0;
41328         NewN1 = NewN1 ? NewN1 : N1;
41329         return TLO.CombineTo(Op,
41330                              TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
41331       }
41332     }
41333     break;
41334   }
41335   case X86ISD::VTRUNC:
41336   case X86ISD::VTRUNCS:
41337   case X86ISD::VTRUNCUS: {
41338     SDValue Src = Op.getOperand(0);
41339     MVT SrcVT = Src.getSimpleValueType();
41340     APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
41341     APInt SrcUndef, SrcZero;
41342     if (SimplifyDemandedVectorElts(Src, DemandedSrc, SrcUndef, SrcZero, TLO,
41343                                    Depth + 1))
41344       return true;
41345     KnownZero = SrcZero.zextOrTrunc(NumElts);
41346     KnownUndef = SrcUndef.zextOrTrunc(NumElts);
41347     break;
41348   }
41349   case X86ISD::BLENDV: {
41350     APInt SelUndef, SelZero;
41351     if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, SelUndef,
41352                                    SelZero, TLO, Depth + 1))
41353       return true;
41354 
41355     // TODO: Use SelZero to adjust LHS/RHS DemandedElts.
41356     APInt LHSUndef, LHSZero;
41357     if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, LHSUndef,
41358                                    LHSZero, TLO, Depth + 1))
41359       return true;
41360 
41361     APInt RHSUndef, RHSZero;
41362     if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedElts, RHSUndef,
41363                                    RHSZero, TLO, Depth + 1))
41364       return true;
41365 
41366     KnownZero = LHSZero & RHSZero;
41367     KnownUndef = LHSUndef & RHSUndef;
41368     break;
41369   }
41370   case X86ISD::VZEXT_MOVL: {
41371     // If upper demanded elements are already zero then we have nothing to do.
41372     SDValue Src = Op.getOperand(0);
41373     APInt DemandedUpperElts = DemandedElts;
41374     DemandedUpperElts.clearLowBits(1);
41375     if (TLO.DAG.MaskedVectorIsZero(Src, DemandedUpperElts, Depth + 1))
41376       return TLO.CombineTo(Op, Src);
41377     break;
41378   }
41379   case X86ISD::VZEXT_LOAD: {
41380     // If upper demanded elements are not demanded then simplify to a
41381     // scalar_to_vector(load()).
41382     MVT SVT = VT.getSimpleVT().getVectorElementType();
41383     if (DemandedElts == 1 && Op.getValue(1).use_empty() && isTypeLegal(SVT)) {
41384       SDLoc DL(Op);
41385       auto *Mem = cast<MemSDNode>(Op);
41386       SDValue Elt = TLO.DAG.getLoad(SVT, DL, Mem->getChain(), Mem->getBasePtr(),
41387                                     Mem->getMemOperand());
41388       SDValue Vec = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Elt);
41389       return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Vec));
41390     }
41391     break;
41392   }
41393   case X86ISD::VBROADCAST: {
41394     SDValue Src = Op.getOperand(0);
41395     MVT SrcVT = Src.getSimpleValueType();
41396     if (!SrcVT.isVector())
41397       break;
41398     // Don't bother broadcasting if we just need the 0'th element.
41399     if (DemandedElts == 1) {
41400       if (Src.getValueType() != VT)
41401         Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG,
41402                              SDLoc(Op));
41403       return TLO.CombineTo(Op, Src);
41404     }
41405     APInt SrcUndef, SrcZero;
41406     APInt SrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(), 0);
41407     if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
41408                                    Depth + 1))
41409       return true;
41410     // Aggressively peek through src to get at the demanded elt.
41411     // TODO - we should do this for all target/faux shuffles ops.
41412     if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
41413             Src, SrcElts, TLO.DAG, Depth + 1))
41414       return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
41415     break;
41416   }
41417   case X86ISD::VPERMV:
41418     if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 0, TLO,
41419                                                    Depth))
41420       return true;
41421     break;
41422   case X86ISD::PSHUFB:
41423   case X86ISD::VPERMV3:
41424   case X86ISD::VPERMILPV:
41425     if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 1, TLO,
41426                                                    Depth))
41427       return true;
41428     break;
41429   case X86ISD::VPPERM:
41430   case X86ISD::VPERMIL2:
41431     if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 2, TLO,
41432                                                    Depth))
41433       return true;
41434     break;
41435   }
41436 
41437   // For 256/512-bit ops that are 128/256-bit ops glued together, if we do not
41438   // demand any of the high elements, then narrow the op to 128/256-bits: e.g.
41439   // (op ymm0, ymm1) --> insert undef, (op xmm0, xmm1), 0
41440   if ((VT.is256BitVector() || VT.is512BitVector()) &&
41441       DemandedElts.lshr(NumElts / 2) == 0) {
41442     unsigned SizeInBits = VT.getSizeInBits();
41443     unsigned ExtSizeInBits = SizeInBits / 2;
41444 
41445     // See if 512-bit ops only use the bottom 128-bits.
41446     if (VT.is512BitVector() && DemandedElts.lshr(NumElts / 4) == 0)
41447       ExtSizeInBits = SizeInBits / 4;
41448 
41449     switch (Opc) {
41450       // Scalar broadcast.
41451     case X86ISD::VBROADCAST: {
41452       SDLoc DL(Op);
41453       SDValue Src = Op.getOperand(0);
41454       if (Src.getValueSizeInBits() > ExtSizeInBits)
41455         Src = extractSubVector(Src, 0, TLO.DAG, DL, ExtSizeInBits);
41456       EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
41457                                     ExtSizeInBits / VT.getScalarSizeInBits());
41458       SDValue Bcst = TLO.DAG.getNode(X86ISD::VBROADCAST, DL, BcstVT, Src);
41459       return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0,
41460                                                TLO.DAG, DL, ExtSizeInBits));
41461     }
41462     case X86ISD::VBROADCAST_LOAD: {
41463       SDLoc DL(Op);
41464       auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
41465       EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
41466                                     ExtSizeInBits / VT.getScalarSizeInBits());
41467       SDVTList Tys = TLO.DAG.getVTList(BcstVT, MVT::Other);
41468       SDValue Ops[] = {MemIntr->getOperand(0), MemIntr->getOperand(1)};
41469       SDValue Bcst = TLO.DAG.getMemIntrinsicNode(
41470           X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MemIntr->getMemoryVT(),
41471           MemIntr->getMemOperand());
41472       TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
41473                                            Bcst.getValue(1));
41474       return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0,
41475                                                TLO.DAG, DL, ExtSizeInBits));
41476     }
41477       // Subvector broadcast.
41478     case X86ISD::SUBV_BROADCAST_LOAD: {
41479       auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
41480       EVT MemVT = MemIntr->getMemoryVT();
41481       if (ExtSizeInBits == MemVT.getStoreSizeInBits()) {
41482         SDLoc DL(Op);
41483         SDValue Ld =
41484             TLO.DAG.getLoad(MemVT, DL, MemIntr->getChain(),
41485                             MemIntr->getBasePtr(), MemIntr->getMemOperand());
41486         TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
41487                                              Ld.getValue(1));
41488         return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Ld, 0,
41489                                                  TLO.DAG, DL, ExtSizeInBits));
41490       } else if ((ExtSizeInBits % MemVT.getStoreSizeInBits()) == 0) {
41491         SDLoc DL(Op);
41492         EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
41493                                       ExtSizeInBits / VT.getScalarSizeInBits());
41494         if (SDValue BcstLd =
41495                 getBROADCAST_LOAD(Opc, DL, BcstVT, MemVT, MemIntr, 0, TLO.DAG))
41496           return TLO.CombineTo(Op,
41497                                insertSubVector(TLO.DAG.getUNDEF(VT), BcstLd, 0,
41498                                                TLO.DAG, DL, ExtSizeInBits));
41499       }
41500       break;
41501     }
41502       // Byte shifts by immediate.
41503     case X86ISD::VSHLDQ:
41504     case X86ISD::VSRLDQ:
41505       // Shift by uniform.
41506     case X86ISD::VSHL:
41507     case X86ISD::VSRL:
41508     case X86ISD::VSRA:
41509       // Shift by immediate.
41510     case X86ISD::VSHLI:
41511     case X86ISD::VSRLI:
41512     case X86ISD::VSRAI: {
41513       SDLoc DL(Op);
41514       SDValue Ext0 =
41515           extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
41516       SDValue ExtOp =
41517           TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Op.getOperand(1));
41518       SDValue UndefVec = TLO.DAG.getUNDEF(VT);
41519       SDValue Insert =
41520           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
41521       return TLO.CombineTo(Op, Insert);
41522     }
41523     case X86ISD::VPERMI: {
41524       // Simplify PERMPD/PERMQ to extract_subvector.
41525       // TODO: This should be done in shuffle combining.
41526       if (VT == MVT::v4f64 || VT == MVT::v4i64) {
41527         SmallVector<int, 4> Mask;
41528         DecodeVPERMMask(NumElts, Op.getConstantOperandVal(1), Mask);
41529         if (isUndefOrEqual(Mask[0], 2) && isUndefOrEqual(Mask[1], 3)) {
41530           SDLoc DL(Op);
41531           SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
41532           SDValue UndefVec = TLO.DAG.getUNDEF(VT);
41533           SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
41534           return TLO.CombineTo(Op, Insert);
41535         }
41536       }
41537       break;
41538     }
41539     case X86ISD::VPERM2X128: {
41540       // Simplify VPERM2F128/VPERM2I128 to extract_subvector.
41541       SDLoc DL(Op);
41542       unsigned LoMask = Op.getConstantOperandVal(2) & 0xF;
41543       if (LoMask & 0x8)
41544         return TLO.CombineTo(
41545             Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, DL));
41546       unsigned EltIdx = (LoMask & 0x1) * (NumElts / 2);
41547       unsigned SrcIdx = (LoMask & 0x2) >> 1;
41548       SDValue ExtOp =
41549           extractSubVector(Op.getOperand(SrcIdx), EltIdx, TLO.DAG, DL, 128);
41550       SDValue UndefVec = TLO.DAG.getUNDEF(VT);
41551       SDValue Insert =
41552           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
41553       return TLO.CombineTo(Op, Insert);
41554     }
41555       // Zero upper elements.
41556     case X86ISD::VZEXT_MOVL:
41557       // Target unary shuffles by immediate:
41558     case X86ISD::PSHUFD:
41559     case X86ISD::PSHUFLW:
41560     case X86ISD::PSHUFHW:
41561     case X86ISD::VPERMILPI:
41562       // (Non-Lane Crossing) Target Shuffles.
41563     case X86ISD::VPERMILPV:
41564     case X86ISD::VPERMIL2:
41565     case X86ISD::PSHUFB:
41566     case X86ISD::UNPCKL:
41567     case X86ISD::UNPCKH:
41568     case X86ISD::BLENDI:
41569       // Integer ops.
41570     case X86ISD::PACKSS:
41571     case X86ISD::PACKUS:
41572     case X86ISD::PCMPEQ:
41573     case X86ISD::PCMPGT:
41574     case X86ISD::PMULUDQ:
41575     case X86ISD::PMULDQ:
41576     case X86ISD::VSHLV:
41577     case X86ISD::VSRLV:
41578     case X86ISD::VSRAV:
41579       // Float ops.
41580     case X86ISD::FMAX:
41581     case X86ISD::FMIN:
41582     case X86ISD::FMAXC:
41583     case X86ISD::FMINC:
41584       // Horizontal Ops.
41585     case X86ISD::HADD:
41586     case X86ISD::HSUB:
41587     case X86ISD::FHADD:
41588     case X86ISD::FHSUB: {
41589       SDLoc DL(Op);
41590       SmallVector<SDValue, 4> Ops;
41591       for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
41592         SDValue SrcOp = Op.getOperand(i);
41593         EVT SrcVT = SrcOp.getValueType();
41594         assert((!SrcVT.isVector() || SrcVT.getSizeInBits() == SizeInBits) &&
41595                "Unsupported vector size");
41596         Ops.push_back(SrcVT.isVector() ? extractSubVector(SrcOp, 0, TLO.DAG, DL,
41597                                                           ExtSizeInBits)
41598                                        : SrcOp);
41599       }
41600       MVT ExtVT = VT.getSimpleVT();
41601       ExtVT = MVT::getVectorVT(ExtVT.getScalarType(),
41602                                ExtSizeInBits / ExtVT.getScalarSizeInBits());
41603       SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops);
41604       SDValue UndefVec = TLO.DAG.getUNDEF(VT);
41605       SDValue Insert =
41606           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
41607       return TLO.CombineTo(Op, Insert);
41608     }
41609     }
41610   }
41611 
41612   // For splats, unless we *only* demand the 0'th element,
41613   // stop attempts at simplification here, we aren't going to improve things,
41614   // this is better than any potential shuffle.
41615   if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/false))
41616     return false;
41617 
41618   // Get target/faux shuffle mask.
41619   APInt OpUndef, OpZero;
41620   SmallVector<int, 64> OpMask;
41621   SmallVector<SDValue, 2> OpInputs;
41622   if (!getTargetShuffleInputs(Op, DemandedElts, OpInputs, OpMask, OpUndef,
41623                               OpZero, TLO.DAG, Depth, false))
41624     return false;
41625 
41626   // Shuffle inputs must be the same size as the result.
41627   if (OpMask.size() != (unsigned)NumElts ||
41628       llvm::any_of(OpInputs, [VT](SDValue V) {
41629         return VT.getSizeInBits() != V.getValueSizeInBits() ||
41630                !V.getValueType().isVector();
41631       }))
41632     return false;
41633 
41634   KnownZero = OpZero;
41635   KnownUndef = OpUndef;
41636 
41637   // Check if shuffle mask can be simplified to undef/zero/identity.
41638   int NumSrcs = OpInputs.size();
41639   for (int i = 0; i != NumElts; ++i)
41640     if (!DemandedElts[i])
41641       OpMask[i] = SM_SentinelUndef;
41642 
41643   if (isUndefInRange(OpMask, 0, NumElts)) {
41644     KnownUndef.setAllBits();
41645     return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
41646   }
41647   if (isUndefOrZeroInRange(OpMask, 0, NumElts)) {
41648     KnownZero.setAllBits();
41649     return TLO.CombineTo(
41650         Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
41651   }
41652   for (int Src = 0; Src != NumSrcs; ++Src)
41653     if (isSequentialOrUndefInRange(OpMask, 0, NumElts, Src * NumElts))
41654       return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src]));
41655 
41656   // Attempt to simplify inputs.
41657   for (int Src = 0; Src != NumSrcs; ++Src) {
41658     // TODO: Support inputs of different types.
41659     if (OpInputs[Src].getValueType() != VT)
41660       continue;
41661 
41662     int Lo = Src * NumElts;
41663     APInt SrcElts = APInt::getZero(NumElts);
41664     for (int i = 0; i != NumElts; ++i)
41665       if (DemandedElts[i]) {
41666         int M = OpMask[i] - Lo;
41667         if (0 <= M && M < NumElts)
41668           SrcElts.setBit(M);
41669       }
41670 
41671     // TODO - Propagate input undef/zero elts.
41672     APInt SrcUndef, SrcZero;
41673     if (SimplifyDemandedVectorElts(OpInputs[Src], SrcElts, SrcUndef, SrcZero,
41674                                    TLO, Depth + 1))
41675       return true;
41676   }
41677 
41678   // If we don't demand all elements, then attempt to combine to a simpler
41679   // shuffle.
41680   // We need to convert the depth to something combineX86ShufflesRecursively
41681   // can handle - so pretend its Depth == 0 again, and reduce the max depth
41682   // to match. This prevents combineX86ShuffleChain from returning a
41683   // combined shuffle that's the same as the original root, causing an
41684   // infinite loop.
41685   if (!DemandedElts.isAllOnes()) {
41686     assert(Depth < X86::MaxShuffleCombineDepth && "Depth out of range");
41687 
41688     SmallVector<int, 64> DemandedMask(NumElts, SM_SentinelUndef);
41689     for (int i = 0; i != NumElts; ++i)
41690       if (DemandedElts[i])
41691         DemandedMask[i] = i;
41692 
41693     SDValue NewShuffle = combineX86ShufflesRecursively(
41694         {Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth,
41695         /*HasVarMask*/ false,
41696         /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, TLO.DAG,
41697         Subtarget);
41698     if (NewShuffle)
41699       return TLO.CombineTo(Op, NewShuffle);
41700   }
41701 
41702   return false;
41703 }
41704 
41705 bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
41706     SDValue Op, const APInt &OriginalDemandedBits,
41707     const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
41708     unsigned Depth) const {
41709   EVT VT = Op.getValueType();
41710   unsigned BitWidth = OriginalDemandedBits.getBitWidth();
41711   unsigned Opc = Op.getOpcode();
41712   switch(Opc) {
41713   case X86ISD::VTRUNC: {
41714     KnownBits KnownOp;
41715     SDValue Src = Op.getOperand(0);
41716     MVT SrcVT = Src.getSimpleValueType();
41717 
41718     // Simplify the input, using demanded bit information.
41719     APInt TruncMask = OriginalDemandedBits.zext(SrcVT.getScalarSizeInBits());
41720     APInt DemandedElts = OriginalDemandedElts.trunc(SrcVT.getVectorNumElements());
41721     if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, KnownOp, TLO, Depth + 1))
41722       return true;
41723     break;
41724   }
41725   case X86ISD::PMULDQ:
41726   case X86ISD::PMULUDQ: {
41727     // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
41728     KnownBits KnownLHS, KnownRHS;
41729     SDValue LHS = Op.getOperand(0);
41730     SDValue RHS = Op.getOperand(1);
41731 
41732     // Don't mask bits on 32-bit AVX512 targets which might lose a broadcast.
41733     // FIXME: Can we bound this better?
41734     APInt DemandedMask = APInt::getLowBitsSet(64, 32);
41735     APInt DemandedMaskLHS = APInt::getAllOnes(64);
41736     APInt DemandedMaskRHS = APInt::getAllOnes(64);
41737 
41738     bool Is32BitAVX512 = !Subtarget.is64Bit() && Subtarget.hasAVX512();
41739     if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(LHS))
41740       DemandedMaskLHS = DemandedMask;
41741     if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(RHS))
41742       DemandedMaskRHS = DemandedMask;
41743 
41744     if (SimplifyDemandedBits(LHS, DemandedMaskLHS, OriginalDemandedElts,
41745                              KnownLHS, TLO, Depth + 1))
41746       return true;
41747     if (SimplifyDemandedBits(RHS, DemandedMaskRHS, OriginalDemandedElts,
41748                              KnownRHS, TLO, Depth + 1))
41749       return true;
41750 
41751     // PMULUDQ(X,1) -> AND(X,(1<<32)-1) 'getZeroExtendInReg'.
41752     KnownRHS = KnownRHS.trunc(32);
41753     if (Opc == X86ISD::PMULUDQ && KnownRHS.isConstant() &&
41754         KnownRHS.getConstant().isOne()) {
41755       SDLoc DL(Op);
41756       SDValue Mask = TLO.DAG.getConstant(DemandedMask, DL, VT);
41757       return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, DL, VT, LHS, Mask));
41758     }
41759 
41760     // Aggressively peek through ops to get at the demanded low bits.
41761     SDValue DemandedLHS = SimplifyMultipleUseDemandedBits(
41762         LHS, DemandedMaskLHS, OriginalDemandedElts, TLO.DAG, Depth + 1);
41763     SDValue DemandedRHS = SimplifyMultipleUseDemandedBits(
41764         RHS, DemandedMaskRHS, OriginalDemandedElts, TLO.DAG, Depth + 1);
41765     if (DemandedLHS || DemandedRHS) {
41766       DemandedLHS = DemandedLHS ? DemandedLHS : LHS;
41767       DemandedRHS = DemandedRHS ? DemandedRHS : RHS;
41768       return TLO.CombineTo(
41769           Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS));
41770     }
41771     break;
41772   }
41773   case X86ISD::ANDNP: {
41774     KnownBits Known2;
41775     SDValue Op0 = Op.getOperand(0);
41776     SDValue Op1 = Op.getOperand(1);
41777 
41778     if (SimplifyDemandedBits(Op1, OriginalDemandedBits, OriginalDemandedElts,
41779                              Known, TLO, Depth + 1))
41780       return true;
41781     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
41782 
41783     if (SimplifyDemandedBits(Op0, ~Known.Zero & OriginalDemandedBits,
41784                              OriginalDemandedElts, Known2, TLO, Depth + 1))
41785       return true;
41786     assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
41787 
41788     // If the RHS is a constant, see if we can simplify it.
41789     if (ShrinkDemandedConstant(Op, ~Known2.One & OriginalDemandedBits,
41790                                OriginalDemandedElts, TLO))
41791       return true;
41792 
41793     // ANDNP = (~Op0 & Op1);
41794     Known.One &= Known2.Zero;
41795     Known.Zero |= Known2.One;
41796     break;
41797   }
41798   case X86ISD::VSHLI: {
41799     SDValue Op0 = Op.getOperand(0);
41800 
41801     unsigned ShAmt = Op.getConstantOperandVal(1);
41802     if (ShAmt >= BitWidth)
41803       break;
41804 
41805     APInt DemandedMask = OriginalDemandedBits.lshr(ShAmt);
41806 
41807     // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
41808     // single shift.  We can do this if the bottom bits (which are shifted
41809     // out) are never demanded.
41810     if (Op0.getOpcode() == X86ISD::VSRLI &&
41811         OriginalDemandedBits.countr_zero() >= ShAmt) {
41812       unsigned Shift2Amt = Op0.getConstantOperandVal(1);
41813       if (Shift2Amt < BitWidth) {
41814         int Diff = ShAmt - Shift2Amt;
41815         if (Diff == 0)
41816           return TLO.CombineTo(Op, Op0.getOperand(0));
41817 
41818         unsigned NewOpc = Diff < 0 ? X86ISD::VSRLI : X86ISD::VSHLI;
41819         SDValue NewShift = TLO.DAG.getNode(
41820             NewOpc, SDLoc(Op), VT, Op0.getOperand(0),
41821             TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
41822         return TLO.CombineTo(Op, NewShift);
41823       }
41824     }
41825 
41826     // If we are only demanding sign bits then we can use the shift source directly.
41827     unsigned NumSignBits =
41828         TLO.DAG.ComputeNumSignBits(Op0, OriginalDemandedElts, Depth + 1);
41829     unsigned UpperDemandedBits = BitWidth - OriginalDemandedBits.countr_zero();
41830     if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits)
41831       return TLO.CombineTo(Op, Op0);
41832 
41833     if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
41834                              TLO, Depth + 1))
41835       return true;
41836 
41837     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
41838     Known.Zero <<= ShAmt;
41839     Known.One <<= ShAmt;
41840 
41841     // Low bits known zero.
41842     Known.Zero.setLowBits(ShAmt);
41843     return false;
41844   }
41845   case X86ISD::VSRLI: {
41846     unsigned ShAmt = Op.getConstantOperandVal(1);
41847     if (ShAmt >= BitWidth)
41848       break;
41849 
41850     APInt DemandedMask = OriginalDemandedBits << ShAmt;
41851 
41852     if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask,
41853                              OriginalDemandedElts, Known, TLO, Depth + 1))
41854       return true;
41855 
41856     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
41857     Known.Zero.lshrInPlace(ShAmt);
41858     Known.One.lshrInPlace(ShAmt);
41859 
41860     // High bits known zero.
41861     Known.Zero.setHighBits(ShAmt);
41862     return false;
41863   }
41864   case X86ISD::VSRAI: {
41865     SDValue Op0 = Op.getOperand(0);
41866     SDValue Op1 = Op.getOperand(1);
41867 
41868     unsigned ShAmt = Op1->getAsZExtVal();
41869     if (ShAmt >= BitWidth)
41870       break;
41871 
41872     APInt DemandedMask = OriginalDemandedBits << ShAmt;
41873 
41874     // If we just want the sign bit then we don't need to shift it.
41875     if (OriginalDemandedBits.isSignMask())
41876       return TLO.CombineTo(Op, Op0);
41877 
41878     // fold (VSRAI (VSHLI X, C1), C1) --> X iff NumSignBits(X) > C1
41879     if (Op0.getOpcode() == X86ISD::VSHLI &&
41880         Op.getOperand(1) == Op0.getOperand(1)) {
41881       SDValue Op00 = Op0.getOperand(0);
41882       unsigned NumSignBits =
41883           TLO.DAG.ComputeNumSignBits(Op00, OriginalDemandedElts);
41884       if (ShAmt < NumSignBits)
41885         return TLO.CombineTo(Op, Op00);
41886     }
41887 
41888     // If any of the demanded bits are produced by the sign extension, we also
41889     // demand the input sign bit.
41890     if (OriginalDemandedBits.countl_zero() < ShAmt)
41891       DemandedMask.setSignBit();
41892 
41893     if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
41894                              TLO, Depth + 1))
41895       return true;
41896 
41897     assert(!Known.hasConflict() && "Bits known to be one AND zero?");
41898     Known.Zero.lshrInPlace(ShAmt);
41899     Known.One.lshrInPlace(ShAmt);
41900 
41901     // If the input sign bit is known to be zero, or if none of the top bits
41902     // are demanded, turn this into an unsigned shift right.
41903     if (Known.Zero[BitWidth - ShAmt - 1] ||
41904         OriginalDemandedBits.countl_zero() >= ShAmt)
41905       return TLO.CombineTo(
41906           Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1));
41907 
41908     // High bits are known one.
41909     if (Known.One[BitWidth - ShAmt - 1])
41910       Known.One.setHighBits(ShAmt);
41911     return false;
41912   }
41913   case X86ISD::BLENDV: {
41914     SDValue Sel = Op.getOperand(0);
41915     SDValue LHS = Op.getOperand(1);
41916     SDValue RHS = Op.getOperand(2);
41917 
41918     APInt SignMask = APInt::getSignMask(BitWidth);
41919     SDValue NewSel = SimplifyMultipleUseDemandedBits(
41920         Sel, SignMask, OriginalDemandedElts, TLO.DAG, Depth + 1);
41921     SDValue NewLHS = SimplifyMultipleUseDemandedBits(
41922         LHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1);
41923     SDValue NewRHS = SimplifyMultipleUseDemandedBits(
41924         RHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1);
41925 
41926     if (NewSel || NewLHS || NewRHS) {
41927       NewSel = NewSel ? NewSel : Sel;
41928       NewLHS = NewLHS ? NewLHS : LHS;
41929       NewRHS = NewRHS ? NewRHS : RHS;
41930       return TLO.CombineTo(Op, TLO.DAG.getNode(X86ISD::BLENDV, SDLoc(Op), VT,
41931                                                NewSel, NewLHS, NewRHS));
41932     }
41933     break;
41934   }
41935   case X86ISD::PEXTRB:
41936   case X86ISD::PEXTRW: {
41937     SDValue Vec = Op.getOperand(0);
41938     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
41939     MVT VecVT = Vec.getSimpleValueType();
41940     unsigned NumVecElts = VecVT.getVectorNumElements();
41941 
41942     if (CIdx && CIdx->getAPIntValue().ult(NumVecElts)) {
41943       unsigned Idx = CIdx->getZExtValue();
41944       unsigned VecBitWidth = VecVT.getScalarSizeInBits();
41945 
41946       // If we demand no bits from the vector then we must have demanded
41947       // bits from the implict zext - simplify to zero.
41948       APInt DemandedVecBits = OriginalDemandedBits.trunc(VecBitWidth);
41949       if (DemandedVecBits == 0)
41950         return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
41951 
41952       APInt KnownUndef, KnownZero;
41953       APInt DemandedVecElts = APInt::getOneBitSet(NumVecElts, Idx);
41954       if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
41955                                      KnownZero, TLO, Depth + 1))
41956         return true;
41957 
41958       KnownBits KnownVec;
41959       if (SimplifyDemandedBits(Vec, DemandedVecBits, DemandedVecElts,
41960                                KnownVec, TLO, Depth + 1))
41961         return true;
41962 
41963       if (SDValue V = SimplifyMultipleUseDemandedBits(
41964               Vec, DemandedVecBits, DemandedVecElts, TLO.DAG, Depth + 1))
41965         return TLO.CombineTo(
41966             Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1)));
41967 
41968       Known = KnownVec.zext(BitWidth);
41969       return false;
41970     }
41971     break;
41972   }
41973   case X86ISD::PINSRB:
41974   case X86ISD::PINSRW: {
41975     SDValue Vec = Op.getOperand(0);
41976     SDValue Scl = Op.getOperand(1);
41977     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
41978     MVT VecVT = Vec.getSimpleValueType();
41979 
41980     if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
41981       unsigned Idx = CIdx->getZExtValue();
41982       if (!OriginalDemandedElts[Idx])
41983         return TLO.CombineTo(Op, Vec);
41984 
41985       KnownBits KnownVec;
41986       APInt DemandedVecElts(OriginalDemandedElts);
41987       DemandedVecElts.clearBit(Idx);
41988       if (SimplifyDemandedBits(Vec, OriginalDemandedBits, DemandedVecElts,
41989                                KnownVec, TLO, Depth + 1))
41990         return true;
41991 
41992       KnownBits KnownScl;
41993       unsigned NumSclBits = Scl.getScalarValueSizeInBits();
41994       APInt DemandedSclBits = OriginalDemandedBits.zext(NumSclBits);
41995       if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
41996         return true;
41997 
41998       KnownScl = KnownScl.trunc(VecVT.getScalarSizeInBits());
41999       Known = KnownVec.intersectWith(KnownScl);
42000       return false;
42001     }
42002     break;
42003   }
42004   case X86ISD::PACKSS:
42005     // PACKSS saturates to MIN/MAX integer values. So if we just want the
42006     // sign bit then we can just ask for the source operands sign bit.
42007     // TODO - add known bits handling.
42008     if (OriginalDemandedBits.isSignMask()) {
42009       APInt DemandedLHS, DemandedRHS;
42010       getPackDemandedElts(VT, OriginalDemandedElts, DemandedLHS, DemandedRHS);
42011 
42012       KnownBits KnownLHS, KnownRHS;
42013       APInt SignMask = APInt::getSignMask(BitWidth * 2);
42014       if (SimplifyDemandedBits(Op.getOperand(0), SignMask, DemandedLHS,
42015                                KnownLHS, TLO, Depth + 1))
42016         return true;
42017       if (SimplifyDemandedBits(Op.getOperand(1), SignMask, DemandedRHS,
42018                                KnownRHS, TLO, Depth + 1))
42019         return true;
42020 
42021       // Attempt to avoid multi-use ops if we don't need anything from them.
42022       SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
42023           Op.getOperand(0), SignMask, DemandedLHS, TLO.DAG, Depth + 1);
42024       SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
42025           Op.getOperand(1), SignMask, DemandedRHS, TLO.DAG, Depth + 1);
42026       if (DemandedOp0 || DemandedOp1) {
42027         SDValue Op0 = DemandedOp0 ? DemandedOp0 : Op.getOperand(0);
42028         SDValue Op1 = DemandedOp1 ? DemandedOp1 : Op.getOperand(1);
42029         return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, Op0, Op1));
42030       }
42031     }
42032     // TODO - add general PACKSS/PACKUS SimplifyDemandedBits support.
42033     break;
42034   case X86ISD::VBROADCAST: {
42035     SDValue Src = Op.getOperand(0);
42036     MVT SrcVT = Src.getSimpleValueType();
42037     APInt DemandedElts = APInt::getOneBitSet(
42038         SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1, 0);
42039     if (SimplifyDemandedBits(Src, OriginalDemandedBits, DemandedElts, Known,
42040                              TLO, Depth + 1))
42041       return true;
42042     // If we don't need the upper bits, attempt to narrow the broadcast source.
42043     // Don't attempt this on AVX512 as it might affect broadcast folding.
42044     // TODO: Should we attempt this for i32/i16 splats? They tend to be slower.
42045     if ((BitWidth == 64) && SrcVT.isScalarInteger() && !Subtarget.hasAVX512() &&
42046         OriginalDemandedBits.countl_zero() >= (BitWidth / 2) &&
42047         Src->hasOneUse()) {
42048       MVT NewSrcVT = MVT::getIntegerVT(BitWidth / 2);
42049       SDValue NewSrc =
42050           TLO.DAG.getNode(ISD::TRUNCATE, SDLoc(Src), NewSrcVT, Src);
42051       MVT NewVT = MVT::getVectorVT(NewSrcVT, VT.getVectorNumElements() * 2);
42052       SDValue NewBcst =
42053           TLO.DAG.getNode(X86ISD::VBROADCAST, SDLoc(Op), NewVT, NewSrc);
42054       return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, NewBcst));
42055     }
42056     break;
42057   }
42058   case X86ISD::PCMPGT:
42059     // icmp sgt(0, R) == ashr(R, BitWidth-1).
42060     // iff we only need the sign bit then we can use R directly.
42061     if (OriginalDemandedBits.isSignMask() &&
42062         ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
42063       return TLO.CombineTo(Op, Op.getOperand(1));
42064     break;
42065   case X86ISD::MOVMSK: {
42066     SDValue Src = Op.getOperand(0);
42067     MVT SrcVT = Src.getSimpleValueType();
42068     unsigned SrcBits = SrcVT.getScalarSizeInBits();
42069     unsigned NumElts = SrcVT.getVectorNumElements();
42070 
42071     // If we don't need the sign bits at all just return zero.
42072     if (OriginalDemandedBits.countr_zero() >= NumElts)
42073       return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
42074 
42075     // See if we only demand bits from the lower 128-bit vector.
42076     if (SrcVT.is256BitVector() &&
42077         OriginalDemandedBits.getActiveBits() <= (NumElts / 2)) {
42078       SDValue NewSrc = extract128BitVector(Src, 0, TLO.DAG, SDLoc(Src));
42079       return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
42080     }
42081 
42082     // Only demand the vector elements of the sign bits we need.
42083     APInt KnownUndef, KnownZero;
42084     APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(NumElts);
42085     if (SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, KnownZero,
42086                                    TLO, Depth + 1))
42087       return true;
42088 
42089     Known.Zero = KnownZero.zext(BitWidth);
42090     Known.Zero.setHighBits(BitWidth - NumElts);
42091 
42092     // MOVMSK only uses the MSB from each vector element.
42093     KnownBits KnownSrc;
42094     APInt DemandedSrcBits = APInt::getSignMask(SrcBits);
42095     if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, KnownSrc, TLO,
42096                              Depth + 1))
42097       return true;
42098 
42099     if (KnownSrc.One[SrcBits - 1])
42100       Known.One.setLowBits(NumElts);
42101     else if (KnownSrc.Zero[SrcBits - 1])
42102       Known.Zero.setLowBits(NumElts);
42103 
42104     // Attempt to avoid multi-use os if we don't need anything from it.
42105     if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
42106             Src, DemandedSrcBits, DemandedElts, TLO.DAG, Depth + 1))
42107       return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
42108     return false;
42109   }
42110   case X86ISD::TESTP: {
42111     SDValue Op0 = Op.getOperand(0);
42112     SDValue Op1 = Op.getOperand(1);
42113     MVT OpVT = Op0.getSimpleValueType();
42114     assert((OpVT.getVectorElementType() == MVT::f32 ||
42115             OpVT.getVectorElementType() == MVT::f64) &&
42116            "Illegal vector type for X86ISD::TESTP");
42117 
42118     // TESTPS/TESTPD only demands the sign bits of ALL the elements.
42119     KnownBits KnownSrc;
42120     APInt SignMask = APInt::getSignMask(OpVT.getScalarSizeInBits());
42121     bool AssumeSingleUse = (Op0 == Op1) && Op->isOnlyUserOf(Op0.getNode());
42122     return SimplifyDemandedBits(Op0, SignMask, KnownSrc, TLO, Depth + 1,
42123                                 AssumeSingleUse) ||
42124            SimplifyDemandedBits(Op1, SignMask, KnownSrc, TLO, Depth + 1,
42125                                 AssumeSingleUse);
42126   }
42127   case X86ISD::BEXTR:
42128   case X86ISD::BEXTRI: {
42129     SDValue Op0 = Op.getOperand(0);
42130     SDValue Op1 = Op.getOperand(1);
42131 
42132     // Only bottom 16-bits of the control bits are required.
42133     if (auto *Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
42134       // NOTE: SimplifyDemandedBits won't do this for constants.
42135       uint64_t Val1 = Cst1->getZExtValue();
42136       uint64_t MaskedVal1 = Val1 & 0xFFFF;
42137       if (Opc == X86ISD::BEXTR && MaskedVal1 != Val1) {
42138         SDLoc DL(Op);
42139         return TLO.CombineTo(
42140             Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0,
42141                                 TLO.DAG.getConstant(MaskedVal1, DL, VT)));
42142       }
42143 
42144       unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0);
42145       unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8);
42146 
42147       // If the length is 0, the result is 0.
42148       if (Length == 0) {
42149         Known.setAllZero();
42150         return false;
42151       }
42152 
42153       if ((Shift + Length) <= BitWidth) {
42154         APInt DemandedMask = APInt::getBitsSet(BitWidth, Shift, Shift + Length);
42155         if (SimplifyDemandedBits(Op0, DemandedMask, Known, TLO, Depth + 1))
42156           return true;
42157 
42158         Known = Known.extractBits(Length, Shift);
42159         Known = Known.zextOrTrunc(BitWidth);
42160         return false;
42161       }
42162     } else {
42163       assert(Opc == X86ISD::BEXTR && "Unexpected opcode!");
42164       KnownBits Known1;
42165       APInt DemandedMask(APInt::getLowBitsSet(BitWidth, 16));
42166       if (SimplifyDemandedBits(Op1, DemandedMask, Known1, TLO, Depth + 1))
42167         return true;
42168 
42169       // If the length is 0, replace with 0.
42170       KnownBits LengthBits = Known1.extractBits(8, 8);
42171       if (LengthBits.isZero())
42172         return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
42173     }
42174 
42175     break;
42176   }
42177   case X86ISD::PDEP: {
42178     SDValue Op0 = Op.getOperand(0);
42179     SDValue Op1 = Op.getOperand(1);
42180 
42181     unsigned DemandedBitsLZ = OriginalDemandedBits.countl_zero();
42182     APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
42183 
42184     // If the demanded bits has leading zeroes, we don't demand those from the
42185     // mask.
42186     if (SimplifyDemandedBits(Op1, LoMask, Known, TLO, Depth + 1))
42187       return true;
42188 
42189     // The number of possible 1s in the mask determines the number of LSBs of
42190     // operand 0 used. Undemanded bits from the mask don't matter so filter
42191     // them before counting.
42192     KnownBits Known2;
42193     uint64_t Count = (~Known.Zero & LoMask).popcount();
42194     APInt DemandedMask(APInt::getLowBitsSet(BitWidth, Count));
42195     if (SimplifyDemandedBits(Op0, DemandedMask, Known2, TLO, Depth + 1))
42196       return true;
42197 
42198     // Zeroes are retained from the mask, but not ones.
42199     Known.One.clearAllBits();
42200     // The result will have at least as many trailing zeros as the non-mask
42201     // operand since bits can only map to the same or higher bit position.
42202     Known.Zero.setLowBits(Known2.countMinTrailingZeros());
42203     return false;
42204   }
42205   }
42206 
42207   return TargetLowering::SimplifyDemandedBitsForTargetNode(
42208       Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
42209 }
42210 
42211 SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
42212     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
42213     SelectionDAG &DAG, unsigned Depth) const {
42214   int NumElts = DemandedElts.getBitWidth();
42215   unsigned Opc = Op.getOpcode();
42216   EVT VT = Op.getValueType();
42217 
42218   switch (Opc) {
42219   case X86ISD::PINSRB:
42220   case X86ISD::PINSRW: {
42221     // If we don't demand the inserted element, return the base vector.
42222     SDValue Vec = Op.getOperand(0);
42223     auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
42224     MVT VecVT = Vec.getSimpleValueType();
42225     if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
42226         !DemandedElts[CIdx->getZExtValue()])
42227       return Vec;
42228     break;
42229   }
42230   case X86ISD::VSHLI: {
42231     // If we are only demanding sign bits then we can use the shift source
42232     // directly.
42233     SDValue Op0 = Op.getOperand(0);
42234     unsigned ShAmt = Op.getConstantOperandVal(1);
42235     unsigned BitWidth = DemandedBits.getBitWidth();
42236     unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
42237     unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
42238     if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits)
42239       return Op0;
42240     break;
42241   }
42242   case X86ISD::VSRAI:
42243     // iff we only need the sign bit then we can use the source directly.
42244     // TODO: generalize where we only demand extended signbits.
42245     if (DemandedBits.isSignMask())
42246       return Op.getOperand(0);
42247     break;
42248   case X86ISD::PCMPGT:
42249     // icmp sgt(0, R) == ashr(R, BitWidth-1).
42250     // iff we only need the sign bit then we can use R directly.
42251     if (DemandedBits.isSignMask() &&
42252         ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
42253       return Op.getOperand(1);
42254     break;
42255   case X86ISD::BLENDV: {
42256     // BLENDV: Cond (MSB) ? LHS : RHS
42257     SDValue Cond = Op.getOperand(0);
42258     SDValue LHS = Op.getOperand(1);
42259     SDValue RHS = Op.getOperand(2);
42260 
42261     KnownBits CondKnown = DAG.computeKnownBits(Cond, DemandedElts, Depth + 1);
42262     if (CondKnown.isNegative())
42263       return LHS;
42264     if (CondKnown.isNonNegative())
42265       return RHS;
42266     break;
42267   }
42268   case X86ISD::ANDNP: {
42269     // ANDNP = (~LHS & RHS);
42270     SDValue LHS = Op.getOperand(0);
42271     SDValue RHS = Op.getOperand(1);
42272 
42273     KnownBits LHSKnown = DAG.computeKnownBits(LHS, DemandedElts, Depth + 1);
42274     KnownBits RHSKnown = DAG.computeKnownBits(RHS, DemandedElts, Depth + 1);
42275 
42276     // If all of the demanded bits are known 0 on LHS and known 0 on RHS, then
42277     // the (inverted) LHS bits cannot contribute to the result of the 'andn' in
42278     // this context, so return RHS.
42279     if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.Zero))
42280       return RHS;
42281     break;
42282   }
42283   }
42284 
42285   APInt ShuffleUndef, ShuffleZero;
42286   SmallVector<int, 16> ShuffleMask;
42287   SmallVector<SDValue, 2> ShuffleOps;
42288   if (getTargetShuffleInputs(Op, DemandedElts, ShuffleOps, ShuffleMask,
42289                              ShuffleUndef, ShuffleZero, DAG, Depth, false)) {
42290     // If all the demanded elts are from one operand and are inline,
42291     // then we can use the operand directly.
42292     int NumOps = ShuffleOps.size();
42293     if (ShuffleMask.size() == (unsigned)NumElts &&
42294         llvm::all_of(ShuffleOps, [VT](SDValue V) {
42295           return VT.getSizeInBits() == V.getValueSizeInBits();
42296         })) {
42297 
42298       if (DemandedElts.isSubsetOf(ShuffleUndef))
42299         return DAG.getUNDEF(VT);
42300       if (DemandedElts.isSubsetOf(ShuffleUndef | ShuffleZero))
42301         return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op));
42302 
42303       // Bitmask that indicates which ops have only been accessed 'inline'.
42304       APInt IdentityOp = APInt::getAllOnes(NumOps);
42305       for (int i = 0; i != NumElts; ++i) {
42306         int M = ShuffleMask[i];
42307         if (!DemandedElts[i] || ShuffleUndef[i])
42308           continue;
42309         int OpIdx = M / NumElts;
42310         int EltIdx = M % NumElts;
42311         if (M < 0 || EltIdx != i) {
42312           IdentityOp.clearAllBits();
42313           break;
42314         }
42315         IdentityOp &= APInt::getOneBitSet(NumOps, OpIdx);
42316         if (IdentityOp == 0)
42317           break;
42318       }
42319       assert((IdentityOp == 0 || IdentityOp.popcount() == 1) &&
42320              "Multiple identity shuffles detected");
42321 
42322       if (IdentityOp != 0)
42323         return DAG.getBitcast(VT, ShuffleOps[IdentityOp.countr_zero()]);
42324     }
42325   }
42326 
42327   return TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
42328       Op, DemandedBits, DemandedElts, DAG, Depth);
42329 }
42330 
42331 bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
42332     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
42333     bool PoisonOnly, unsigned Depth) const {
42334   unsigned EltsBits = Op.getScalarValueSizeInBits();
42335   unsigned NumElts = DemandedElts.getBitWidth();
42336 
42337   // TODO: Add more target shuffles.
42338   switch (Op.getOpcode()) {
42339   case X86ISD::PSHUFD:
42340   case X86ISD::VPERMILPI: {
42341     SmallVector<int, 8> Mask;
42342     DecodePSHUFMask(NumElts, EltsBits, Op.getConstantOperandVal(1), Mask);
42343 
42344     APInt DemandedSrcElts = APInt::getZero(NumElts);
42345     for (unsigned I = 0; I != NumElts; ++I)
42346       if (DemandedElts[I])
42347         DemandedSrcElts.setBit(Mask[I]);
42348 
42349     return DAG.isGuaranteedNotToBeUndefOrPoison(
42350         Op.getOperand(0), DemandedSrcElts, PoisonOnly, Depth + 1);
42351   }
42352   }
42353   return TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
42354       Op, DemandedElts, DAG, PoisonOnly, Depth);
42355 }
42356 
42357 bool X86TargetLowering::canCreateUndefOrPoisonForTargetNode(
42358     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
42359     bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
42360 
42361   // TODO: Add more target shuffles.
42362   switch (Op.getOpcode()) {
42363   case X86ISD::PSHUFD:
42364   case X86ISD::VPERMILPI:
42365     return false;
42366   }
42367   return TargetLowering::canCreateUndefOrPoisonForTargetNode(
42368       Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
42369 }
42370 
42371 bool X86TargetLowering::isSplatValueForTargetNode(SDValue Op,
42372                                                   const APInt &DemandedElts,
42373                                                   APInt &UndefElts,
42374                                                   const SelectionDAG &DAG,
42375                                                   unsigned Depth) const {
42376   unsigned NumElts = DemandedElts.getBitWidth();
42377   unsigned Opc = Op.getOpcode();
42378 
42379   switch (Opc) {
42380   case X86ISD::VBROADCAST:
42381   case X86ISD::VBROADCAST_LOAD:
42382     UndefElts = APInt::getZero(NumElts);
42383     return true;
42384   }
42385 
42386   return TargetLowering::isSplatValueForTargetNode(Op, DemandedElts, UndefElts,
42387                                                    DAG, Depth);
42388 }
42389 
42390 // Helper to peek through bitops/trunc/setcc to determine size of source vector.
42391 // Allows combineBitcastvxi1 to determine what size vector generated a <X x i1>.
42392 static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
42393                                       bool AllowTruncate) {
42394   switch (Src.getOpcode()) {
42395   case ISD::TRUNCATE:
42396     if (!AllowTruncate)
42397       return false;
42398     [[fallthrough]];
42399   case ISD::SETCC:
42400     return Src.getOperand(0).getValueSizeInBits() == Size;
42401   case ISD::AND:
42402   case ISD::XOR:
42403   case ISD::OR:
42404     return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate) &&
42405            checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate);
42406   case ISD::SELECT:
42407   case ISD::VSELECT:
42408     return Src.getOperand(0).getScalarValueSizeInBits() == 1 &&
42409            checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate) &&
42410            checkBitcastSrcVectorSize(Src.getOperand(2), Size, AllowTruncate);
42411   case ISD::BUILD_VECTOR:
42412     return ISD::isBuildVectorAllZeros(Src.getNode()) ||
42413            ISD::isBuildVectorAllOnes(Src.getNode());
42414   }
42415   return false;
42416 }
42417 
42418 // Helper to flip between AND/OR/XOR opcodes and their X86ISD FP equivalents.
42419 static unsigned getAltBitOpcode(unsigned Opcode) {
42420   switch(Opcode) {
42421   case ISD::AND: return X86ISD::FAND;
42422   case ISD::OR: return X86ISD::FOR;
42423   case ISD::XOR: return X86ISD::FXOR;
42424   case X86ISD::ANDNP: return X86ISD::FANDN;
42425   }
42426   llvm_unreachable("Unknown bitwise opcode");
42427 }
42428 
42429 // Helper to adjust v4i32 MOVMSK expansion to work with SSE1-only targets.
42430 static SDValue adjustBitcastSrcVectorSSE1(SelectionDAG &DAG, SDValue Src,
42431                                           const SDLoc &DL) {
42432   EVT SrcVT = Src.getValueType();
42433   if (SrcVT != MVT::v4i1)
42434     return SDValue();
42435 
42436   switch (Src.getOpcode()) {
42437   case ISD::SETCC:
42438     if (Src.getOperand(0).getValueType() == MVT::v4i32 &&
42439         ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode()) &&
42440         cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT) {
42441       SDValue Op0 = Src.getOperand(0);
42442       if (ISD::isNormalLoad(Op0.getNode()))
42443         return DAG.getBitcast(MVT::v4f32, Op0);
42444       if (Op0.getOpcode() == ISD::BITCAST &&
42445           Op0.getOperand(0).getValueType() == MVT::v4f32)
42446         return Op0.getOperand(0);
42447     }
42448     break;
42449   case ISD::AND:
42450   case ISD::XOR:
42451   case ISD::OR: {
42452     SDValue Op0 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(0), DL);
42453     SDValue Op1 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(1), DL);
42454     if (Op0 && Op1)
42455       return DAG.getNode(getAltBitOpcode(Src.getOpcode()), DL, MVT::v4f32, Op0,
42456                          Op1);
42457     break;
42458   }
42459   }
42460   return SDValue();
42461 }
42462 
42463 // Helper to push sign extension of vXi1 SETCC result through bitops.
42464 static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
42465                                           SDValue Src, const SDLoc &DL) {
42466   switch (Src.getOpcode()) {
42467   case ISD::SETCC:
42468   case ISD::TRUNCATE:
42469   case ISD::BUILD_VECTOR:
42470     return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
42471   case ISD::AND:
42472   case ISD::XOR:
42473   case ISD::OR:
42474     return DAG.getNode(
42475         Src.getOpcode(), DL, SExtVT,
42476         signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL),
42477         signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL));
42478   case ISD::SELECT:
42479   case ISD::VSELECT:
42480     return DAG.getSelect(
42481         DL, SExtVT, Src.getOperand(0),
42482         signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL),
42483         signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(2), DL));
42484   }
42485   llvm_unreachable("Unexpected node type for vXi1 sign extension");
42486 }
42487 
42488 // Try to match patterns such as
42489 // (i16 bitcast (v16i1 x))
42490 // ->
42491 // (i16 movmsk (16i8 sext (v16i1 x)))
42492 // before the illegal vector is scalarized on subtargets that don't have legal
42493 // vxi1 types.
42494 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
42495                                   const SDLoc &DL,
42496                                   const X86Subtarget &Subtarget) {
42497   EVT SrcVT = Src.getValueType();
42498   if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
42499     return SDValue();
42500 
42501   // Recognize the IR pattern for the movmsk intrinsic under SSE1 before type
42502   // legalization destroys the v4i32 type.
42503   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2()) {
42504     if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) {
42505       V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32,
42506                       DAG.getBitcast(MVT::v4f32, V));
42507       return DAG.getZExtOrTrunc(V, DL, VT);
42508     }
42509   }
42510 
42511   // If the input is a truncate from v16i8 or v32i8 go ahead and use a
42512   // movmskb even with avx512. This will be better than truncating to vXi1 and
42513   // using a kmov. This can especially help KNL if the input is a v16i8/v32i8
42514   // vpcmpeqb/vpcmpgtb.
42515   bool PreferMovMsk = Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse() &&
42516                       (Src.getOperand(0).getValueType() == MVT::v16i8 ||
42517                        Src.getOperand(0).getValueType() == MVT::v32i8 ||
42518                        Src.getOperand(0).getValueType() == MVT::v64i8);
42519 
42520   // Prefer movmsk for AVX512 for (bitcast (setlt X, 0)) which can be handled
42521   // directly with vpmovmskb/vmovmskps/vmovmskpd.
42522   if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse() &&
42523       cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT &&
42524       ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode())) {
42525     EVT CmpVT = Src.getOperand(0).getValueType();
42526     EVT EltVT = CmpVT.getVectorElementType();
42527     if (CmpVT.getSizeInBits() <= 256 &&
42528         (EltVT == MVT::i8 || EltVT == MVT::i32 || EltVT == MVT::i64))
42529       PreferMovMsk = true;
42530   }
42531 
42532   // With AVX512 vxi1 types are legal and we prefer using k-regs.
42533   // MOVMSK is supported in SSE2 or later.
42534   if (!Subtarget.hasSSE2() || (Subtarget.hasAVX512() && !PreferMovMsk))
42535     return SDValue();
42536 
42537   // If the upper ops of a concatenation are undef, then try to bitcast the
42538   // lower op and extend.
42539   SmallVector<SDValue, 4> SubSrcOps;
42540   if (collectConcatOps(Src.getNode(), SubSrcOps, DAG) &&
42541       SubSrcOps.size() >= 2) {
42542     SDValue LowerOp = SubSrcOps[0];
42543     ArrayRef<SDValue> UpperOps(std::next(SubSrcOps.begin()), SubSrcOps.end());
42544     if (LowerOp.getOpcode() == ISD::SETCC &&
42545         all_of(UpperOps, [](SDValue Op) { return Op.isUndef(); })) {
42546       EVT SubVT = VT.getIntegerVT(
42547           *DAG.getContext(), LowerOp.getValueType().getVectorMinNumElements());
42548       if (SDValue V = combineBitcastvxi1(DAG, SubVT, LowerOp, DL, Subtarget)) {
42549         EVT IntVT = VT.getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
42550         return DAG.getBitcast(VT, DAG.getNode(ISD::ANY_EXTEND, DL, IntVT, V));
42551       }
42552     }
42553   }
42554 
42555   // There are MOVMSK flavors for types v16i8, v32i8, v4f32, v8f32, v4f64 and
42556   // v8f64. So all legal 128-bit and 256-bit vectors are covered except for
42557   // v8i16 and v16i16.
42558   // For these two cases, we can shuffle the upper element bytes to a
42559   // consecutive sequence at the start of the vector and treat the results as
42560   // v16i8 or v32i8, and for v16i8 this is the preferable solution. However,
42561   // for v16i16 this is not the case, because the shuffle is expensive, so we
42562   // avoid sign-extending to this type entirely.
42563   // For example, t0 := (v8i16 sext(v8i1 x)) needs to be shuffled as:
42564   // (v16i8 shuffle <0,2,4,6,8,10,12,14,u,u,...,u> (v16i8 bitcast t0), undef)
42565   MVT SExtVT;
42566   bool PropagateSExt = false;
42567   switch (SrcVT.getSimpleVT().SimpleTy) {
42568   default:
42569     return SDValue();
42570   case MVT::v2i1:
42571     SExtVT = MVT::v2i64;
42572     break;
42573   case MVT::v4i1:
42574     SExtVT = MVT::v4i32;
42575     // For cases such as (i4 bitcast (v4i1 setcc v4i64 v1, v2))
42576     // sign-extend to a 256-bit operation to avoid truncation.
42577     if (Subtarget.hasAVX() &&
42578         checkBitcastSrcVectorSize(Src, 256, Subtarget.hasAVX2())) {
42579       SExtVT = MVT::v4i64;
42580       PropagateSExt = true;
42581     }
42582     break;
42583   case MVT::v8i1:
42584     SExtVT = MVT::v8i16;
42585     // For cases such as (i8 bitcast (v8i1 setcc v8i32 v1, v2)),
42586     // sign-extend to a 256-bit operation to match the compare.
42587     // If the setcc operand is 128-bit, prefer sign-extending to 128-bit over
42588     // 256-bit because the shuffle is cheaper than sign extending the result of
42589     // the compare.
42590     if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256, true) ||
42591                                checkBitcastSrcVectorSize(Src, 512, true))) {
42592       SExtVT = MVT::v8i32;
42593       PropagateSExt = true;
42594     }
42595     break;
42596   case MVT::v16i1:
42597     SExtVT = MVT::v16i8;
42598     // For the case (i16 bitcast (v16i1 setcc v16i16 v1, v2)),
42599     // it is not profitable to sign-extend to 256-bit because this will
42600     // require an extra cross-lane shuffle which is more expensive than
42601     // truncating the result of the compare to 128-bits.
42602     break;
42603   case MVT::v32i1:
42604     SExtVT = MVT::v32i8;
42605     break;
42606   case MVT::v64i1:
42607     // If we have AVX512F, but not AVX512BW and the input is truncated from
42608     // v64i8 checked earlier. Then split the input and make two pmovmskbs.
42609     if (Subtarget.hasAVX512()) {
42610       if (Subtarget.hasBWI())
42611         return SDValue();
42612       SExtVT = MVT::v64i8;
42613       break;
42614     }
42615     // Split if this is a <64 x i8> comparison result.
42616     if (checkBitcastSrcVectorSize(Src, 512, false)) {
42617       SExtVT = MVT::v64i8;
42618       break;
42619     }
42620     return SDValue();
42621   };
42622 
42623   SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
42624                             : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
42625 
42626   if (SExtVT == MVT::v16i8 || SExtVT == MVT::v32i8 || SExtVT == MVT::v64i8) {
42627     V = getPMOVMSKB(DL, V, DAG, Subtarget);
42628   } else {
42629     if (SExtVT == MVT::v8i16) {
42630       V = widenSubVector(V, false, Subtarget, DAG, DL, 256);
42631       V = DAG.getNode(ISD::TRUNCATE, DL, MVT::v16i8, V);
42632     }
42633     V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
42634   }
42635 
42636   EVT IntVT =
42637       EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements());
42638   V = DAG.getZExtOrTrunc(V, DL, IntVT);
42639   return DAG.getBitcast(VT, V);
42640 }
42641 
42642 // Convert a vXi1 constant build vector to the same width scalar integer.
42643 static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
42644   EVT SrcVT = Op.getValueType();
42645   assert(SrcVT.getVectorElementType() == MVT::i1 &&
42646          "Expected a vXi1 vector");
42647   assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
42648          "Expected a constant build vector");
42649 
42650   APInt Imm(SrcVT.getVectorNumElements(), 0);
42651   for (unsigned Idx = 0, e = Op.getNumOperands(); Idx < e; ++Idx) {
42652     SDValue In = Op.getOperand(Idx);
42653     if (!In.isUndef() && (In->getAsZExtVal() & 0x1))
42654       Imm.setBit(Idx);
42655   }
42656   EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), Imm.getBitWidth());
42657   return DAG.getConstant(Imm, SDLoc(Op), IntVT);
42658 }
42659 
42660 static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG,
42661                                            TargetLowering::DAGCombinerInfo &DCI,
42662                                            const X86Subtarget &Subtarget) {
42663   assert(N->getOpcode() == ISD::BITCAST && "Expected a bitcast");
42664 
42665   if (!DCI.isBeforeLegalizeOps())
42666     return SDValue();
42667 
42668   // Only do this if we have k-registers.
42669   if (!Subtarget.hasAVX512())
42670     return SDValue();
42671 
42672   EVT DstVT = N->getValueType(0);
42673   SDValue Op = N->getOperand(0);
42674   EVT SrcVT = Op.getValueType();
42675 
42676   if (!Op.hasOneUse())
42677     return SDValue();
42678 
42679   // Look for logic ops.
42680   if (Op.getOpcode() != ISD::AND &&
42681       Op.getOpcode() != ISD::OR &&
42682       Op.getOpcode() != ISD::XOR)
42683     return SDValue();
42684 
42685   // Make sure we have a bitcast between mask registers and a scalar type.
42686   if (!(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&
42687         DstVT.isScalarInteger()) &&
42688       !(DstVT.isVector() && DstVT.getVectorElementType() == MVT::i1 &&
42689         SrcVT.isScalarInteger()))
42690     return SDValue();
42691 
42692   SDValue LHS = Op.getOperand(0);
42693   SDValue RHS = Op.getOperand(1);
42694 
42695   if (LHS.hasOneUse() && LHS.getOpcode() == ISD::BITCAST &&
42696       LHS.getOperand(0).getValueType() == DstVT)
42697     return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, LHS.getOperand(0),
42698                        DAG.getBitcast(DstVT, RHS));
42699 
42700   if (RHS.hasOneUse() && RHS.getOpcode() == ISD::BITCAST &&
42701       RHS.getOperand(0).getValueType() == DstVT)
42702     return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
42703                        DAG.getBitcast(DstVT, LHS), RHS.getOperand(0));
42704 
42705   // If the RHS is a vXi1 build vector, this is a good reason to flip too.
42706   // Most of these have to move a constant from the scalar domain anyway.
42707   if (ISD::isBuildVectorOfConstantSDNodes(RHS.getNode())) {
42708     RHS = combinevXi1ConstantToInteger(RHS, DAG);
42709     return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
42710                        DAG.getBitcast(DstVT, LHS), RHS);
42711   }
42712 
42713   return SDValue();
42714 }
42715 
42716 static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
42717                                     const X86Subtarget &Subtarget) {
42718   SDLoc DL(BV);
42719   unsigned NumElts = BV->getNumOperands();
42720   SDValue Splat = BV->getSplatValue();
42721 
42722   // Build MMX element from integer GPR or SSE float values.
42723   auto CreateMMXElement = [&](SDValue V) {
42724     if (V.isUndef())
42725       return DAG.getUNDEF(MVT::x86mmx);
42726     if (V.getValueType().isFloatingPoint()) {
42727       if (Subtarget.hasSSE1() && !isa<ConstantFPSDNode>(V)) {
42728         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V);
42729         V = DAG.getBitcast(MVT::v2i64, V);
42730         return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V);
42731       }
42732       V = DAG.getBitcast(MVT::i32, V);
42733     } else {
42734       V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32);
42735     }
42736     return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V);
42737   };
42738 
42739   // Convert build vector ops to MMX data in the bottom elements.
42740   SmallVector<SDValue, 8> Ops;
42741 
42742   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42743 
42744   // Broadcast - use (PUNPCKL+)PSHUFW to broadcast single element.
42745   if (Splat) {
42746     if (Splat.isUndef())
42747       return DAG.getUNDEF(MVT::x86mmx);
42748 
42749     Splat = CreateMMXElement(Splat);
42750 
42751     if (Subtarget.hasSSE1()) {
42752       // Unpack v8i8 to splat i8 elements to lowest 16-bits.
42753       if (NumElts == 8)
42754         Splat = DAG.getNode(
42755             ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
42756             DAG.getTargetConstant(Intrinsic::x86_mmx_punpcklbw, DL,
42757                                   TLI.getPointerTy(DAG.getDataLayout())),
42758             Splat, Splat);
42759 
42760       // Use PSHUFW to repeat 16-bit elements.
42761       unsigned ShufMask = (NumElts > 2 ? 0 : 0x44);
42762       return DAG.getNode(
42763           ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
42764           DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL,
42765                                 TLI.getPointerTy(DAG.getDataLayout())),
42766           Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8));
42767     }
42768     Ops.append(NumElts, Splat);
42769   } else {
42770     for (unsigned i = 0; i != NumElts; ++i)
42771       Ops.push_back(CreateMMXElement(BV->getOperand(i)));
42772   }
42773 
42774   // Use tree of PUNPCKLs to build up general MMX vector.
42775   while (Ops.size() > 1) {
42776     unsigned NumOps = Ops.size();
42777     unsigned IntrinOp =
42778         (NumOps == 2 ? Intrinsic::x86_mmx_punpckldq
42779                      : (NumOps == 4 ? Intrinsic::x86_mmx_punpcklwd
42780                                     : Intrinsic::x86_mmx_punpcklbw));
42781     SDValue Intrin = DAG.getTargetConstant(
42782         IntrinOp, DL, TLI.getPointerTy(DAG.getDataLayout()));
42783     for (unsigned i = 0; i != NumOps; i += 2)
42784       Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin,
42785                                Ops[i], Ops[i + 1]);
42786     Ops.resize(NumOps / 2);
42787   }
42788 
42789   return Ops[0];
42790 }
42791 
42792 // Recursive function that attempts to find if a bool vector node was originally
42793 // a vector/float/double that got truncated/extended/bitcast to/from a scalar
42794 // integer. If so, replace the scalar ops with bool vector equivalents back down
42795 // the chain.
42796 static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL,
42797                                           SelectionDAG &DAG,
42798                                           const X86Subtarget &Subtarget) {
42799   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42800   unsigned Opc = V.getOpcode();
42801   switch (Opc) {
42802   case ISD::BITCAST: {
42803     // Bitcast from a vector/float/double, we can cheaply bitcast to VT.
42804     SDValue Src = V.getOperand(0);
42805     EVT SrcVT = Src.getValueType();
42806     if (SrcVT.isVector() || SrcVT.isFloatingPoint())
42807       return DAG.getBitcast(VT, Src);
42808     break;
42809   }
42810   case ISD::TRUNCATE: {
42811     // If we find a suitable source, a truncated scalar becomes a subvector.
42812     SDValue Src = V.getOperand(0);
42813     EVT NewSrcVT =
42814         EVT::getVectorVT(*DAG.getContext(), MVT::i1, Src.getValueSizeInBits());
42815     if (TLI.isTypeLegal(NewSrcVT))
42816       if (SDValue N0 =
42817               combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, Subtarget))
42818         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0,
42819                            DAG.getIntPtrConstant(0, DL));
42820     break;
42821   }
42822   case ISD::ANY_EXTEND:
42823   case ISD::ZERO_EXTEND: {
42824     // If we find a suitable source, an extended scalar becomes a subvector.
42825     SDValue Src = V.getOperand(0);
42826     EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
42827                                     Src.getScalarValueSizeInBits());
42828     if (TLI.isTypeLegal(NewSrcVT))
42829       if (SDValue N0 =
42830               combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG, Subtarget))
42831         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
42832                            Opc == ISD::ANY_EXTEND ? DAG.getUNDEF(VT)
42833                                                   : DAG.getConstant(0, DL, VT),
42834                            N0, DAG.getIntPtrConstant(0, DL));
42835     break;
42836   }
42837   case ISD::OR: {
42838     // If we find suitable sources, we can just move an OR to the vector domain.
42839     SDValue Src0 = V.getOperand(0);
42840     SDValue Src1 = V.getOperand(1);
42841     if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
42842       if (SDValue N1 = combineBitcastToBoolVector(VT, Src1, DL, DAG, Subtarget))
42843         return DAG.getNode(Opc, DL, VT, N0, N1);
42844     break;
42845   }
42846   case ISD::SHL: {
42847     // If we find a suitable source, a SHL becomes a KSHIFTL.
42848     SDValue Src0 = V.getOperand(0);
42849     if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) ||
42850         ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI()))
42851       break;
42852 
42853     if (auto *Amt = dyn_cast<ConstantSDNode>(V.getOperand(1)))
42854       if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget))
42855         return DAG.getNode(
42856             X86ISD::KSHIFTL, DL, VT, N0,
42857             DAG.getTargetConstant(Amt->getZExtValue(), DL, MVT::i8));
42858     break;
42859   }
42860   }
42861   return SDValue();
42862 }
42863 
42864 static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
42865                               TargetLowering::DAGCombinerInfo &DCI,
42866                               const X86Subtarget &Subtarget) {
42867   SDValue N0 = N->getOperand(0);
42868   EVT VT = N->getValueType(0);
42869   EVT SrcVT = N0.getValueType();
42870   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42871 
42872   // Try to match patterns such as
42873   // (i16 bitcast (v16i1 x))
42874   // ->
42875   // (i16 movmsk (16i8 sext (v16i1 x)))
42876   // before the setcc result is scalarized on subtargets that don't have legal
42877   // vxi1 types.
42878   if (DCI.isBeforeLegalize()) {
42879     SDLoc dl(N);
42880     if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget))
42881       return V;
42882 
42883     // If this is a bitcast between a MVT::v4i1/v2i1 and an illegal integer
42884     // type, widen both sides to avoid a trip through memory.
42885     if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() &&
42886         Subtarget.hasAVX512()) {
42887       N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0);
42888       N0 = DAG.getBitcast(MVT::v8i1, N0);
42889       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0,
42890                          DAG.getIntPtrConstant(0, dl));
42891     }
42892 
42893     // If this is a bitcast between a MVT::v4i1/v2i1 and an illegal integer
42894     // type, widen both sides to avoid a trip through memory.
42895     if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() &&
42896         Subtarget.hasAVX512()) {
42897       // Use zeros for the widening if we already have some zeroes. This can
42898       // allow SimplifyDemandedBits to remove scalar ANDs that may be down
42899       // stream of this.
42900       // FIXME: It might make sense to detect a concat_vectors with a mix of
42901       // zeroes and undef and turn it into insert_subvector for i1 vectors as
42902       // a separate combine. What we can't do is canonicalize the operands of
42903       // such a concat or we'll get into a loop with SimplifyDemandedBits.
42904       if (N0.getOpcode() == ISD::CONCAT_VECTORS) {
42905         SDValue LastOp = N0.getOperand(N0.getNumOperands() - 1);
42906         if (ISD::isBuildVectorAllZeros(LastOp.getNode())) {
42907           SrcVT = LastOp.getValueType();
42908           unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
42909           SmallVector<SDValue, 4> Ops(N0->op_begin(), N0->op_end());
42910           Ops.resize(NumConcats, DAG.getConstant(0, dl, SrcVT));
42911           N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
42912           N0 = DAG.getBitcast(MVT::i8, N0);
42913           return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
42914         }
42915       }
42916 
42917       unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
42918       SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
42919       Ops[0] = N0;
42920       N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
42921       N0 = DAG.getBitcast(MVT::i8, N0);
42922       return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
42923     }
42924   } else {
42925     // If we're bitcasting from iX to vXi1, see if the integer originally
42926     // began as a vXi1 and whether we can remove the bitcast entirely.
42927     if (VT.isVector() && VT.getScalarType() == MVT::i1 &&
42928         SrcVT.isScalarInteger() && TLI.isTypeLegal(VT)) {
42929       if (SDValue V =
42930               combineBitcastToBoolVector(VT, N0, SDLoc(N), DAG, Subtarget))
42931         return V;
42932     }
42933   }
42934 
42935   // Look for (i8 (bitcast (v8i1 (extract_subvector (v16i1 X), 0)))) and
42936   // replace with (i8 (trunc (i16 (bitcast (v16i1 X))))). This can occur
42937   // due to insert_subvector legalization on KNL. By promoting the copy to i16
42938   // we can help with known bits propagation from the vXi1 domain to the
42939   // scalar domain.
42940   if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() &&
42941       !Subtarget.hasDQI() && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
42942       N0.getOperand(0).getValueType() == MVT::v16i1 &&
42943       isNullConstant(N0.getOperand(1)))
42944     return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT,
42945                        DAG.getBitcast(MVT::i16, N0.getOperand(0)));
42946 
42947   // Canonicalize (bitcast (vbroadcast_load)) so that the output of the bitcast
42948   // and the vbroadcast_load are both integer or both fp. In some cases this
42949   // will remove the bitcast entirely.
42950   if (N0.getOpcode() == X86ISD::VBROADCAST_LOAD && N0.hasOneUse() &&
42951        VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) {
42952     auto *BCast = cast<MemIntrinsicSDNode>(N0);
42953     unsigned SrcVTSize = SrcVT.getScalarSizeInBits();
42954     unsigned MemSize = BCast->getMemoryVT().getScalarSizeInBits();
42955     // Don't swap i8/i16 since don't have fp types that size.
42956     if (MemSize >= 32) {
42957       MVT MemVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(MemSize)
42958                                        : MVT::getIntegerVT(MemSize);
42959       MVT LoadVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(SrcVTSize)
42960                                         : MVT::getIntegerVT(SrcVTSize);
42961       LoadVT = MVT::getVectorVT(LoadVT, SrcVT.getVectorNumElements());
42962 
42963       SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
42964       SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
42965       SDValue ResNode =
42966           DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
42967                                   MemVT, BCast->getMemOperand());
42968       DAG.ReplaceAllUsesOfValueWith(SDValue(BCast, 1), ResNode.getValue(1));
42969       return DAG.getBitcast(VT, ResNode);
42970     }
42971   }
42972 
42973   // Since MMX types are special and don't usually play with other vector types,
42974   // it's better to handle them early to be sure we emit efficient code by
42975   // avoiding store-load conversions.
42976   if (VT == MVT::x86mmx) {
42977     // Detect MMX constant vectors.
42978     APInt UndefElts;
42979     SmallVector<APInt, 1> EltBits;
42980     if (getTargetConstantBitsFromNode(N0, 64, UndefElts, EltBits)) {
42981       SDLoc DL(N0);
42982       // Handle zero-extension of i32 with MOVD.
42983       if (EltBits[0].countl_zero() >= 32)
42984         return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT,
42985                            DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32));
42986       // Else, bitcast to a double.
42987       // TODO - investigate supporting sext 32-bit immediates on x86_64.
42988       APFloat F64(APFloat::IEEEdouble(), EltBits[0]);
42989       return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64));
42990     }
42991 
42992     // Detect bitcasts to x86mmx low word.
42993     if (N0.getOpcode() == ISD::BUILD_VECTOR &&
42994         (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) &&
42995         N0.getOperand(0).getValueType() == SrcVT.getScalarType()) {
42996       bool LowUndef = true, AllUndefOrZero = true;
42997       for (unsigned i = 1, e = SrcVT.getVectorNumElements(); i != e; ++i) {
42998         SDValue Op = N0.getOperand(i);
42999         LowUndef &= Op.isUndef() || (i >= e/2);
43000         AllUndefOrZero &= (Op.isUndef() || isNullConstant(Op));
43001       }
43002       if (AllUndefOrZero) {
43003         SDValue N00 = N0.getOperand(0);
43004         SDLoc dl(N00);
43005         N00 = LowUndef ? DAG.getAnyExtOrTrunc(N00, dl, MVT::i32)
43006                        : DAG.getZExtOrTrunc(N00, dl, MVT::i32);
43007         return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00);
43008       }
43009     }
43010 
43011     // Detect bitcasts of 64-bit build vectors and convert to a
43012     // MMX UNPCK/PSHUFW which takes MMX type inputs with the value in the
43013     // lowest element.
43014     if (N0.getOpcode() == ISD::BUILD_VECTOR &&
43015         (SrcVT == MVT::v2f32 || SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 ||
43016          SrcVT == MVT::v8i8))
43017       return createMMXBuildVector(cast<BuildVectorSDNode>(N0), DAG, Subtarget);
43018 
43019     // Detect bitcasts between element or subvector extraction to x86mmx.
43020     if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
43021          N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) &&
43022         isNullConstant(N0.getOperand(1))) {
43023       SDValue N00 = N0.getOperand(0);
43024       if (N00.getValueType().is128BitVector())
43025         return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT,
43026                            DAG.getBitcast(MVT::v2i64, N00));
43027     }
43028 
43029     // Detect bitcasts from FP_TO_SINT to x86mmx.
43030     if (SrcVT == MVT::v2i32 && N0.getOpcode() == ISD::FP_TO_SINT) {
43031       SDLoc DL(N0);
43032       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
43033                                 DAG.getUNDEF(MVT::v2i32));
43034       return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT,
43035                          DAG.getBitcast(MVT::v2i64, Res));
43036     }
43037   }
43038 
43039   // Try to remove a bitcast of constant vXi1 vector. We have to legalize
43040   // most of these to scalar anyway.
43041   if (Subtarget.hasAVX512() && VT.isScalarInteger() &&
43042       SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&
43043       ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
43044     return combinevXi1ConstantToInteger(N0, DAG);
43045   }
43046 
43047   if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() &&
43048       VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
43049       isa<ConstantSDNode>(N0)) {
43050     auto *C = cast<ConstantSDNode>(N0);
43051     if (C->isAllOnes())
43052       return DAG.getConstant(1, SDLoc(N0), VT);
43053     if (C->isZero())
43054       return DAG.getConstant(0, SDLoc(N0), VT);
43055   }
43056 
43057   // Look for MOVMSK that is maybe truncated and then bitcasted to vXi1.
43058   // Turn it into a sign bit compare that produces a k-register. This avoids
43059   // a trip through a GPR.
43060   if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() &&
43061       VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
43062       isPowerOf2_32(VT.getVectorNumElements())) {
43063     unsigned NumElts = VT.getVectorNumElements();
43064     SDValue Src = N0;
43065 
43066     // Peek through truncate.
43067     if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse())
43068       Src = N0.getOperand(0);
43069 
43070     if (Src.getOpcode() == X86ISD::MOVMSK && Src.hasOneUse()) {
43071       SDValue MovmskIn = Src.getOperand(0);
43072       MVT MovmskVT = MovmskIn.getSimpleValueType();
43073       unsigned MovMskElts = MovmskVT.getVectorNumElements();
43074 
43075       // We allow extra bits of the movmsk to be used since they are known zero.
43076       // We can't convert a VPMOVMSKB without avx512bw.
43077       if (MovMskElts <= NumElts &&
43078           (Subtarget.hasBWI() || MovmskVT.getVectorElementType() != MVT::i8)) {
43079         EVT IntVT = EVT(MovmskVT).changeVectorElementTypeToInteger();
43080         MovmskIn = DAG.getBitcast(IntVT, MovmskIn);
43081         SDLoc dl(N);
43082         MVT CmpVT = MVT::getVectorVT(MVT::i1, MovMskElts);
43083         SDValue Cmp = DAG.getSetCC(dl, CmpVT, MovmskIn,
43084                                    DAG.getConstant(0, dl, IntVT), ISD::SETLT);
43085         if (EVT(CmpVT) == VT)
43086           return Cmp;
43087 
43088         // Pad with zeroes up to original VT to replace the zeroes that were
43089         // being used from the MOVMSK.
43090         unsigned NumConcats = NumElts / MovMskElts;
43091         SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, CmpVT));
43092         Ops[0] = Cmp;
43093         return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops);
43094       }
43095     }
43096   }
43097 
43098   // Try to remove bitcasts from input and output of mask arithmetic to
43099   // remove GPR<->K-register crossings.
43100   if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
43101     return V;
43102 
43103   // Convert a bitcasted integer logic operation that has one bitcasted
43104   // floating-point operand into a floating-point logic operation. This may
43105   // create a load of a constant, but that is cheaper than materializing the
43106   // constant in an integer register and transferring it to an SSE register or
43107   // transferring the SSE operand to integer register and back.
43108   unsigned FPOpcode;
43109   switch (N0.getOpcode()) {
43110     case ISD::AND: FPOpcode = X86ISD::FAND; break;
43111     case ISD::OR:  FPOpcode = X86ISD::FOR;  break;
43112     case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
43113     default: return SDValue();
43114   }
43115 
43116   // Check if we have a bitcast from another integer type as well.
43117   if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
43118         (Subtarget.hasSSE2() && VT == MVT::f64) ||
43119         (Subtarget.hasFP16() && VT == MVT::f16) ||
43120         (Subtarget.hasSSE2() && VT.isInteger() && VT.isVector() &&
43121          TLI.isTypeLegal(VT))))
43122     return SDValue();
43123 
43124   SDValue LogicOp0 = N0.getOperand(0);
43125   SDValue LogicOp1 = N0.getOperand(1);
43126   SDLoc DL0(N0);
43127 
43128   // bitcast(logic(bitcast(X), Y)) --> logic'(X, bitcast(Y))
43129   if (N0.hasOneUse() && LogicOp0.getOpcode() == ISD::BITCAST &&
43130       LogicOp0.hasOneUse() && LogicOp0.getOperand(0).hasOneUse() &&
43131       LogicOp0.getOperand(0).getValueType() == VT &&
43132       !isa<ConstantSDNode>(LogicOp0.getOperand(0))) {
43133     SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);
43134     unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode();
43135     return DAG.getNode(Opcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1);
43136   }
43137   // bitcast(logic(X, bitcast(Y))) --> logic'(bitcast(X), Y)
43138   if (N0.hasOneUse() && LogicOp1.getOpcode() == ISD::BITCAST &&
43139       LogicOp1.hasOneUse() && LogicOp1.getOperand(0).hasOneUse() &&
43140       LogicOp1.getOperand(0).getValueType() == VT &&
43141       !isa<ConstantSDNode>(LogicOp1.getOperand(0))) {
43142     SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);
43143     unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode();
43144     return DAG.getNode(Opcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0);
43145   }
43146 
43147   return SDValue();
43148 }
43149 
43150 // (mul (zext a), (sext, b))
43151 static bool detectExtMul(SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0,
43152                          SDValue &Op1) {
43153   Op0 = Mul.getOperand(0);
43154   Op1 = Mul.getOperand(1);
43155 
43156   // The operand1 should be signed extend
43157   if (Op0.getOpcode() == ISD::SIGN_EXTEND)
43158     std::swap(Op0, Op1);
43159 
43160   auto IsFreeTruncation = [](SDValue &Op) -> bool {
43161     if ((Op.getOpcode() == ISD::ZERO_EXTEND ||
43162          Op.getOpcode() == ISD::SIGN_EXTEND) &&
43163         Op.getOperand(0).getScalarValueSizeInBits() <= 8)
43164       return true;
43165 
43166     auto *BV = dyn_cast<BuildVectorSDNode>(Op);
43167     return (BV && BV->isConstant());
43168   };
43169 
43170   // (dpbusd (zext a), (sext, b)). Since the first operand should be unsigned
43171   // value, we need to check Op0 is zero extended value. Op1 should be signed
43172   // value, so we just check the signed bits.
43173   if ((IsFreeTruncation(Op0) &&
43174        DAG.computeKnownBits(Op0).countMaxActiveBits() <= 8) &&
43175       (IsFreeTruncation(Op1) && DAG.ComputeMaxSignificantBits(Op1) <= 8))
43176     return true;
43177 
43178   return false;
43179 }
43180 
43181 // Given a ABS node, detect the following pattern:
43182 // (ABS (SUB (ZERO_EXTEND a), (ZERO_EXTEND b))).
43183 // This is useful as it is the input into a SAD pattern.
43184 static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
43185   SDValue AbsOp1 = Abs->getOperand(0);
43186   if (AbsOp1.getOpcode() != ISD::SUB)
43187     return false;
43188 
43189   Op0 = AbsOp1.getOperand(0);
43190   Op1 = AbsOp1.getOperand(1);
43191 
43192   // Check if the operands of the sub are zero-extended from vectors of i8.
43193   if (Op0.getOpcode() != ISD::ZERO_EXTEND ||
43194       Op0.getOperand(0).getValueType().getVectorElementType() != MVT::i8 ||
43195       Op1.getOpcode() != ISD::ZERO_EXTEND ||
43196       Op1.getOperand(0).getValueType().getVectorElementType() != MVT::i8)
43197     return false;
43198 
43199   return true;
43200 }
43201 
43202 static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
43203                               unsigned &LogBias, const SDLoc &DL,
43204                               const X86Subtarget &Subtarget) {
43205   // Extend or truncate to MVT::i8 first.
43206   MVT Vi8VT =
43207       MVT::getVectorVT(MVT::i8, LHS.getValueType().getVectorElementCount());
43208   LHS = DAG.getZExtOrTrunc(LHS, DL, Vi8VT);
43209   RHS = DAG.getSExtOrTrunc(RHS, DL, Vi8VT);
43210 
43211   // VPDPBUSD(<16 x i32>C, <16 x i8>A, <16 x i8>B). For each dst element
43212   // C[0] = C[0] + A[0]B[0] + A[1]B[1] + A[2]B[2] + A[3]B[3].
43213   // The src A, B element type is i8, but the dst C element type is i32.
43214   // When we calculate the reduce stage, we use src vector type vXi8 for it
43215   // so we need logbias 2 to avoid extra 2 stages.
43216   LogBias = 2;
43217 
43218   unsigned RegSize = std::max(128u, (unsigned)Vi8VT.getSizeInBits());
43219   if (Subtarget.hasVNNI() && !Subtarget.hasVLX())
43220     RegSize = std::max(512u, RegSize);
43221 
43222   // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we
43223   // fill in the missing vector elements with 0.
43224   unsigned NumConcat = RegSize / Vi8VT.getSizeInBits();
43225   SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, Vi8VT));
43226   Ops[0] = LHS;
43227   MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);
43228   SDValue DpOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
43229   Ops[0] = RHS;
43230   SDValue DpOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
43231 
43232   // Actually build the DotProduct, split as 256/512 bits for
43233   // AVXVNNI/AVX512VNNI.
43234   auto DpBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
43235                        ArrayRef<SDValue> Ops) {
43236     MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
43237     return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops);
43238   };
43239   MVT DpVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
43240   SDValue Zero = DAG.getConstant(0, DL, DpVT);
43241 
43242   return SplitOpsAndApply(DAG, Subtarget, DL, DpVT, {Zero, DpOp0, DpOp1},
43243                           DpBuilder, false);
43244 }
43245 
43246 // Given two zexts of <k x i8> to <k x i32>, create a PSADBW of the inputs
43247 // to these zexts.
43248 static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
43249                             const SDValue &Zext1, const SDLoc &DL,
43250                             const X86Subtarget &Subtarget) {
43251   // Find the appropriate width for the PSADBW.
43252   EVT InVT = Zext0.getOperand(0).getValueType();
43253   unsigned RegSize = std::max(128u, (unsigned)InVT.getSizeInBits());
43254 
43255   // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we
43256   // fill in the missing vector elements with 0.
43257   unsigned NumConcat = RegSize / InVT.getSizeInBits();
43258   SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
43259   Ops[0] = Zext0.getOperand(0);
43260   MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);
43261   SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
43262   Ops[0] = Zext1.getOperand(0);
43263   SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
43264 
43265   // Actually build the SAD, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
43266   auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
43267                           ArrayRef<SDValue> Ops) {
43268     MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
43269     return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops);
43270   };
43271   MVT SadVT = MVT::getVectorVT(MVT::i64, RegSize / 64);
43272   return SplitOpsAndApply(DAG, Subtarget, DL, SadVT, { SadOp0, SadOp1 },
43273                           PSADBWBuilder);
43274 }
43275 
43276 // Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
43277 // PHMINPOSUW.
43278 static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
43279                                       const X86Subtarget &Subtarget) {
43280   // Bail without SSE41.
43281   if (!Subtarget.hasSSE41())
43282     return SDValue();
43283 
43284   EVT ExtractVT = Extract->getValueType(0);
43285   if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
43286     return SDValue();
43287 
43288   // Check for SMAX/SMIN/UMAX/UMIN horizontal reduction patterns.
43289   ISD::NodeType BinOp;
43290   SDValue Src = DAG.matchBinOpReduction(
43291       Extract, BinOp, {ISD::SMAX, ISD::SMIN, ISD::UMAX, ISD::UMIN}, true);
43292   if (!Src)
43293     return SDValue();
43294 
43295   EVT SrcVT = Src.getValueType();
43296   EVT SrcSVT = SrcVT.getScalarType();
43297   if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
43298     return SDValue();
43299 
43300   SDLoc DL(Extract);
43301   SDValue MinPos = Src;
43302 
43303   // First, reduce the source down to 128-bit, applying BinOp to lo/hi.
43304   while (SrcVT.getSizeInBits() > 128) {
43305     SDValue Lo, Hi;
43306     std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
43307     SrcVT = Lo.getValueType();
43308     MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
43309   }
43310   assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||
43311           (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
43312          "Unexpected value type");
43313 
43314   // PHMINPOSUW applies to UMIN(v8i16), for SMIN/SMAX/UMAX we must apply a mask
43315   // to flip the value accordingly.
43316   SDValue Mask;
43317   unsigned MaskEltsBits = ExtractVT.getSizeInBits();
43318   if (BinOp == ISD::SMAX)
43319     Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
43320   else if (BinOp == ISD::SMIN)
43321     Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
43322   else if (BinOp == ISD::UMAX)
43323     Mask = DAG.getAllOnesConstant(DL, SrcVT);
43324 
43325   if (Mask)
43326     MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
43327 
43328   // For v16i8 cases we need to perform UMIN on pairs of byte elements,
43329   // shuffling each upper element down and insert zeros. This means that the
43330   // v16i8 UMIN will leave the upper element as zero, performing zero-extension
43331   // ready for the PHMINPOS.
43332   if (ExtractVT == MVT::i8) {
43333     SDValue Upper = DAG.getVectorShuffle(
43334         SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
43335         {1, 16, 3, 16, 5, 16, 7, 16, 9, 16, 11, 16, 13, 16, 15, 16});
43336     MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
43337   }
43338 
43339   // Perform the PHMINPOS on a v8i16 vector,
43340   MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
43341   MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
43342   MinPos = DAG.getBitcast(SrcVT, MinPos);
43343 
43344   if (Mask)
43345     MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
43346 
43347   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
43348                      DAG.getIntPtrConstant(0, DL));
43349 }
43350 
43351 // Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
43352 static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG,
43353                                          const X86Subtarget &Subtarget) {
43354   // Bail without SSE2.
43355   if (!Subtarget.hasSSE2())
43356     return SDValue();
43357 
43358   EVT ExtractVT = Extract->getValueType(0);
43359   unsigned BitWidth = ExtractVT.getSizeInBits();
43360   if (ExtractVT != MVT::i64 && ExtractVT != MVT::i32 && ExtractVT != MVT::i16 &&
43361       ExtractVT != MVT::i8 && ExtractVT != MVT::i1)
43362     return SDValue();
43363 
43364   // Check for OR(any_of)/AND(all_of)/XOR(parity) horizontal reduction patterns.
43365   ISD::NodeType BinOp;
43366   SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND});
43367   if (!Match && ExtractVT == MVT::i1)
43368     Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::XOR});
43369   if (!Match)
43370     return SDValue();
43371 
43372   // EXTRACT_VECTOR_ELT can require implicit extension of the vector element
43373   // which we can't support here for now.
43374   if (Match.getScalarValueSizeInBits() != BitWidth)
43375     return SDValue();
43376 
43377   SDValue Movmsk;
43378   SDLoc DL(Extract);
43379   EVT MatchVT = Match.getValueType();
43380   unsigned NumElts = MatchVT.getVectorNumElements();
43381   unsigned MaxElts = Subtarget.hasInt256() ? 32 : 16;
43382   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43383   LLVMContext &Ctx = *DAG.getContext();
43384 
43385   if (ExtractVT == MVT::i1) {
43386     // Special case for (pre-legalization) vXi1 reductions.
43387     if (NumElts > 64 || !isPowerOf2_32(NumElts))
43388       return SDValue();
43389     if (Match.getOpcode() == ISD::SETCC) {
43390       ISD::CondCode CC = cast<CondCodeSDNode>(Match.getOperand(2))->get();
43391       if ((BinOp == ISD::AND && CC == ISD::CondCode::SETEQ) ||
43392           (BinOp == ISD::OR && CC == ISD::CondCode::SETNE)) {
43393         // For all_of(setcc(x,y,eq)) - use (iX)x == (iX)y.
43394         // For any_of(setcc(x,y,ne)) - use (iX)x != (iX)y.
43395         X86::CondCode X86CC;
43396         SDValue LHS = DAG.getFreeze(Match.getOperand(0));
43397         SDValue RHS = DAG.getFreeze(Match.getOperand(1));
43398         APInt Mask = APInt::getAllOnes(LHS.getScalarValueSizeInBits());
43399         if (SDValue V = LowerVectorAllEqual(DL, LHS, RHS, CC, Mask, Subtarget,
43400                                             DAG, X86CC))
43401           return DAG.getNode(ISD::TRUNCATE, DL, ExtractVT,
43402                              getSETCC(X86CC, V, DL, DAG));
43403       }
43404     }
43405     if (TLI.isTypeLegal(MatchVT)) {
43406       // If this is a legal AVX512 predicate type then we can just bitcast.
43407       EVT MovmskVT = EVT::getIntegerVT(Ctx, NumElts);
43408       Movmsk = DAG.getBitcast(MovmskVT, Match);
43409     } else {
43410       // Use combineBitcastvxi1 to create the MOVMSK.
43411       while (NumElts > MaxElts) {
43412         SDValue Lo, Hi;
43413         std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
43414         Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
43415         NumElts /= 2;
43416       }
43417       EVT MovmskVT = EVT::getIntegerVT(Ctx, NumElts);
43418       Movmsk = combineBitcastvxi1(DAG, MovmskVT, Match, DL, Subtarget);
43419     }
43420     if (!Movmsk)
43421       return SDValue();
43422     Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32);
43423   } else {
43424     // FIXME: Better handling of k-registers or 512-bit vectors?
43425     unsigned MatchSizeInBits = Match.getValueSizeInBits();
43426     if (!(MatchSizeInBits == 128 ||
43427           (MatchSizeInBits == 256 && Subtarget.hasAVX())))
43428       return SDValue();
43429 
43430     // Make sure this isn't a vector of 1 element. The perf win from using
43431     // MOVMSK diminishes with less elements in the reduction, but it is
43432     // generally better to get the comparison over to the GPRs as soon as
43433     // possible to reduce the number of vector ops.
43434     if (Match.getValueType().getVectorNumElements() < 2)
43435       return SDValue();
43436 
43437     // Check that we are extracting a reduction of all sign bits.
43438     if (DAG.ComputeNumSignBits(Match) != BitWidth)
43439       return SDValue();
43440 
43441     if (MatchSizeInBits == 256 && BitWidth < 32 && !Subtarget.hasInt256()) {
43442       SDValue Lo, Hi;
43443       std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
43444       Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
43445       MatchSizeInBits = Match.getValueSizeInBits();
43446     }
43447 
43448     // For 32/64 bit comparisons use MOVMSKPS/MOVMSKPD, else PMOVMSKB.
43449     MVT MaskSrcVT;
43450     if (64 == BitWidth || 32 == BitWidth)
43451       MaskSrcVT = MVT::getVectorVT(MVT::getFloatingPointVT(BitWidth),
43452                                    MatchSizeInBits / BitWidth);
43453     else
43454       MaskSrcVT = MVT::getVectorVT(MVT::i8, MatchSizeInBits / 8);
43455 
43456     SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match);
43457     Movmsk = getPMOVMSKB(DL, BitcastLogicOp, DAG, Subtarget);
43458     NumElts = MaskSrcVT.getVectorNumElements();
43459   }
43460   assert((NumElts <= 32 || NumElts == 64) &&
43461          "Not expecting more than 64 elements");
43462 
43463   MVT CmpVT = NumElts == 64 ? MVT::i64 : MVT::i32;
43464   if (BinOp == ISD::XOR) {
43465     // parity -> (PARITY(MOVMSK X))
43466     SDValue Result = DAG.getNode(ISD::PARITY, DL, CmpVT, Movmsk);
43467     return DAG.getZExtOrTrunc(Result, DL, ExtractVT);
43468   }
43469 
43470   SDValue CmpC;
43471   ISD::CondCode CondCode;
43472   if (BinOp == ISD::OR) {
43473     // any_of -> MOVMSK != 0
43474     CmpC = DAG.getConstant(0, DL, CmpVT);
43475     CondCode = ISD::CondCode::SETNE;
43476   } else {
43477     // all_of -> MOVMSK == ((1 << NumElts) - 1)
43478     CmpC = DAG.getConstant(APInt::getLowBitsSet(CmpVT.getSizeInBits(), NumElts),
43479                            DL, CmpVT);
43480     CondCode = ISD::CondCode::SETEQ;
43481   }
43482 
43483   // The setcc produces an i8 of 0/1, so extend that to the result width and
43484   // negate to get the final 0/-1 mask value.
43485   EVT SetccVT = TLI.getSetCCResultType(DAG.getDataLayout(), Ctx, CmpVT);
43486   SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
43487   SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
43488   SDValue Zero = DAG.getConstant(0, DL, ExtractVT);
43489   return DAG.getNode(ISD::SUB, DL, ExtractVT, Zero, Zext);
43490 }
43491 
43492 static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
43493                                       const X86Subtarget &Subtarget) {
43494   if (!Subtarget.hasVNNI() && !Subtarget.hasAVXVNNI())
43495     return SDValue();
43496 
43497   EVT ExtractVT = Extract->getValueType(0);
43498   // Verify the type we're extracting is i32, as the output element type of
43499   // vpdpbusd is i32.
43500   if (ExtractVT != MVT::i32)
43501     return SDValue();
43502 
43503   EVT VT = Extract->getOperand(0).getValueType();
43504   if (!isPowerOf2_32(VT.getVectorNumElements()))
43505     return SDValue();
43506 
43507   // Match shuffle + add pyramid.
43508   ISD::NodeType BinOp;
43509   SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
43510 
43511   // We can't combine to vpdpbusd for zext, because each of the 4 multiplies
43512   // done by vpdpbusd compute a signed 16-bit product that will be sign extended
43513   // before adding into the accumulator.
43514   // TODO:
43515   // We also need to verify that the multiply has at least 2x the number of bits
43516   // of the input. We shouldn't match
43517   // (sign_extend (mul (vXi9 (zext (vXi8 X))), (vXi9 (zext (vXi8 Y)))).
43518   // if (Root && (Root.getOpcode() == ISD::SIGN_EXTEND))
43519   //   Root = Root.getOperand(0);
43520 
43521   // If there was a match, we want Root to be a mul.
43522   if (!Root || Root.getOpcode() != ISD::MUL)
43523     return SDValue();
43524 
43525   // Check whether we have an extend and mul pattern
43526   SDValue LHS, RHS;
43527   if (!detectExtMul(DAG, Root, LHS, RHS))
43528     return SDValue();
43529 
43530   // Create the dot product instruction.
43531   SDLoc DL(Extract);
43532   unsigned StageBias;
43533   SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget);
43534 
43535   // If the original vector was wider than 4 elements, sum over the results
43536   // in the DP vector.
43537   unsigned Stages = Log2_32(VT.getVectorNumElements());
43538   EVT DpVT = DP.getValueType();
43539 
43540   if (Stages > StageBias) {
43541     unsigned DpElems = DpVT.getVectorNumElements();
43542 
43543     for (unsigned i = Stages - StageBias; i > 0; --i) {
43544       SmallVector<int, 16> Mask(DpElems, -1);
43545       for (unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
43546         Mask[j] = MaskEnd + j;
43547 
43548       SDValue Shuffle =
43549           DAG.getVectorShuffle(DpVT, DL, DP, DAG.getUNDEF(DpVT), Mask);
43550       DP = DAG.getNode(ISD::ADD, DL, DpVT, DP, Shuffle);
43551     }
43552   }
43553 
43554   // Return the lowest ExtractSizeInBits bits.
43555   EVT ResVT =
43556       EVT::getVectorVT(*DAG.getContext(), ExtractVT,
43557                        DpVT.getSizeInBits() / ExtractVT.getSizeInBits());
43558   DP = DAG.getBitcast(ResVT, DP);
43559   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, DP,
43560                      Extract->getOperand(1));
43561 }
43562 
43563 static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
43564                                       const X86Subtarget &Subtarget) {
43565   // PSADBW is only supported on SSE2 and up.
43566   if (!Subtarget.hasSSE2())
43567     return SDValue();
43568 
43569   EVT ExtractVT = Extract->getValueType(0);
43570   // Verify the type we're extracting is either i32 or i64.
43571   // FIXME: Could support other types, but this is what we have coverage for.
43572   if (ExtractVT != MVT::i32 && ExtractVT != MVT::i64)
43573     return SDValue();
43574 
43575   EVT VT = Extract->getOperand(0).getValueType();
43576   if (!isPowerOf2_32(VT.getVectorNumElements()))
43577     return SDValue();
43578 
43579   // Match shuffle + add pyramid.
43580   ISD::NodeType BinOp;
43581   SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
43582 
43583   // The operand is expected to be zero extended from i8
43584   // (verified in detectZextAbsDiff).
43585   // In order to convert to i64 and above, additional any/zero/sign
43586   // extend is expected.
43587   // The zero extend from 32 bit has no mathematical effect on the result.
43588   // Also the sign extend is basically zero extend
43589   // (extends the sign bit which is zero).
43590   // So it is correct to skip the sign/zero extend instruction.
43591   if (Root && (Root.getOpcode() == ISD::SIGN_EXTEND ||
43592                Root.getOpcode() == ISD::ZERO_EXTEND ||
43593                Root.getOpcode() == ISD::ANY_EXTEND))
43594     Root = Root.getOperand(0);
43595 
43596   // If there was a match, we want Root to be a select that is the root of an
43597   // abs-diff pattern.
43598   if (!Root || Root.getOpcode() != ISD::ABS)
43599     return SDValue();
43600 
43601   // Check whether we have an abs-diff pattern feeding into the select.
43602   SDValue Zext0, Zext1;
43603   if (!detectZextAbsDiff(Root, Zext0, Zext1))
43604     return SDValue();
43605 
43606   // Create the SAD instruction.
43607   SDLoc DL(Extract);
43608   SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
43609 
43610   // If the original vector was wider than 8 elements, sum over the results
43611   // in the SAD vector.
43612   unsigned Stages = Log2_32(VT.getVectorNumElements());
43613   EVT SadVT = SAD.getValueType();
43614   if (Stages > 3) {
43615     unsigned SadElems = SadVT.getVectorNumElements();
43616 
43617     for(unsigned i = Stages - 3; i > 0; --i) {
43618       SmallVector<int, 16> Mask(SadElems, -1);
43619       for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
43620         Mask[j] = MaskEnd + j;
43621 
43622       SDValue Shuffle =
43623           DAG.getVectorShuffle(SadVT, DL, SAD, DAG.getUNDEF(SadVT), Mask);
43624       SAD = DAG.getNode(ISD::ADD, DL, SadVT, SAD, Shuffle);
43625     }
43626   }
43627 
43628   unsigned ExtractSizeInBits = ExtractVT.getSizeInBits();
43629   // Return the lowest ExtractSizeInBits bits.
43630   EVT ResVT = EVT::getVectorVT(*DAG.getContext(), ExtractVT,
43631                                SadVT.getSizeInBits() / ExtractSizeInBits);
43632   SAD = DAG.getBitcast(ResVT, SAD);
43633   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD,
43634                      Extract->getOperand(1));
43635 }
43636 
43637 // Attempt to peek through a target shuffle and extract the scalar from the
43638 // source.
43639 static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
43640                                          TargetLowering::DAGCombinerInfo &DCI,
43641                                          const X86Subtarget &Subtarget) {
43642   if (DCI.isBeforeLegalizeOps())
43643     return SDValue();
43644 
43645   SDLoc dl(N);
43646   SDValue Src = N->getOperand(0);
43647   SDValue Idx = N->getOperand(1);
43648 
43649   EVT VT = N->getValueType(0);
43650   EVT SrcVT = Src.getValueType();
43651   EVT SrcSVT = SrcVT.getVectorElementType();
43652   unsigned SrcEltBits = SrcSVT.getSizeInBits();
43653   unsigned NumSrcElts = SrcVT.getVectorNumElements();
43654 
43655   // Don't attempt this for boolean mask vectors or unknown extraction indices.
43656   if (SrcSVT == MVT::i1 || !isa<ConstantSDNode>(Idx))
43657     return SDValue();
43658 
43659   const APInt &IdxC = N->getConstantOperandAPInt(1);
43660   if (IdxC.uge(NumSrcElts))
43661     return SDValue();
43662 
43663   SDValue SrcBC = peekThroughBitcasts(Src);
43664 
43665   // Handle extract(bitcast(broadcast(scalar_value))).
43666   if (X86ISD::VBROADCAST == SrcBC.getOpcode()) {
43667     SDValue SrcOp = SrcBC.getOperand(0);
43668     EVT SrcOpVT = SrcOp.getValueType();
43669     if (SrcOpVT.isScalarInteger() && VT.isInteger() &&
43670         (SrcOpVT.getSizeInBits() % SrcEltBits) == 0) {
43671       unsigned Scale = SrcOpVT.getSizeInBits() / SrcEltBits;
43672       unsigned Offset = IdxC.urem(Scale) * SrcEltBits;
43673       // TODO support non-zero offsets.
43674       if (Offset == 0) {
43675         SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, SrcVT.getScalarType());
43676         SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, VT);
43677         return SrcOp;
43678       }
43679     }
43680   }
43681 
43682   // If we're extracting a single element from a broadcast load and there are
43683   // no other users, just create a single load.
43684   if (SrcBC.getOpcode() == X86ISD::VBROADCAST_LOAD && SrcBC.hasOneUse()) {
43685     auto *MemIntr = cast<MemIntrinsicSDNode>(SrcBC);
43686     unsigned SrcBCWidth = SrcBC.getScalarValueSizeInBits();
43687     if (MemIntr->getMemoryVT().getSizeInBits() == SrcBCWidth &&
43688         VT.getSizeInBits() == SrcBCWidth && SrcEltBits == SrcBCWidth) {
43689       SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
43690                                  MemIntr->getBasePtr(),
43691                                  MemIntr->getPointerInfo(),
43692                                  MemIntr->getOriginalAlign(),
43693                                  MemIntr->getMemOperand()->getFlags());
43694       DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
43695       return Load;
43696     }
43697   }
43698 
43699   // Handle extract(bitcast(scalar_to_vector(scalar_value))) for integers.
43700   // TODO: Move to DAGCombine?
43701   if (SrcBC.getOpcode() == ISD::SCALAR_TO_VECTOR && VT.isInteger() &&
43702       SrcBC.getValueType().isInteger() &&
43703       (SrcBC.getScalarValueSizeInBits() % SrcEltBits) == 0 &&
43704       SrcBC.getScalarValueSizeInBits() ==
43705           SrcBC.getOperand(0).getValueSizeInBits()) {
43706     unsigned Scale = SrcBC.getScalarValueSizeInBits() / SrcEltBits;
43707     if (IdxC.ult(Scale)) {
43708       unsigned Offset = IdxC.getZExtValue() * SrcVT.getScalarSizeInBits();
43709       SDValue Scl = SrcBC.getOperand(0);
43710       EVT SclVT = Scl.getValueType();
43711       if (Offset) {
43712         Scl = DAG.getNode(ISD::SRL, dl, SclVT, Scl,
43713                           DAG.getShiftAmountConstant(Offset, SclVT, dl));
43714       }
43715       Scl = DAG.getZExtOrTrunc(Scl, dl, SrcVT.getScalarType());
43716       Scl = DAG.getZExtOrTrunc(Scl, dl, VT);
43717       return Scl;
43718     }
43719   }
43720 
43721   // Handle extract(truncate(x)) for 0'th index.
43722   // TODO: Treat this as a faux shuffle?
43723   // TODO: When can we use this for general indices?
43724   if (ISD::TRUNCATE == Src.getOpcode() && IdxC == 0 &&
43725       (SrcVT.getSizeInBits() % 128) == 0) {
43726     Src = extract128BitVector(Src.getOperand(0), 0, DAG, dl);
43727     MVT ExtractVT = MVT::getVectorVT(SrcSVT.getSimpleVT(), 128 / SrcEltBits);
43728     return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src),
43729                        Idx);
43730   }
43731 
43732   // We can only legally extract other elements from 128-bit vectors and in
43733   // certain circumstances, depending on SSE-level.
43734   // TODO: Investigate float/double extraction if it will be just stored.
43735   auto GetLegalExtract = [&Subtarget, &DAG, &dl](SDValue Vec, EVT VecVT,
43736                                                  unsigned Idx) {
43737     EVT VecSVT = VecVT.getScalarType();
43738     if ((VecVT.is256BitVector() || VecVT.is512BitVector()) &&
43739         (VecSVT == MVT::i8 || VecSVT == MVT::i16 || VecSVT == MVT::i32 ||
43740          VecSVT == MVT::i64)) {
43741       unsigned EltSizeInBits = VecSVT.getSizeInBits();
43742       unsigned NumEltsPerLane = 128 / EltSizeInBits;
43743       unsigned LaneOffset = (Idx & ~(NumEltsPerLane - 1)) * EltSizeInBits;
43744       unsigned LaneIdx = LaneOffset / Vec.getScalarValueSizeInBits();
43745       VecVT = EVT::getVectorVT(*DAG.getContext(), VecSVT, NumEltsPerLane);
43746       Vec = extract128BitVector(Vec, LaneIdx, DAG, dl);
43747       Idx &= (NumEltsPerLane - 1);
43748     }
43749     if ((VecVT == MVT::v4i32 || VecVT == MVT::v2i64) &&
43750         ((Idx == 0 && Subtarget.hasSSE2()) || Subtarget.hasSSE41())) {
43751       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VecVT.getScalarType(),
43752                          DAG.getBitcast(VecVT, Vec),
43753                          DAG.getIntPtrConstant(Idx, dl));
43754     }
43755     if ((VecVT == MVT::v8i16 && Subtarget.hasSSE2()) ||
43756         (VecVT == MVT::v16i8 && Subtarget.hasSSE41())) {
43757       unsigned OpCode = (VecVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB);
43758       return DAG.getNode(OpCode, dl, MVT::i32, DAG.getBitcast(VecVT, Vec),
43759                          DAG.getTargetConstant(Idx, dl, MVT::i8));
43760     }
43761     return SDValue();
43762   };
43763 
43764   // Resolve the target shuffle inputs and mask.
43765   SmallVector<int, 16> Mask;
43766   SmallVector<SDValue, 2> Ops;
43767   if (!getTargetShuffleInputs(SrcBC, Ops, Mask, DAG))
43768     return SDValue();
43769 
43770   // Shuffle inputs must be the same size as the result.
43771   if (llvm::any_of(Ops, [SrcVT](SDValue Op) {
43772         return SrcVT.getSizeInBits() != Op.getValueSizeInBits();
43773       }))
43774     return SDValue();
43775 
43776   // Attempt to narrow/widen the shuffle mask to the correct size.
43777   if (Mask.size() != NumSrcElts) {
43778     if ((NumSrcElts % Mask.size()) == 0) {
43779       SmallVector<int, 16> ScaledMask;
43780       int Scale = NumSrcElts / Mask.size();
43781       narrowShuffleMaskElts(Scale, Mask, ScaledMask);
43782       Mask = std::move(ScaledMask);
43783     } else if ((Mask.size() % NumSrcElts) == 0) {
43784       // Simplify Mask based on demanded element.
43785       int ExtractIdx = (int)IdxC.getZExtValue();
43786       int Scale = Mask.size() / NumSrcElts;
43787       int Lo = Scale * ExtractIdx;
43788       int Hi = Scale * (ExtractIdx + 1);
43789       for (int i = 0, e = (int)Mask.size(); i != e; ++i)
43790         if (i < Lo || Hi <= i)
43791           Mask[i] = SM_SentinelUndef;
43792 
43793       SmallVector<int, 16> WidenedMask;
43794       while (Mask.size() > NumSrcElts &&
43795              canWidenShuffleElements(Mask, WidenedMask))
43796         Mask = std::move(WidenedMask);
43797     }
43798   }
43799 
43800   // If narrowing/widening failed, see if we can extract+zero-extend.
43801   int ExtractIdx;
43802   EVT ExtractVT;
43803   if (Mask.size() == NumSrcElts) {
43804     ExtractIdx = Mask[IdxC.getZExtValue()];
43805     ExtractVT = SrcVT;
43806   } else {
43807     unsigned Scale = Mask.size() / NumSrcElts;
43808     if ((Mask.size() % NumSrcElts) != 0 || SrcVT.isFloatingPoint())
43809       return SDValue();
43810     unsigned ScaledIdx = Scale * IdxC.getZExtValue();
43811     if (!isUndefOrZeroInRange(Mask, ScaledIdx + 1, Scale - 1))
43812       return SDValue();
43813     ExtractIdx = Mask[ScaledIdx];
43814     EVT ExtractSVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltBits / Scale);
43815     ExtractVT = EVT::getVectorVT(*DAG.getContext(), ExtractSVT, Mask.size());
43816     assert(SrcVT.getSizeInBits() == ExtractVT.getSizeInBits() &&
43817            "Failed to widen vector type");
43818   }
43819 
43820   // If the shuffle source element is undef/zero then we can just accept it.
43821   if (ExtractIdx == SM_SentinelUndef)
43822     return DAG.getUNDEF(VT);
43823 
43824   if (ExtractIdx == SM_SentinelZero)
43825     return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT)
43826                                 : DAG.getConstant(0, dl, VT);
43827 
43828   SDValue SrcOp = Ops[ExtractIdx / Mask.size()];
43829   ExtractIdx = ExtractIdx % Mask.size();
43830   if (SDValue V = GetLegalExtract(SrcOp, ExtractVT, ExtractIdx))
43831     return DAG.getZExtOrTrunc(V, dl, VT);
43832 
43833   return SDValue();
43834 }
43835 
43836 /// Extracting a scalar FP value from vector element 0 is free, so extract each
43837 /// operand first, then perform the math as a scalar op.
43838 static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
43839                                  const X86Subtarget &Subtarget) {
43840   assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Expected extract");
43841   SDValue Vec = ExtElt->getOperand(0);
43842   SDValue Index = ExtElt->getOperand(1);
43843   EVT VT = ExtElt->getValueType(0);
43844   EVT VecVT = Vec.getValueType();
43845 
43846   // TODO: If this is a unary/expensive/expand op, allow extraction from a
43847   // non-zero element because the shuffle+scalar op will be cheaper?
43848   if (!Vec.hasOneUse() || !isNullConstant(Index) || VecVT.getScalarType() != VT)
43849     return SDValue();
43850 
43851   // Vector FP compares don't fit the pattern of FP math ops (propagate, not
43852   // extract, the condition code), so deal with those as a special-case.
43853   if (Vec.getOpcode() == ISD::SETCC && VT == MVT::i1) {
43854     EVT OpVT = Vec.getOperand(0).getValueType().getScalarType();
43855     if (OpVT != MVT::f32 && OpVT != MVT::f64)
43856       return SDValue();
43857 
43858     // extract (setcc X, Y, CC), 0 --> setcc (extract X, 0), (extract Y, 0), CC
43859     SDLoc DL(ExtElt);
43860     SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
43861                                Vec.getOperand(0), Index);
43862     SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
43863                                Vec.getOperand(1), Index);
43864     return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2));
43865   }
43866 
43867   if (!(VT == MVT::f16 && Subtarget.hasFP16()) && VT != MVT::f32 &&
43868       VT != MVT::f64)
43869     return SDValue();
43870 
43871   // Vector FP selects don't fit the pattern of FP math ops (because the
43872   // condition has a different type and we have to change the opcode), so deal
43873   // with those here.
43874   // FIXME: This is restricted to pre type legalization by ensuring the setcc
43875   // has i1 elements. If we loosen this we need to convert vector bool to a
43876   // scalar bool.
43877   if (Vec.getOpcode() == ISD::VSELECT &&
43878       Vec.getOperand(0).getOpcode() == ISD::SETCC &&
43879       Vec.getOperand(0).getValueType().getScalarType() == MVT::i1 &&
43880       Vec.getOperand(0).getOperand(0).getValueType() == VecVT) {
43881     // ext (sel Cond, X, Y), 0 --> sel (ext Cond, 0), (ext X, 0), (ext Y, 0)
43882     SDLoc DL(ExtElt);
43883     SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
43884                                Vec.getOperand(0).getValueType().getScalarType(),
43885                                Vec.getOperand(0), Index);
43886     SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
43887                                Vec.getOperand(1), Index);
43888     SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
43889                                Vec.getOperand(2), Index);
43890     return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2);
43891   }
43892 
43893   // TODO: This switch could include FNEG and the x86-specific FP logic ops
43894   // (FAND, FANDN, FOR, FXOR). But that may require enhancements to avoid
43895   // missed load folding and fma+fneg combining.
43896   switch (Vec.getOpcode()) {
43897   case ISD::FMA: // Begin 3 operands
43898   case ISD::FMAD:
43899   case ISD::FADD: // Begin 2 operands
43900   case ISD::FSUB:
43901   case ISD::FMUL:
43902   case ISD::FDIV:
43903   case ISD::FREM:
43904   case ISD::FCOPYSIGN:
43905   case ISD::FMINNUM:
43906   case ISD::FMAXNUM:
43907   case ISD::FMINNUM_IEEE:
43908   case ISD::FMAXNUM_IEEE:
43909   case ISD::FMAXIMUM:
43910   case ISD::FMINIMUM:
43911   case X86ISD::FMAX:
43912   case X86ISD::FMIN:
43913   case ISD::FABS: // Begin 1 operand
43914   case ISD::FSQRT:
43915   case ISD::FRINT:
43916   case ISD::FCEIL:
43917   case ISD::FTRUNC:
43918   case ISD::FNEARBYINT:
43919   case ISD::FROUNDEVEN:
43920   case ISD::FROUND:
43921   case ISD::FFLOOR:
43922   case X86ISD::FRCP:
43923   case X86ISD::FRSQRT: {
43924     // extract (fp X, Y, ...), 0 --> fp (extract X, 0), (extract Y, 0), ...
43925     SDLoc DL(ExtElt);
43926     SmallVector<SDValue, 4> ExtOps;
43927     for (SDValue Op : Vec->ops())
43928       ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index));
43929     return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps);
43930   }
43931   default:
43932     return SDValue();
43933   }
43934   llvm_unreachable("All opcodes should return within switch");
43935 }
43936 
43937 /// Try to convert a vector reduction sequence composed of binops and shuffles
43938 /// into horizontal ops.
43939 static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
43940                                      const X86Subtarget &Subtarget) {
43941   assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unexpected caller");
43942 
43943   // We need at least SSE2 to anything here.
43944   if (!Subtarget.hasSSE2())
43945     return SDValue();
43946 
43947   ISD::NodeType Opc;
43948   SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
43949                                         {ISD::ADD, ISD::MUL, ISD::FADD}, true);
43950   if (!Rdx)
43951     return SDValue();
43952 
43953   SDValue Index = ExtElt->getOperand(1);
43954   assert(isNullConstant(Index) &&
43955          "Reduction doesn't end in an extract from index 0");
43956 
43957   EVT VT = ExtElt->getValueType(0);
43958   EVT VecVT = Rdx.getValueType();
43959   if (VecVT.getScalarType() != VT)
43960     return SDValue();
43961 
43962   SDLoc DL(ExtElt);
43963   unsigned NumElts = VecVT.getVectorNumElements();
43964   unsigned EltSizeInBits = VecVT.getScalarSizeInBits();
43965 
43966   // Extend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
43967   auto WidenToV16I8 = [&](SDValue V, bool ZeroExtend) {
43968     if (V.getValueType() == MVT::v4i8) {
43969       if (ZeroExtend && Subtarget.hasSSE41()) {
43970         V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
43971                         DAG.getConstant(0, DL, MVT::v4i32),
43972                         DAG.getBitcast(MVT::i32, V),
43973                         DAG.getIntPtrConstant(0, DL));
43974         return DAG.getBitcast(MVT::v16i8, V);
43975       }
43976       V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V,
43977                       ZeroExtend ? DAG.getConstant(0, DL, MVT::v4i8)
43978                                  : DAG.getUNDEF(MVT::v4i8));
43979     }
43980     return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V,
43981                        DAG.getUNDEF(MVT::v8i8));
43982   };
43983 
43984   // vXi8 mul reduction - promote to vXi16 mul reduction.
43985   if (Opc == ISD::MUL) {
43986     if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
43987       return SDValue();
43988     if (VecVT.getSizeInBits() >= 128) {
43989       EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
43990       SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
43991       SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
43992       Lo = DAG.getBitcast(WideVT, Lo);
43993       Hi = DAG.getBitcast(WideVT, Hi);
43994       Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
43995       while (Rdx.getValueSizeInBits() > 128) {
43996         std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
43997         Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
43998       }
43999     } else {
44000       Rdx = WidenToV16I8(Rdx, false);
44001       Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
44002       Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
44003     }
44004     if (NumElts >= 8)
44005       Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
44006                         DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
44007                                              {4, 5, 6, 7, -1, -1, -1, -1}));
44008     Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
44009                       DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
44010                                            {2, 3, -1, -1, -1, -1, -1, -1}));
44011     Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
44012                       DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
44013                                            {1, -1, -1, -1, -1, -1, -1, -1}));
44014     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
44015     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44016   }
44017 
44018   // vXi8 add reduction - sub 128-bit vector.
44019   if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
44020     Rdx = WidenToV16I8(Rdx, true);
44021     Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
44022                       DAG.getConstant(0, DL, MVT::v16i8));
44023     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
44024     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44025   }
44026 
44027   // Must be a >=128-bit vector with pow2 elements.
44028   if ((VecVT.getSizeInBits() % 128) != 0 || !isPowerOf2_32(NumElts))
44029     return SDValue();
44030 
44031   // vXi8 add reduction - sum lo/hi halves then use PSADBW.
44032   if (VT == MVT::i8) {
44033     while (Rdx.getValueSizeInBits() > 128) {
44034       SDValue Lo, Hi;
44035       std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
44036       VecVT = Lo.getValueType();
44037       Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
44038     }
44039     assert(VecVT == MVT::v16i8 && "v16i8 reduction expected");
44040 
44041     SDValue Hi = DAG.getVectorShuffle(
44042         MVT::v16i8, DL, Rdx, Rdx,
44043         {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
44044     Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);
44045     Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
44046                       getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
44047     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
44048     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44049   }
44050 
44051   // See if we can use vXi8 PSADBW add reduction for larger zext types.
44052   // If the source vector values are 0-255, then we can use PSADBW to
44053   // sum+zext v8i8 subvectors to vXi64, then perform the reduction.
44054   // TODO: See if its worth avoiding vXi16/i32 truncations?
44055   if (Opc == ISD::ADD && NumElts >= 4 && EltSizeInBits >= 16 &&
44056       DAG.computeKnownBits(Rdx).getMaxValue().ule(255) &&
44057       (EltSizeInBits == 16 || Rdx.getOpcode() == ISD::ZERO_EXTEND ||
44058        Subtarget.hasAVX512())) {
44059     if (Rdx.getValueType() == MVT::v8i16) {
44060       Rdx = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Rdx,
44061                         DAG.getUNDEF(MVT::v8i16));
44062     } else {
44063       EVT ByteVT = VecVT.changeVectorElementType(MVT::i8);
44064       Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx);
44065       if (ByteVT.getSizeInBits() < 128)
44066         Rdx = WidenToV16I8(Rdx, true);
44067     }
44068 
44069     // Build the PSADBW, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
44070     auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
44071                             ArrayRef<SDValue> Ops) {
44072       MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
44073       SDValue Zero = DAG.getConstant(0, DL, Ops[0].getValueType());
44074       return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero);
44075     };
44076     MVT SadVT = MVT::getVectorVT(MVT::i64, Rdx.getValueSizeInBits() / 64);
44077     Rdx = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Rdx}, PSADBWBuilder);
44078 
44079     // TODO: We could truncate to vXi16/vXi32 before performing the reduction.
44080     while (Rdx.getValueSizeInBits() > 128) {
44081       SDValue Lo, Hi;
44082       std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
44083       VecVT = Lo.getValueType();
44084       Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
44085     }
44086     assert(Rdx.getValueType() == MVT::v2i64 && "v2i64 reduction expected");
44087 
44088     if (NumElts > 8) {
44089       SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1});
44090       Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi);
44091     }
44092 
44093     VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits());
44094     Rdx = DAG.getBitcast(VecVT, Rdx);
44095     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44096   }
44097 
44098   // Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
44099   if (!shouldUseHorizontalOp(true, DAG, Subtarget))
44100     return SDValue();
44101 
44102   unsigned HorizOpcode = Opc == ISD::ADD ? X86ISD::HADD : X86ISD::FHADD;
44103 
44104   // 256-bit horizontal instructions operate on 128-bit chunks rather than
44105   // across the whole vector, so we need an extract + hop preliminary stage.
44106   // This is the only step where the operands of the hop are not the same value.
44107   // TODO: We could extend this to handle 512-bit or even longer vectors.
44108   if (((VecVT == MVT::v16i16 || VecVT == MVT::v8i32) && Subtarget.hasSSSE3()) ||
44109       ((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) {
44110     unsigned NumElts = VecVT.getVectorNumElements();
44111     SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
44112     SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
44113     Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo);
44114     VecVT = Rdx.getValueType();
44115   }
44116   if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) &&
44117       !((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))
44118     return SDValue();
44119 
44120   // extract (add (shuf X), X), 0 --> extract (hadd X, X), 0
44121   unsigned ReductionSteps = Log2_32(VecVT.getVectorNumElements());
44122   for (unsigned i = 0; i != ReductionSteps; ++i)
44123     Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx);
44124 
44125   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
44126 }
44127 
44128 /// Detect vector gather/scatter index generation and convert it from being a
44129 /// bunch of shuffles and extracts into a somewhat faster sequence.
44130 /// For i686, the best sequence is apparently storing the value and loading
44131 /// scalars back, while for x64 we should use 64-bit extracts and shifts.
44132 static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
44133                                        TargetLowering::DAGCombinerInfo &DCI,
44134                                        const X86Subtarget &Subtarget) {
44135   if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
44136     return NewOp;
44137 
44138   SDValue InputVector = N->getOperand(0);
44139   SDValue EltIdx = N->getOperand(1);
44140   auto *CIdx = dyn_cast<ConstantSDNode>(EltIdx);
44141 
44142   EVT SrcVT = InputVector.getValueType();
44143   EVT VT = N->getValueType(0);
44144   SDLoc dl(InputVector);
44145   bool IsPextr = N->getOpcode() != ISD::EXTRACT_VECTOR_ELT;
44146   unsigned NumSrcElts = SrcVT.getVectorNumElements();
44147   unsigned NumEltBits = VT.getScalarSizeInBits();
44148   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44149 
44150   if (CIdx && CIdx->getAPIntValue().uge(NumSrcElts))
44151     return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
44152 
44153   // Integer Constant Folding.
44154   if (CIdx && VT.isInteger()) {
44155     APInt UndefVecElts;
44156     SmallVector<APInt, 16> EltBits;
44157     unsigned VecEltBitWidth = SrcVT.getScalarSizeInBits();
44158     if (getTargetConstantBitsFromNode(InputVector, VecEltBitWidth, UndefVecElts,
44159                                       EltBits, true, false)) {
44160       uint64_t Idx = CIdx->getZExtValue();
44161       if (UndefVecElts[Idx])
44162         return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
44163       return DAG.getConstant(EltBits[Idx].zext(NumEltBits), dl, VT);
44164     }
44165 
44166     // Convert extract_element(bitcast(<X x i1>) -> bitcast(extract_subvector()).
44167     // Improves lowering of bool masks on rust which splits them into byte array.
44168     if (InputVector.getOpcode() == ISD::BITCAST && (NumEltBits % 8) == 0) {
44169       SDValue Src = peekThroughBitcasts(InputVector);
44170       if (Src.getValueType().getScalarType() == MVT::i1 &&
44171           TLI.isTypeLegal(Src.getValueType())) {
44172         MVT SubVT = MVT::getVectorVT(MVT::i1, NumEltBits);
44173         SDValue Sub = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Src,
44174             DAG.getIntPtrConstant(CIdx->getZExtValue() * NumEltBits, dl));
44175         return DAG.getBitcast(VT, Sub);
44176       }
44177     }
44178   }
44179 
44180   if (IsPextr) {
44181     if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumEltBits),
44182                                  DCI))
44183       return SDValue(N, 0);
44184 
44185     // PEXTR*(PINSR*(v, s, c), c) -> s (with implicit zext handling).
44186     if ((InputVector.getOpcode() == X86ISD::PINSRB ||
44187          InputVector.getOpcode() == X86ISD::PINSRW) &&
44188         InputVector.getOperand(2) == EltIdx) {
44189       assert(SrcVT == InputVector.getOperand(0).getValueType() &&
44190              "Vector type mismatch");
44191       SDValue Scl = InputVector.getOperand(1);
44192       Scl = DAG.getNode(ISD::TRUNCATE, dl, SrcVT.getScalarType(), Scl);
44193       return DAG.getZExtOrTrunc(Scl, dl, VT);
44194     }
44195 
44196     // TODO - Remove this once we can handle the implicit zero-extension of
44197     // X86ISD::PEXTRW/X86ISD::PEXTRB in combinePredicateReduction and
44198     // combineBasicSADPattern.
44199     return SDValue();
44200   }
44201 
44202   // Detect mmx extraction of all bits as a i64. It works better as a bitcast.
44203   if (VT == MVT::i64 && SrcVT == MVT::v1i64 &&
44204       InputVector.getOpcode() == ISD::BITCAST &&
44205       InputVector.getOperand(0).getValueType() == MVT::x86mmx &&
44206       isNullConstant(EltIdx) && InputVector.hasOneUse())
44207     return DAG.getBitcast(VT, InputVector);
44208 
44209   // Detect mmx to i32 conversion through a v2i32 elt extract.
44210   if (VT == MVT::i32 && SrcVT == MVT::v2i32 &&
44211       InputVector.getOpcode() == ISD::BITCAST &&
44212       InputVector.getOperand(0).getValueType() == MVT::x86mmx &&
44213       isNullConstant(EltIdx) && InputVector.hasOneUse())
44214     return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32,
44215                        InputVector.getOperand(0));
44216 
44217   // Check whether this extract is the root of a sum of absolute differences
44218   // pattern. This has to be done here because we really want it to happen
44219   // pre-legalization,
44220   if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
44221     return SAD;
44222 
44223   if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget))
44224     return VPDPBUSD;
44225 
44226   // Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK.
44227   if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
44228     return Cmp;
44229 
44230   // Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW.
44231   if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
44232     return MinMax;
44233 
44234   // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
44235   if (SDValue V = combineArithReduction(N, DAG, Subtarget))
44236     return V;
44237 
44238   if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget))
44239     return V;
44240 
44241   // Attempt to extract a i1 element by using MOVMSK to extract the signbits
44242   // and then testing the relevant element.
44243   //
44244   // Note that we only combine extracts on the *same* result number, i.e.
44245   //   t0 = merge_values a0, a1, a2, a3
44246   //   i1 = extract_vector_elt t0, Constant:i64<2>
44247   //   i1 = extract_vector_elt t0, Constant:i64<3>
44248   // but not
44249   //   i1 = extract_vector_elt t0:1, Constant:i64<2>
44250   // since the latter would need its own MOVMSK.
44251   if (SrcVT.getScalarType() == MVT::i1) {
44252     bool IsVar = !CIdx;
44253     SmallVector<SDNode *, 16> BoolExtracts;
44254     unsigned ResNo = InputVector.getResNo();
44255     auto IsBoolExtract = [&BoolExtracts, &ResNo, &IsVar](SDNode *Use) {
44256       if (Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
44257           Use->getOperand(0).getResNo() == ResNo &&
44258           Use->getValueType(0) == MVT::i1) {
44259         BoolExtracts.push_back(Use);
44260         IsVar |= !isa<ConstantSDNode>(Use->getOperand(1));
44261         return true;
44262       }
44263       return false;
44264     };
44265     // TODO: Can we drop the oneuse check for constant extracts?
44266     if (all_of(InputVector->uses(), IsBoolExtract) &&
44267         (IsVar || BoolExtracts.size() > 1)) {
44268       EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcElts);
44269       if (SDValue BC =
44270               combineBitcastvxi1(DAG, BCVT, InputVector, dl, Subtarget)) {
44271         for (SDNode *Use : BoolExtracts) {
44272           // extractelement vXi1 X, MaskIdx --> ((movmsk X) & Mask) == Mask
44273           // Mask = 1 << MaskIdx
44274           SDValue MaskIdx = DAG.getZExtOrTrunc(Use->getOperand(1), dl, MVT::i8);
44275           SDValue MaskBit = DAG.getConstant(1, dl, BCVT);
44276           SDValue Mask = DAG.getNode(ISD::SHL, dl, BCVT, MaskBit, MaskIdx);
44277           SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask);
44278           Res = DAG.getSetCC(dl, MVT::i1, Res, Mask, ISD::SETEQ);
44279           DCI.CombineTo(Use, Res);
44280         }
44281         return SDValue(N, 0);
44282       }
44283     }
44284   }
44285 
44286   // If this extract is from a loaded vector value and will be used as an
44287   // integer, that requires a potentially expensive XMM -> GPR transfer.
44288   // Additionally, if we can convert to a scalar integer load, that will likely
44289   // be folded into a subsequent integer op.
44290   // Note: Unlike the related fold for this in DAGCombiner, this is not limited
44291   //       to a single-use of the loaded vector. For the reasons above, we
44292   //       expect this to be profitable even if it creates an extra load.
44293   bool LikelyUsedAsVector = any_of(N->uses(), [](SDNode *Use) {
44294     return Use->getOpcode() == ISD::STORE ||
44295            Use->getOpcode() == ISD::INSERT_VECTOR_ELT ||
44296            Use->getOpcode() == ISD::SCALAR_TO_VECTOR;
44297   });
44298   auto *LoadVec = dyn_cast<LoadSDNode>(InputVector);
44299   if (LoadVec && CIdx && ISD::isNormalLoad(LoadVec) && VT.isInteger() &&
44300       SrcVT.getVectorElementType() == VT && DCI.isAfterLegalizeDAG() &&
44301       !LikelyUsedAsVector && LoadVec->isSimple()) {
44302     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44303     SDValue NewPtr =
44304         TLI.getVectorElementPointer(DAG, LoadVec->getBasePtr(), SrcVT, EltIdx);
44305     unsigned PtrOff = VT.getSizeInBits() * CIdx->getZExtValue() / 8;
44306     MachinePointerInfo MPI = LoadVec->getPointerInfo().getWithOffset(PtrOff);
44307     Align Alignment = commonAlignment(LoadVec->getAlign(), PtrOff);
44308     SDValue Load =
44309         DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment,
44310                     LoadVec->getMemOperand()->getFlags(), LoadVec->getAAInfo());
44311     DAG.makeEquivalentMemoryOrdering(LoadVec, Load);
44312     return Load;
44313   }
44314 
44315   return SDValue();
44316 }
44317 
44318 // Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
44319 // This is more or less the reverse of combineBitcastvxi1.
44320 static SDValue combineToExtendBoolVectorInReg(
44321     unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG,
44322     TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) {
44323   if (Opcode != ISD::SIGN_EXTEND && Opcode != ISD::ZERO_EXTEND &&
44324       Opcode != ISD::ANY_EXTEND)
44325     return SDValue();
44326   if (!DCI.isBeforeLegalizeOps())
44327     return SDValue();
44328   if (!Subtarget.hasSSE2() || Subtarget.hasAVX512())
44329     return SDValue();
44330 
44331   EVT SVT = VT.getScalarType();
44332   EVT InSVT = N0.getValueType().getScalarType();
44333   unsigned EltSizeInBits = SVT.getSizeInBits();
44334 
44335   // Input type must be extending a bool vector (bit-casted from a scalar
44336   // integer) to legal integer types.
44337   if (!VT.isVector())
44338     return SDValue();
44339   if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16 && SVT != MVT::i8)
44340     return SDValue();
44341   if (InSVT != MVT::i1 || N0.getOpcode() != ISD::BITCAST)
44342     return SDValue();
44343 
44344   SDValue N00 = N0.getOperand(0);
44345   EVT SclVT = N00.getValueType();
44346   if (!SclVT.isScalarInteger())
44347     return SDValue();
44348 
44349   SDValue Vec;
44350   SmallVector<int> ShuffleMask;
44351   unsigned NumElts = VT.getVectorNumElements();
44352   assert(NumElts == SclVT.getSizeInBits() && "Unexpected bool vector size");
44353 
44354   // Broadcast the scalar integer to the vector elements.
44355   if (NumElts > EltSizeInBits) {
44356     // If the scalar integer is greater than the vector element size, then we
44357     // must split it down into sub-sections for broadcasting. For example:
44358     //   i16 -> v16i8 (i16 -> v8i16 -> v16i8) with 2 sub-sections.
44359     //   i32 -> v32i8 (i32 -> v8i32 -> v32i8) with 4 sub-sections.
44360     assert((NumElts % EltSizeInBits) == 0 && "Unexpected integer scale");
44361     unsigned Scale = NumElts / EltSizeInBits;
44362     EVT BroadcastVT = EVT::getVectorVT(*DAG.getContext(), SclVT, EltSizeInBits);
44363     Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
44364     Vec = DAG.getBitcast(VT, Vec);
44365 
44366     for (unsigned i = 0; i != Scale; ++i)
44367       ShuffleMask.append(EltSizeInBits, i);
44368     Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
44369   } else if (Subtarget.hasAVX2() && NumElts < EltSizeInBits &&
44370              (SclVT == MVT::i8 || SclVT == MVT::i16 || SclVT == MVT::i32)) {
44371     // If we have register broadcast instructions, use the scalar size as the
44372     // element type for the shuffle. Then cast to the wider element type. The
44373     // widened bits won't be used, and this might allow the use of a broadcast
44374     // load.
44375     assert((EltSizeInBits % NumElts) == 0 && "Unexpected integer scale");
44376     unsigned Scale = EltSizeInBits / NumElts;
44377     EVT BroadcastVT =
44378         EVT::getVectorVT(*DAG.getContext(), SclVT, NumElts * Scale);
44379     Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
44380     ShuffleMask.append(NumElts * Scale, 0);
44381     Vec = DAG.getVectorShuffle(BroadcastVT, DL, Vec, Vec, ShuffleMask);
44382     Vec = DAG.getBitcast(VT, Vec);
44383   } else {
44384     // For smaller scalar integers, we can simply any-extend it to the vector
44385     // element size (we don't care about the upper bits) and broadcast it to all
44386     // elements.
44387     SDValue Scl = DAG.getAnyExtOrTrunc(N00, DL, SVT);
44388     Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
44389     ShuffleMask.append(NumElts, 0);
44390     Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
44391   }
44392 
44393   // Now, mask the relevant bit in each element.
44394   SmallVector<SDValue, 32> Bits;
44395   for (unsigned i = 0; i != NumElts; ++i) {
44396     int BitIdx = (i % EltSizeInBits);
44397     APInt Bit = APInt::getBitsSet(EltSizeInBits, BitIdx, BitIdx + 1);
44398     Bits.push_back(DAG.getConstant(Bit, DL, SVT));
44399   }
44400   SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
44401   Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask);
44402 
44403   // Compare against the bitmask and extend the result.
44404   EVT CCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
44405   Vec = DAG.getSetCC(DL, CCVT, Vec, BitMask, ISD::SETEQ);
44406   Vec = DAG.getSExtOrTrunc(Vec, DL, VT);
44407 
44408   // For SEXT, this is now done, otherwise shift the result down for
44409   // zero-extension.
44410   if (Opcode == ISD::SIGN_EXTEND)
44411     return Vec;
44412   return DAG.getNode(ISD::SRL, DL, VT, Vec,
44413                      DAG.getConstant(EltSizeInBits - 1, DL, VT));
44414 }
44415 
44416 /// If a vector select has an operand that is -1 or 0, try to simplify the
44417 /// select to a bitwise logic operation.
44418 /// TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?
44419 static SDValue
44420 combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG,
44421                                  TargetLowering::DAGCombinerInfo &DCI,
44422                                  const X86Subtarget &Subtarget) {
44423   SDValue Cond = N->getOperand(0);
44424   SDValue LHS = N->getOperand(1);
44425   SDValue RHS = N->getOperand(2);
44426   EVT VT = LHS.getValueType();
44427   EVT CondVT = Cond.getValueType();
44428   SDLoc DL(N);
44429   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44430 
44431   if (N->getOpcode() != ISD::VSELECT)
44432     return SDValue();
44433 
44434   assert(CondVT.isVector() && "Vector select expects a vector selector!");
44435 
44436   // TODO: Use isNullOrNullSplat() to distinguish constants with undefs?
44437   // TODO: Can we assert that both operands are not zeros (because that should
44438   //       get simplified at node creation time)?
44439   bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
44440   bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
44441 
44442   // If both inputs are 0/undef, create a complete zero vector.
44443   // FIXME: As noted above this should be handled by DAGCombiner/getNode.
44444   if (TValIsAllZeros && FValIsAllZeros) {
44445     if (VT.isFloatingPoint())
44446       return DAG.getConstantFP(0.0, DL, VT);
44447     return DAG.getConstant(0, DL, VT);
44448   }
44449 
44450   // To use the condition operand as a bitwise mask, it must have elements that
44451   // are the same size as the select elements. Ie, the condition operand must
44452   // have already been promoted from the IR select condition type <N x i1>.
44453   // Don't check if the types themselves are equal because that excludes
44454   // vector floating-point selects.
44455   if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
44456     return SDValue();
44457 
44458   // Try to invert the condition if true value is not all 1s and false value is
44459   // not all 0s. Only do this if the condition has one use.
44460   bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
44461   if (!TValIsAllOnes && !FValIsAllZeros && Cond.hasOneUse() &&
44462       // Check if the selector will be produced by CMPP*/PCMP*.
44463       Cond.getOpcode() == ISD::SETCC &&
44464       // Check if SETCC has already been promoted.
44465       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
44466           CondVT) {
44467     bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
44468 
44469     if (TValIsAllZeros || FValIsAllOnes) {
44470       SDValue CC = Cond.getOperand(2);
44471       ISD::CondCode NewCC = ISD::getSetCCInverse(
44472           cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
44473       Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
44474                           NewCC);
44475       std::swap(LHS, RHS);
44476       TValIsAllOnes = FValIsAllOnes;
44477       FValIsAllZeros = TValIsAllZeros;
44478     }
44479   }
44480 
44481   // Cond value must be 'sign splat' to be converted to a logical op.
44482   if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
44483     return SDValue();
44484 
44485   // vselect Cond, 111..., 000... -> Cond
44486   if (TValIsAllOnes && FValIsAllZeros)
44487     return DAG.getBitcast(VT, Cond);
44488 
44489   if (!TLI.isTypeLegal(CondVT))
44490     return SDValue();
44491 
44492   // vselect Cond, 111..., X -> or Cond, X
44493   if (TValIsAllOnes) {
44494     SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
44495     SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
44496     return DAG.getBitcast(VT, Or);
44497   }
44498 
44499   // vselect Cond, X, 000... -> and Cond, X
44500   if (FValIsAllZeros) {
44501     SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
44502     SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
44503     return DAG.getBitcast(VT, And);
44504   }
44505 
44506   // vselect Cond, 000..., X -> andn Cond, X
44507   if (TValIsAllZeros) {
44508     SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
44509     SDValue AndN;
44510     // The canonical form differs for i1 vectors - x86andnp is not used
44511     if (CondVT.getScalarType() == MVT::i1)
44512       AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT),
44513                          CastRHS);
44514     else
44515       AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS);
44516     return DAG.getBitcast(VT, AndN);
44517   }
44518 
44519   return SDValue();
44520 }
44521 
44522 /// If both arms of a vector select are concatenated vectors, split the select,
44523 /// and concatenate the result to eliminate a wide (256-bit) vector instruction:
44524 ///   vselect Cond, (concat T0, T1), (concat F0, F1) -->
44525 ///   concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)
44526 static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG,
44527                                   const X86Subtarget &Subtarget) {
44528   unsigned Opcode = N->getOpcode();
44529   if (Opcode != X86ISD::BLENDV && Opcode != ISD::VSELECT)
44530     return SDValue();
44531 
44532   // TODO: Split 512-bit vectors too?
44533   EVT VT = N->getValueType(0);
44534   if (!VT.is256BitVector())
44535     return SDValue();
44536 
44537   // TODO: Split as long as any 2 of the 3 operands are concatenated?
44538   SDValue Cond = N->getOperand(0);
44539   SDValue TVal = N->getOperand(1);
44540   SDValue FVal = N->getOperand(2);
44541   if (!TVal.hasOneUse() || !FVal.hasOneUse() ||
44542       !isFreeToSplitVector(TVal.getNode(), DAG) ||
44543       !isFreeToSplitVector(FVal.getNode(), DAG))
44544     return SDValue();
44545 
44546   auto makeBlend = [Opcode](SelectionDAG &DAG, const SDLoc &DL,
44547                             ArrayRef<SDValue> Ops) {
44548     return DAG.getNode(Opcode, DL, Ops[1].getValueType(), Ops);
44549   };
44550   return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { Cond, TVal, FVal },
44551                           makeBlend, /*CheckBWI*/ false);
44552 }
44553 
44554 static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG) {
44555   SDValue Cond = N->getOperand(0);
44556   SDValue LHS = N->getOperand(1);
44557   SDValue RHS = N->getOperand(2);
44558   SDLoc DL(N);
44559 
44560   auto *TrueC = dyn_cast<ConstantSDNode>(LHS);
44561   auto *FalseC = dyn_cast<ConstantSDNode>(RHS);
44562   if (!TrueC || !FalseC)
44563     return SDValue();
44564 
44565   // Don't do this for crazy integer types.
44566   EVT VT = N->getValueType(0);
44567   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
44568     return SDValue();
44569 
44570   // We're going to use the condition bit in math or logic ops. We could allow
44571   // this with a wider condition value (post-legalization it becomes an i8),
44572   // but if nothing is creating selects that late, it doesn't matter.
44573   if (Cond.getValueType() != MVT::i1)
44574     return SDValue();
44575 
44576   // A power-of-2 multiply is just a shift. LEA also cheaply handles multiply by
44577   // 3, 5, or 9 with i32/i64, so those get transformed too.
44578   // TODO: For constants that overflow or do not differ by power-of-2 or small
44579   // multiplier, convert to 'and' + 'add'.
44580   const APInt &TrueVal = TrueC->getAPIntValue();
44581   const APInt &FalseVal = FalseC->getAPIntValue();
44582 
44583   // We have a more efficient lowering for "(X == 0) ? Y : -1" using SBB.
44584   if ((TrueVal.isAllOnes() || FalseVal.isAllOnes()) &&
44585       Cond.getOpcode() == ISD::SETCC && isNullConstant(Cond.getOperand(1))) {
44586     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
44587     if (CC == ISD::SETEQ || CC == ISD::SETNE)
44588       return SDValue();
44589   }
44590 
44591   bool OV;
44592   APInt Diff = TrueVal.ssub_ov(FalseVal, OV);
44593   if (OV)
44594     return SDValue();
44595 
44596   APInt AbsDiff = Diff.abs();
44597   if (AbsDiff.isPowerOf2() ||
44598       ((VT == MVT::i32 || VT == MVT::i64) &&
44599        (AbsDiff == 3 || AbsDiff == 5 || AbsDiff == 9))) {
44600 
44601     // We need a positive multiplier constant for shift/LEA codegen. The 'not'
44602     // of the condition can usually be folded into a compare predicate, but even
44603     // without that, the sequence should be cheaper than a CMOV alternative.
44604     if (TrueVal.slt(FalseVal)) {
44605       Cond = DAG.getNOT(DL, Cond, MVT::i1);
44606       std::swap(TrueC, FalseC);
44607     }
44608 
44609     // select Cond, TC, FC --> (zext(Cond) * (TC - FC)) + FC
44610     SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
44611 
44612     // Multiply condition by the difference if non-one.
44613     if (!AbsDiff.isOne())
44614       R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT));
44615 
44616     // Add the base if non-zero.
44617     if (!FalseC->isZero())
44618       R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
44619 
44620     return R;
44621   }
44622 
44623   return SDValue();
44624 }
44625 
44626 /// If this is a *dynamic* select (non-constant condition) and we can match
44627 /// this node with one of the variable blend instructions, restructure the
44628 /// condition so that blends can use the high (sign) bit of each element.
44629 /// This function will also call SimplifyDemandedBits on already created
44630 /// BLENDV to perform additional simplifications.
44631 static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
44632                                       TargetLowering::DAGCombinerInfo &DCI,
44633                                       const X86Subtarget &Subtarget) {
44634   SDValue Cond = N->getOperand(0);
44635   if ((N->getOpcode() != ISD::VSELECT &&
44636        N->getOpcode() != X86ISD::BLENDV) ||
44637       ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
44638     return SDValue();
44639 
44640   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44641   unsigned BitWidth = Cond.getScalarValueSizeInBits();
44642   EVT VT = N->getValueType(0);
44643 
44644   // We can only handle the cases where VSELECT is directly legal on the
44645   // subtarget. We custom lower VSELECT nodes with constant conditions and
44646   // this makes it hard to see whether a dynamic VSELECT will correctly
44647   // lower, so we both check the operation's status and explicitly handle the
44648   // cases where a *dynamic* blend will fail even though a constant-condition
44649   // blend could be custom lowered.
44650   // FIXME: We should find a better way to handle this class of problems.
44651   // Potentially, we should combine constant-condition vselect nodes
44652   // pre-legalization into shuffles and not mark as many types as custom
44653   // lowered.
44654   if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
44655     return SDValue();
44656   // FIXME: We don't support i16-element blends currently. We could and
44657   // should support them by making *all* the bits in the condition be set
44658   // rather than just the high bit and using an i8-element blend.
44659   if (VT.getVectorElementType() == MVT::i16)
44660     return SDValue();
44661   // Dynamic blending was only available from SSE4.1 onward.
44662   if (VT.is128BitVector() && !Subtarget.hasSSE41())
44663     return SDValue();
44664   // Byte blends are only available in AVX2
44665   if (VT == MVT::v32i8 && !Subtarget.hasAVX2())
44666     return SDValue();
44667   // There are no 512-bit blend instructions that use sign bits.
44668   if (VT.is512BitVector())
44669     return SDValue();
44670 
44671   // Don't optimize before the condition has been transformed to a legal type
44672   // and don't ever optimize vector selects that map to AVX512 mask-registers.
44673   if (BitWidth < 8 || BitWidth > 64)
44674     return SDValue();
44675 
44676   auto OnlyUsedAsSelectCond = [](SDValue Cond) {
44677     for (SDNode::use_iterator UI = Cond->use_begin(), UE = Cond->use_end();
44678          UI != UE; ++UI)
44679       if ((UI->getOpcode() != ISD::VSELECT &&
44680            UI->getOpcode() != X86ISD::BLENDV) ||
44681           UI.getOperandNo() != 0)
44682         return false;
44683 
44684     return true;
44685   };
44686 
44687   APInt DemandedBits(APInt::getSignMask(BitWidth));
44688 
44689   if (OnlyUsedAsSelectCond(Cond)) {
44690     KnownBits Known;
44691     TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
44692                                           !DCI.isBeforeLegalizeOps());
44693     if (!TLI.SimplifyDemandedBits(Cond, DemandedBits, Known, TLO, 0, true))
44694       return SDValue();
44695 
44696     // If we changed the computation somewhere in the DAG, this change will
44697     // affect all users of Cond. Update all the nodes so that we do not use
44698     // the generic VSELECT anymore. Otherwise, we may perform wrong
44699     // optimizations as we messed with the actual expectation for the vector
44700     // boolean values.
44701     for (SDNode *U : Cond->uses()) {
44702       if (U->getOpcode() == X86ISD::BLENDV)
44703         continue;
44704 
44705       SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
44706                                Cond, U->getOperand(1), U->getOperand(2));
44707       DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB);
44708       DCI.AddToWorklist(U);
44709     }
44710     DCI.CommitTargetLoweringOpt(TLO);
44711     return SDValue(N, 0);
44712   }
44713 
44714   // Otherwise we can still at least try to simplify multiple use bits.
44715   if (SDValue V = TLI.SimplifyMultipleUseDemandedBits(Cond, DemandedBits, DAG))
44716       return DAG.getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0), V,
44717                          N->getOperand(1), N->getOperand(2));
44718 
44719   return SDValue();
44720 }
44721 
44722 // Try to match:
44723 //   (or (and (M, (sub 0, X)), (pandn M, X)))
44724 // which is a special case of:
44725 //   (select M, (sub 0, X), X)
44726 // Per:
44727 // http://graphics.stanford.edu/~seander/bithacks.html#ConditionalNegate
44728 // We know that, if fNegate is 0 or 1:
44729 //   (fNegate ? -v : v) == ((v ^ -fNegate) + fNegate)
44730 //
44731 // Here, we have a mask, M (all 1s or 0), and, similarly, we know that:
44732 //   ((M & 1) ? -X : X) == ((X ^ -(M & 1)) + (M & 1))
44733 //   ( M      ? -X : X) == ((X ^   M     ) + (M & 1))
44734 // This lets us transform our vselect to:
44735 //   (add (xor X, M), (and M, 1))
44736 // And further to:
44737 //   (sub (xor X, M), M)
44738 static SDValue combineLogicBlendIntoConditionalNegate(
44739     EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
44740     SelectionDAG &DAG, const X86Subtarget &Subtarget) {
44741   EVT MaskVT = Mask.getValueType();
44742   assert(MaskVT.isInteger() &&
44743          DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() &&
44744          "Mask must be zero/all-bits");
44745 
44746   if (X.getValueType() != MaskVT || Y.getValueType() != MaskVT)
44747     return SDValue();
44748   if (!DAG.getTargetLoweringInfo().isOperationLegal(ISD::SUB, MaskVT))
44749     return SDValue();
44750 
44751   auto IsNegV = [](SDNode *N, SDValue V) {
44752     return N->getOpcode() == ISD::SUB && N->getOperand(1) == V &&
44753            ISD::isBuildVectorAllZeros(N->getOperand(0).getNode());
44754   };
44755 
44756   SDValue V;
44757   if (IsNegV(Y.getNode(), X))
44758     V = X;
44759   else if (IsNegV(X.getNode(), Y))
44760     V = Y;
44761   else
44762     return SDValue();
44763 
44764   SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
44765   SDValue SubOp2 = Mask;
44766 
44767   // If the negate was on the false side of the select, then
44768   // the operands of the SUB need to be swapped. PR 27251.
44769   // This is because the pattern being matched above is
44770   // (vselect M, (sub (0, X), X)  -> (sub (xor X, M), M)
44771   // but if the pattern matched was
44772   // (vselect M, X, (sub (0, X))), that is really negation of the pattern
44773   // above, -(vselect M, (sub 0, X), X), and therefore the replacement
44774   // pattern also needs to be a negation of the replacement pattern above.
44775   // And -(sub X, Y) is just sub (Y, X), so swapping the operands of the
44776   // sub accomplishes the negation of the replacement pattern.
44777   if (V == Y)
44778     std::swap(SubOp1, SubOp2);
44779 
44780   SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
44781   return DAG.getBitcast(VT, Res);
44782 }
44783 
44784 static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG,
44785                                   const X86Subtarget &Subtarget) {
44786   if (!Subtarget.hasAVX512())
44787     return SDValue();
44788   if (N->getOpcode() != ISD::VSELECT)
44789     return SDValue();
44790 
44791   SDLoc DL(N);
44792   SDValue Cond = N->getOperand(0);
44793   SDValue LHS = N->getOperand(1);
44794   SDValue RHS = N->getOperand(2);
44795 
44796   if (canCombineAsMaskOperation(LHS, Subtarget))
44797     return SDValue();
44798 
44799   if (!canCombineAsMaskOperation(RHS, Subtarget))
44800     return SDValue();
44801 
44802   if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
44803     return SDValue();
44804 
44805   // Commute LHS and RHS to create opportunity to select mask instruction.
44806   // (vselect M, L, R) -> (vselect ~M, R, L)
44807   ISD::CondCode NewCC =
44808       ISD::getSetCCInverse(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
44809                            Cond.getOperand(0).getValueType());
44810   Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), Cond.getOperand(0),
44811 		                        Cond.getOperand(1), NewCC);
44812   return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
44813 }
44814 
44815 /// Do target-specific dag combines on SELECT and VSELECT nodes.
44816 static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
44817                              TargetLowering::DAGCombinerInfo &DCI,
44818                              const X86Subtarget &Subtarget) {
44819   SDLoc DL(N);
44820   SDValue Cond = N->getOperand(0);
44821   SDValue LHS = N->getOperand(1);
44822   SDValue RHS = N->getOperand(2);
44823 
44824   // Try simplification again because we use this function to optimize
44825   // BLENDV nodes that are not handled by the generic combiner.
44826   if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
44827     return V;
44828 
44829   // When avx512 is available the lhs operand of select instruction can be
44830   // folded with mask instruction, while the rhs operand can't. Commute the
44831   // lhs and rhs of the select instruction to create the opportunity of
44832   // folding.
44833   if (SDValue V = commuteSelect(N, DAG, Subtarget))
44834     return V;
44835 
44836   EVT VT = LHS.getValueType();
44837   EVT CondVT = Cond.getValueType();
44838   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44839   bool CondConstantVector = ISD::isBuildVectorOfConstantSDNodes(Cond.getNode());
44840 
44841   // Attempt to combine (select M, (sub 0, X), X) -> (sub (xor X, M), M).
44842   // Limit this to cases of non-constant masks that createShuffleMaskFromVSELECT
44843   // can't catch, plus vXi8 cases where we'd likely end up with BLENDV.
44844   if (CondVT.isVector() && CondVT.isInteger() &&
44845       CondVT.getScalarSizeInBits() == VT.getScalarSizeInBits() &&
44846       (!CondConstantVector || CondVT.getScalarType() == MVT::i8) &&
44847       DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits())
44848     if (SDValue V = combineLogicBlendIntoConditionalNegate(VT, Cond, RHS, LHS,
44849                                                            DL, DAG, Subtarget))
44850       return V;
44851 
44852   // Convert vselects with constant condition into shuffles.
44853   if (CondConstantVector && DCI.isBeforeLegalizeOps() &&
44854       (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV)) {
44855     SmallVector<int, 64> Mask;
44856     if (createShuffleMaskFromVSELECT(Mask, Cond,
44857                                      N->getOpcode() == X86ISD::BLENDV))
44858       return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
44859   }
44860 
44861   // fold vselect(cond, pshufb(x), pshufb(y)) -> or (pshufb(x), pshufb(y))
44862   // by forcing the unselected elements to zero.
44863   // TODO: Can we handle more shuffles with this?
44864   if (N->getOpcode() == ISD::VSELECT && CondVT.isVector() &&
44865       LHS.getOpcode() == X86ISD::PSHUFB && RHS.getOpcode() == X86ISD::PSHUFB &&
44866       LHS.hasOneUse() && RHS.hasOneUse()) {
44867     MVT SimpleVT = VT.getSimpleVT();
44868     SmallVector<SDValue, 1> LHSOps, RHSOps;
44869     SmallVector<int, 64> LHSMask, RHSMask, CondMask;
44870     if (createShuffleMaskFromVSELECT(CondMask, Cond) &&
44871         getTargetShuffleMask(LHS.getNode(), SimpleVT, true, LHSOps, LHSMask) &&
44872         getTargetShuffleMask(RHS.getNode(), SimpleVT, true, RHSOps, RHSMask)) {
44873       int NumElts = VT.getVectorNumElements();
44874       for (int i = 0; i != NumElts; ++i) {
44875         // getConstVector sets negative shuffle mask values as undef, so ensure
44876         // we hardcode SM_SentinelZero values to zero (0x80).
44877         if (CondMask[i] < NumElts) {
44878           LHSMask[i] = isUndefOrZero(LHSMask[i]) ? 0x80 : LHSMask[i];
44879           RHSMask[i] = 0x80;
44880         } else {
44881           LHSMask[i] = 0x80;
44882           RHSMask[i] = isUndefOrZero(RHSMask[i]) ? 0x80 : RHSMask[i];
44883         }
44884       }
44885       LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0),
44886                         getConstVector(LHSMask, SimpleVT, DAG, DL, true));
44887       RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0),
44888                         getConstVector(RHSMask, SimpleVT, DAG, DL, true));
44889       return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
44890     }
44891   }
44892 
44893   // If we have SSE[12] support, try to form min/max nodes. SSE min/max
44894   // instructions match the semantics of the common C idiom x<y?x:y but not
44895   // x<=y?x:y, because of how they handle negative zero (which can be
44896   // ignored in unsafe-math mode).
44897   // We also try to create v2f32 min/max nodes, which we later widen to v4f32.
44898   if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
44899       VT != MVT::f80 && VT != MVT::f128 && !isSoftF16(VT, Subtarget) &&
44900       (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&
44901       (Subtarget.hasSSE2() ||
44902        (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) {
44903     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
44904 
44905     unsigned Opcode = 0;
44906     // Check for x CC y ? x : y.
44907     if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
44908         DAG.isEqualTo(RHS, Cond.getOperand(1))) {
44909       switch (CC) {
44910       default: break;
44911       case ISD::SETULT:
44912         // Converting this to a min would handle NaNs incorrectly, and swapping
44913         // the operands would cause it to handle comparisons between positive
44914         // and negative zero incorrectly.
44915         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
44916           if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44917               !(DAG.isKnownNeverZeroFloat(LHS) ||
44918                 DAG.isKnownNeverZeroFloat(RHS)))
44919             break;
44920           std::swap(LHS, RHS);
44921         }
44922         Opcode = X86ISD::FMIN;
44923         break;
44924       case ISD::SETOLE:
44925         // Converting this to a min would handle comparisons between positive
44926         // and negative zero incorrectly.
44927         if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44928             !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
44929           break;
44930         Opcode = X86ISD::FMIN;
44931         break;
44932       case ISD::SETULE:
44933         // Converting this to a min would handle both negative zeros and NaNs
44934         // incorrectly, but we can swap the operands to fix both.
44935         std::swap(LHS, RHS);
44936         [[fallthrough]];
44937       case ISD::SETOLT:
44938       case ISD::SETLT:
44939       case ISD::SETLE:
44940         Opcode = X86ISD::FMIN;
44941         break;
44942 
44943       case ISD::SETOGE:
44944         // Converting this to a max would handle comparisons between positive
44945         // and negative zero incorrectly.
44946         if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44947             !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
44948           break;
44949         Opcode = X86ISD::FMAX;
44950         break;
44951       case ISD::SETUGT:
44952         // Converting this to a max would handle NaNs incorrectly, and swapping
44953         // the operands would cause it to handle comparisons between positive
44954         // and negative zero incorrectly.
44955         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
44956           if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44957               !(DAG.isKnownNeverZeroFloat(LHS) ||
44958                 DAG.isKnownNeverZeroFloat(RHS)))
44959             break;
44960           std::swap(LHS, RHS);
44961         }
44962         Opcode = X86ISD::FMAX;
44963         break;
44964       case ISD::SETUGE:
44965         // Converting this to a max would handle both negative zeros and NaNs
44966         // incorrectly, but we can swap the operands to fix both.
44967         std::swap(LHS, RHS);
44968         [[fallthrough]];
44969       case ISD::SETOGT:
44970       case ISD::SETGT:
44971       case ISD::SETGE:
44972         Opcode = X86ISD::FMAX;
44973         break;
44974       }
44975     // Check for x CC y ? y : x -- a min/max with reversed arms.
44976     } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
44977                DAG.isEqualTo(RHS, Cond.getOperand(0))) {
44978       switch (CC) {
44979       default: break;
44980       case ISD::SETOGE:
44981         // Converting this to a min would handle comparisons between positive
44982         // and negative zero incorrectly, and swapping the operands would
44983         // cause it to handle NaNs incorrectly.
44984         if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
44985             !(DAG.isKnownNeverZeroFloat(LHS) ||
44986               DAG.isKnownNeverZeroFloat(RHS))) {
44987           if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
44988             break;
44989           std::swap(LHS, RHS);
44990         }
44991         Opcode = X86ISD::FMIN;
44992         break;
44993       case ISD::SETUGT:
44994         // Converting this to a min would handle NaNs incorrectly.
44995         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
44996           break;
44997         Opcode = X86ISD::FMIN;
44998         break;
44999       case ISD::SETUGE:
45000         // Converting this to a min would handle both negative zeros and NaNs
45001         // incorrectly, but we can swap the operands to fix both.
45002         std::swap(LHS, RHS);
45003         [[fallthrough]];
45004       case ISD::SETOGT:
45005       case ISD::SETGT:
45006       case ISD::SETGE:
45007         Opcode = X86ISD::FMIN;
45008         break;
45009 
45010       case ISD::SETULT:
45011         // Converting this to a max would handle NaNs incorrectly.
45012         if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
45013           break;
45014         Opcode = X86ISD::FMAX;
45015         break;
45016       case ISD::SETOLE:
45017         // Converting this to a max would handle comparisons between positive
45018         // and negative zero incorrectly, and swapping the operands would
45019         // cause it to handle NaNs incorrectly.
45020         if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
45021             !DAG.isKnownNeverZeroFloat(LHS) &&
45022             !DAG.isKnownNeverZeroFloat(RHS)) {
45023           if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
45024             break;
45025           std::swap(LHS, RHS);
45026         }
45027         Opcode = X86ISD::FMAX;
45028         break;
45029       case ISD::SETULE:
45030         // Converting this to a max would handle both negative zeros and NaNs
45031         // incorrectly, but we can swap the operands to fix both.
45032         std::swap(LHS, RHS);
45033         [[fallthrough]];
45034       case ISD::SETOLT:
45035       case ISD::SETLT:
45036       case ISD::SETLE:
45037         Opcode = X86ISD::FMAX;
45038         break;
45039       }
45040     }
45041 
45042     if (Opcode)
45043       return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
45044   }
45045 
45046   // Some mask scalar intrinsics rely on checking if only one bit is set
45047   // and implement it in C code like this:
45048   // A[0] = (U & 1) ? A[0] : W[0];
45049   // This creates some redundant instructions that break pattern matching.
45050   // fold (select (setcc (and (X, 1), 0, seteq), Y, Z)) -> select(and(X, 1),Z,Y)
45051   if (Subtarget.hasAVX512() && N->getOpcode() == ISD::SELECT &&
45052       Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) {
45053     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
45054     SDValue AndNode = Cond.getOperand(0);
45055     if (AndNode.getOpcode() == ISD::AND && CC == ISD::SETEQ &&
45056         isNullConstant(Cond.getOperand(1)) &&
45057         isOneConstant(AndNode.getOperand(1))) {
45058       // LHS and RHS swapped due to
45059       // setcc outputting 1 when AND resulted in 0 and vice versa.
45060       AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8);
45061       return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS);
45062     }
45063   }
45064 
45065   // v16i8 (select v16i1, v16i8, v16i8) does not have a proper
45066   // lowering on KNL. In this case we convert it to
45067   // v16i8 (select v16i8, v16i8, v16i8) and use AVX instruction.
45068   // The same situation all vectors of i8 and i16 without BWI.
45069   // Make sure we extend these even before type legalization gets a chance to
45070   // split wide vectors.
45071   // Since SKX these selects have a proper lowering.
45072   if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && CondVT.isVector() &&
45073       CondVT.getVectorElementType() == MVT::i1 &&
45074       (VT.getVectorElementType() == MVT::i8 ||
45075        VT.getVectorElementType() == MVT::i16)) {
45076     Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
45077     return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS);
45078   }
45079 
45080   // AVX512 - Extend select with zero to merge with target shuffle.
45081   // select(mask, extract_subvector(shuffle(x)), zero) -->
45082   // extract_subvector(select(insert_subvector(mask), shuffle(x), zero))
45083   // TODO - support non target shuffles as well.
45084   if (Subtarget.hasAVX512() && CondVT.isVector() &&
45085       CondVT.getVectorElementType() == MVT::i1) {
45086     auto SelectableOp = [&TLI](SDValue Op) {
45087       return Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
45088              isTargetShuffle(Op.getOperand(0).getOpcode()) &&
45089              isNullConstant(Op.getOperand(1)) &&
45090              TLI.isTypeLegal(Op.getOperand(0).getValueType()) &&
45091              Op.hasOneUse() && Op.getOperand(0).hasOneUse();
45092     };
45093 
45094     bool SelectableLHS = SelectableOp(LHS);
45095     bool SelectableRHS = SelectableOp(RHS);
45096     bool ZeroLHS = ISD::isBuildVectorAllZeros(LHS.getNode());
45097     bool ZeroRHS = ISD::isBuildVectorAllZeros(RHS.getNode());
45098 
45099     if ((SelectableLHS && ZeroRHS) || (SelectableRHS && ZeroLHS)) {
45100       EVT SrcVT = SelectableLHS ? LHS.getOperand(0).getValueType()
45101                                 : RHS.getOperand(0).getValueType();
45102       EVT SrcCondVT = SrcVT.changeVectorElementType(MVT::i1);
45103       LHS = insertSubVector(DAG.getUNDEF(SrcVT), LHS, 0, DAG, DL,
45104                             VT.getSizeInBits());
45105       RHS = insertSubVector(DAG.getUNDEF(SrcVT), RHS, 0, DAG, DL,
45106                             VT.getSizeInBits());
45107       Cond = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcCondVT,
45108                          DAG.getUNDEF(SrcCondVT), Cond,
45109                          DAG.getIntPtrConstant(0, DL));
45110       SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
45111       return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
45112     }
45113   }
45114 
45115   if (SDValue V = combineSelectOfTwoConstants(N, DAG))
45116     return V;
45117 
45118   if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC &&
45119       Cond.hasOneUse()) {
45120     EVT CondVT = Cond.getValueType();
45121     SDValue Cond0 = Cond.getOperand(0);
45122     SDValue Cond1 = Cond.getOperand(1);
45123     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
45124 
45125     // Canonicalize min/max:
45126     // (x > 0) ? x : 0 -> (x >= 0) ? x : 0
45127     // (x < -1) ? x : -1 -> (x <= -1) ? x : -1
45128     // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates
45129     // the need for an extra compare against zero. e.g.
45130     // (a - b) > 0 : (a - b) ? 0 -> (a - b) >= 0 : (a - b) ? 0
45131     // subl   %esi, %edi
45132     // testl  %edi, %edi
45133     // movl   $0, %eax
45134     // cmovgl %edi, %eax
45135     // =>
45136     // xorl   %eax, %eax
45137     // subl   %esi, $edi
45138     // cmovsl %eax, %edi
45139     //
45140     // We can also canonicalize
45141     //  (x s> 1) ? x : 1 -> (x s>= 1) ? x : 1 -> (x s> 0) ? x : 1
45142     //  (x u> 1) ? x : 1 -> (x u>= 1) ? x : 1 -> (x != 0) ? x : 1
45143     // This allows the use of a test instruction for the compare.
45144     if (LHS == Cond0 && RHS == Cond1) {
45145       if ((CC == ISD::SETGT && (isNullConstant(RHS) || isOneConstant(RHS))) ||
45146           (CC == ISD::SETLT && isAllOnesConstant(RHS))) {
45147         ISD::CondCode NewCC = CC == ISD::SETGT ? ISD::SETGE : ISD::SETLE;
45148         Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC);
45149         return DAG.getSelect(DL, VT, Cond, LHS, RHS);
45150       }
45151       if (CC == ISD::SETUGT && isOneConstant(RHS)) {
45152         ISD::CondCode NewCC = ISD::SETUGE;
45153         Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC);
45154         return DAG.getSelect(DL, VT, Cond, LHS, RHS);
45155       }
45156     }
45157 
45158     // Similar to DAGCombine's select(or(CC0,CC1),X,Y) fold but for legal types.
45159     // fold eq + gt/lt nested selects into ge/le selects
45160     // select (cmpeq Cond0, Cond1), LHS, (select (cmpugt Cond0, Cond1), LHS, Y)
45161     // --> (select (cmpuge Cond0, Cond1), LHS, Y)
45162     // select (cmpslt Cond0, Cond1), LHS, (select (cmpeq Cond0, Cond1), LHS, Y)
45163     // --> (select (cmpsle Cond0, Cond1), LHS, Y)
45164     // .. etc ..
45165     if (RHS.getOpcode() == ISD::SELECT && RHS.getOperand(1) == LHS &&
45166         RHS.getOperand(0).getOpcode() == ISD::SETCC) {
45167       SDValue InnerSetCC = RHS.getOperand(0);
45168       ISD::CondCode InnerCC =
45169           cast<CondCodeSDNode>(InnerSetCC.getOperand(2))->get();
45170       if ((CC == ISD::SETEQ || InnerCC == ISD::SETEQ) &&
45171           Cond0 == InnerSetCC.getOperand(0) &&
45172           Cond1 == InnerSetCC.getOperand(1)) {
45173         ISD::CondCode NewCC;
45174         switch (CC == ISD::SETEQ ? InnerCC : CC) {
45175         case ISD::SETGT:  NewCC = ISD::SETGE; break;
45176         case ISD::SETLT:  NewCC = ISD::SETLE; break;
45177         case ISD::SETUGT: NewCC = ISD::SETUGE; break;
45178         case ISD::SETULT: NewCC = ISD::SETULE; break;
45179         default: NewCC = ISD::SETCC_INVALID; break;
45180         }
45181         if (NewCC != ISD::SETCC_INVALID) {
45182           Cond = DAG.getSetCC(DL, CondVT, Cond0, Cond1, NewCC);
45183           return DAG.getSelect(DL, VT, Cond, LHS, RHS.getOperand(2));
45184         }
45185       }
45186     }
45187   }
45188 
45189   // Check if the first operand is all zeros and Cond type is vXi1.
45190   // If this an avx512 target we can improve the use of zero masking by
45191   // swapping the operands and inverting the condition.
45192   if (N->getOpcode() == ISD::VSELECT && Cond.hasOneUse() &&
45193       Subtarget.hasAVX512() && CondVT.getVectorElementType() == MVT::i1 &&
45194       ISD::isBuildVectorAllZeros(LHS.getNode()) &&
45195       !ISD::isBuildVectorAllZeros(RHS.getNode())) {
45196     // Invert the cond to not(cond) : xor(op,allones)=not(op)
45197     SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
45198     // Vselect cond, op1, op2 = Vselect not(cond), op2, op1
45199     return DAG.getSelect(DL, VT, CondNew, RHS, LHS);
45200   }
45201 
45202   // Attempt to convert a (vXi1 bitcast(iX Cond)) selection mask before it might
45203   // get split by legalization.
45204   if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::BITCAST &&
45205       CondVT.getVectorElementType() == MVT::i1 &&
45206       TLI.isTypeLegal(VT.getScalarType())) {
45207     EVT ExtCondVT = VT.changeVectorElementTypeToInteger();
45208     if (SDValue ExtCond = combineToExtendBoolVectorInReg(
45209             ISD::SIGN_EXTEND, DL, ExtCondVT, Cond, DAG, DCI, Subtarget)) {
45210       ExtCond = DAG.getNode(ISD::TRUNCATE, DL, CondVT, ExtCond);
45211       return DAG.getSelect(DL, VT, ExtCond, LHS, RHS);
45212     }
45213   }
45214 
45215   // Early exit check
45216   if (!TLI.isTypeLegal(VT) || isSoftF16(VT, Subtarget))
45217     return SDValue();
45218 
45219   if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DCI, Subtarget))
45220     return V;
45221 
45222   if (SDValue V = combineVSelectToBLENDV(N, DAG, DCI, Subtarget))
45223     return V;
45224 
45225   if (SDValue V = narrowVectorSelect(N, DAG, Subtarget))
45226     return V;
45227 
45228   // select(~Cond, X, Y) -> select(Cond, Y, X)
45229   if (CondVT.getScalarType() != MVT::i1) {
45230     if (SDValue CondNot = IsNOT(Cond, DAG))
45231       return DAG.getNode(N->getOpcode(), DL, VT,
45232                          DAG.getBitcast(CondVT, CondNot), RHS, LHS);
45233 
45234     // pcmpgt(X, -1) -> pcmpgt(0, X) to help select/blendv just use the
45235     // signbit.
45236     if (Cond.getOpcode() == X86ISD::PCMPGT &&
45237         ISD::isBuildVectorAllOnes(Cond.getOperand(1).getNode()) &&
45238         Cond.hasOneUse()) {
45239       Cond = DAG.getNode(X86ISD::PCMPGT, DL, CondVT,
45240                          DAG.getConstant(0, DL, CondVT), Cond.getOperand(0));
45241       return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS);
45242     }
45243   }
45244 
45245   // Try to optimize vXi1 selects if both operands are either all constants or
45246   // bitcasts from scalar integer type. In that case we can convert the operands
45247   // to integer and use an integer select which will be converted to a CMOV.
45248   // We need to take a little bit of care to avoid creating an i64 type after
45249   // type legalization.
45250   if (N->getOpcode() == ISD::SELECT && VT.isVector() &&
45251       VT.getVectorElementType() == MVT::i1 &&
45252       (DCI.isBeforeLegalize() || (VT != MVT::v64i1 || Subtarget.is64Bit()))) {
45253     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
45254     if (DCI.isBeforeLegalize() || TLI.isTypeLegal(IntVT)) {
45255       bool LHSIsConst = ISD::isBuildVectorOfConstantSDNodes(LHS.getNode());
45256       bool RHSIsConst = ISD::isBuildVectorOfConstantSDNodes(RHS.getNode());
45257 
45258       if ((LHSIsConst || (LHS.getOpcode() == ISD::BITCAST &&
45259                           LHS.getOperand(0).getValueType() == IntVT)) &&
45260           (RHSIsConst || (RHS.getOpcode() == ISD::BITCAST &&
45261                           RHS.getOperand(0).getValueType() == IntVT))) {
45262         if (LHSIsConst)
45263           LHS = combinevXi1ConstantToInteger(LHS, DAG);
45264         else
45265           LHS = LHS.getOperand(0);
45266 
45267         if (RHSIsConst)
45268           RHS = combinevXi1ConstantToInteger(RHS, DAG);
45269         else
45270           RHS = RHS.getOperand(0);
45271 
45272         SDValue Select = DAG.getSelect(DL, IntVT, Cond, LHS, RHS);
45273         return DAG.getBitcast(VT, Select);
45274       }
45275     }
45276   }
45277 
45278   // If this is "((X & C) == 0) ? Y : Z" and C is a constant mask vector of
45279   // single bits, then invert the predicate and swap the select operands.
45280   // This can lower using a vector shift bit-hack rather than mask and compare.
45281   if (DCI.isBeforeLegalize() && !Subtarget.hasAVX512() &&
45282       N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
45283       Cond.hasOneUse() && CondVT.getVectorElementType() == MVT::i1 &&
45284       Cond.getOperand(0).getOpcode() == ISD::AND &&
45285       isNullOrNullSplat(Cond.getOperand(1)) &&
45286       cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ &&
45287       Cond.getOperand(0).getValueType() == VT) {
45288     // The 'and' mask must be composed of power-of-2 constants.
45289     SDValue And = Cond.getOperand(0);
45290     auto *C = isConstOrConstSplat(And.getOperand(1));
45291     if (C && C->getAPIntValue().isPowerOf2()) {
45292       // vselect (X & C == 0), LHS, RHS --> vselect (X & C != 0), RHS, LHS
45293       SDValue NotCond =
45294           DAG.getSetCC(DL, CondVT, And, Cond.getOperand(1), ISD::SETNE);
45295       return DAG.getSelect(DL, VT, NotCond, RHS, LHS);
45296     }
45297 
45298     // If we have a non-splat but still powers-of-2 mask, AVX1 can use pmulld
45299     // and AVX2 can use vpsllv{dq}. 8-bit lacks a proper shift or multiply.
45300     // 16-bit lacks a proper blendv.
45301     unsigned EltBitWidth = VT.getScalarSizeInBits();
45302     bool CanShiftBlend =
45303         TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) ||
45304                                 (Subtarget.hasAVX2() && EltBitWidth == 64) ||
45305                                 (Subtarget.hasXOP()));
45306     if (CanShiftBlend &&
45307         ISD::matchUnaryPredicate(And.getOperand(1), [](ConstantSDNode *C) {
45308           return C->getAPIntValue().isPowerOf2();
45309         })) {
45310       // Create a left-shift constant to get the mask bits over to the sign-bit.
45311       SDValue Mask = And.getOperand(1);
45312       SmallVector<int, 32> ShlVals;
45313       for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
45314         auto *MaskVal = cast<ConstantSDNode>(Mask.getOperand(i));
45315         ShlVals.push_back(EltBitWidth - 1 -
45316                           MaskVal->getAPIntValue().exactLogBase2());
45317       }
45318       // vsel ((X & C) == 0), LHS, RHS --> vsel ((shl X, C') < 0), RHS, LHS
45319       SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL);
45320       SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt);
45321       SDValue NewCond =
45322           DAG.getSetCC(DL, CondVT, Shl, Cond.getOperand(1), ISD::SETLT);
45323       return DAG.getSelect(DL, VT, NewCond, RHS, LHS);
45324     }
45325   }
45326 
45327   return SDValue();
45328 }
45329 
45330 /// Combine:
45331 ///   (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S)
45332 /// to:
45333 ///   (brcond/cmov/setcc .., (LADD x, 1), COND_LE)
45334 /// i.e., reusing the EFLAGS produced by the LOCKed instruction.
45335 /// Note that this is only legal for some op/cc combinations.
45336 static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
45337                                        SelectionDAG &DAG,
45338                                        const X86Subtarget &Subtarget) {
45339   // This combine only operates on CMP-like nodes.
45340   if (!(Cmp.getOpcode() == X86ISD::CMP ||
45341         (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
45342     return SDValue();
45343 
45344   // Can't replace the cmp if it has more uses than the one we're looking at.
45345   // FIXME: We would like to be able to handle this, but would need to make sure
45346   // all uses were updated.
45347   if (!Cmp.hasOneUse())
45348     return SDValue();
45349 
45350   // This only applies to variations of the common case:
45351   //   (icmp slt x, 0) -> (icmp sle (add x, 1), 0)
45352   //   (icmp sge x, 0) -> (icmp sgt (add x, 1), 0)
45353   //   (icmp sle x, 0) -> (icmp slt (sub x, 1), 0)
45354   //   (icmp sgt x, 0) -> (icmp sge (sub x, 1), 0)
45355   // Using the proper condcodes (see below), overflow is checked for.
45356 
45357   // FIXME: We can generalize both constraints:
45358   // - XOR/OR/AND (if they were made to survive AtomicExpand)
45359   // - LHS != 1
45360   // if the result is compared.
45361 
45362   SDValue CmpLHS = Cmp.getOperand(0);
45363   SDValue CmpRHS = Cmp.getOperand(1);
45364   EVT CmpVT = CmpLHS.getValueType();
45365 
45366   if (!CmpLHS.hasOneUse())
45367     return SDValue();
45368 
45369   unsigned Opc = CmpLHS.getOpcode();
45370   if (Opc != ISD::ATOMIC_LOAD_ADD && Opc != ISD::ATOMIC_LOAD_SUB)
45371     return SDValue();
45372 
45373   SDValue OpRHS = CmpLHS.getOperand(2);
45374   auto *OpRHSC = dyn_cast<ConstantSDNode>(OpRHS);
45375   if (!OpRHSC)
45376     return SDValue();
45377 
45378   APInt Addend = OpRHSC->getAPIntValue();
45379   if (Opc == ISD::ATOMIC_LOAD_SUB)
45380     Addend = -Addend;
45381 
45382   auto *CmpRHSC = dyn_cast<ConstantSDNode>(CmpRHS);
45383   if (!CmpRHSC)
45384     return SDValue();
45385 
45386   APInt Comparison = CmpRHSC->getAPIntValue();
45387   APInt NegAddend = -Addend;
45388 
45389   // See if we can adjust the CC to make the comparison match the negated
45390   // addend.
45391   if (Comparison != NegAddend) {
45392     APInt IncComparison = Comparison + 1;
45393     if (IncComparison == NegAddend) {
45394       if (CC == X86::COND_A && !Comparison.isMaxValue()) {
45395         Comparison = IncComparison;
45396         CC = X86::COND_AE;
45397       } else if (CC == X86::COND_LE && !Comparison.isMaxSignedValue()) {
45398         Comparison = IncComparison;
45399         CC = X86::COND_L;
45400       }
45401     }
45402     APInt DecComparison = Comparison - 1;
45403     if (DecComparison == NegAddend) {
45404       if (CC == X86::COND_AE && !Comparison.isMinValue()) {
45405         Comparison = DecComparison;
45406         CC = X86::COND_A;
45407       } else if (CC == X86::COND_L && !Comparison.isMinSignedValue()) {
45408         Comparison = DecComparison;
45409         CC = X86::COND_LE;
45410       }
45411     }
45412   }
45413 
45414   // If the addend is the negation of the comparison value, then we can do
45415   // a full comparison by emitting the atomic arithmetic as a locked sub.
45416   if (Comparison == NegAddend) {
45417     // The CC is fine, but we need to rewrite the LHS of the comparison as an
45418     // atomic sub.
45419     auto *AN = cast<AtomicSDNode>(CmpLHS.getNode());
45420     auto AtomicSub = DAG.getAtomic(
45421         ISD::ATOMIC_LOAD_SUB, SDLoc(CmpLHS), CmpVT,
45422         /*Chain*/ CmpLHS.getOperand(0), /*LHS*/ CmpLHS.getOperand(1),
45423         /*RHS*/ DAG.getConstant(NegAddend, SDLoc(CmpRHS), CmpVT),
45424         AN->getMemOperand());
45425     auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget);
45426     DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
45427     DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
45428     return LockOp;
45429   }
45430 
45431   // We can handle comparisons with zero in a number of cases by manipulating
45432   // the CC used.
45433   if (!Comparison.isZero())
45434     return SDValue();
45435 
45436   if (CC == X86::COND_S && Addend == 1)
45437     CC = X86::COND_LE;
45438   else if (CC == X86::COND_NS && Addend == 1)
45439     CC = X86::COND_G;
45440   else if (CC == X86::COND_G && Addend == -1)
45441     CC = X86::COND_GE;
45442   else if (CC == X86::COND_LE && Addend == -1)
45443     CC = X86::COND_L;
45444   else
45445     return SDValue();
45446 
45447   SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget);
45448   DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
45449   DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
45450   return LockOp;
45451 }
45452 
45453 // Check whether a boolean test is testing a boolean value generated by
45454 // X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
45455 // code.
45456 //
45457 // Simplify the following patterns:
45458 // (Op (CMP (SETCC Cond EFLAGS) 1) EQ) or
45459 // (Op (CMP (SETCC Cond EFLAGS) 0) NEQ)
45460 // to (Op EFLAGS Cond)
45461 //
45462 // (Op (CMP (SETCC Cond EFLAGS) 0) EQ) or
45463 // (Op (CMP (SETCC Cond EFLAGS) 1) NEQ)
45464 // to (Op EFLAGS !Cond)
45465 //
45466 // where Op could be BRCOND or CMOV.
45467 //
45468 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
45469   // This combine only operates on CMP-like nodes.
45470   if (!(Cmp.getOpcode() == X86ISD::CMP ||
45471         (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
45472     return SDValue();
45473 
45474   // Quit if not used as a boolean value.
45475   if (CC != X86::COND_E && CC != X86::COND_NE)
45476     return SDValue();
45477 
45478   // Check CMP operands. One of them should be 0 or 1 and the other should be
45479   // an SetCC or extended from it.
45480   SDValue Op1 = Cmp.getOperand(0);
45481   SDValue Op2 = Cmp.getOperand(1);
45482 
45483   SDValue SetCC;
45484   const ConstantSDNode* C = nullptr;
45485   bool needOppositeCond = (CC == X86::COND_E);
45486   bool checkAgainstTrue = false; // Is it a comparison against 1?
45487 
45488   if ((C = dyn_cast<ConstantSDNode>(Op1)))
45489     SetCC = Op2;
45490   else if ((C = dyn_cast<ConstantSDNode>(Op2)))
45491     SetCC = Op1;
45492   else // Quit if all operands are not constants.
45493     return SDValue();
45494 
45495   if (C->getZExtValue() == 1) {
45496     needOppositeCond = !needOppositeCond;
45497     checkAgainstTrue = true;
45498   } else if (C->getZExtValue() != 0)
45499     // Quit if the constant is neither 0 or 1.
45500     return SDValue();
45501 
45502   bool truncatedToBoolWithAnd = false;
45503   // Skip (zext $x), (trunc $x), or (and $x, 1) node.
45504   while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
45505          SetCC.getOpcode() == ISD::TRUNCATE ||
45506          SetCC.getOpcode() == ISD::AND) {
45507     if (SetCC.getOpcode() == ISD::AND) {
45508       int OpIdx = -1;
45509       if (isOneConstant(SetCC.getOperand(0)))
45510         OpIdx = 1;
45511       if (isOneConstant(SetCC.getOperand(1)))
45512         OpIdx = 0;
45513       if (OpIdx < 0)
45514         break;
45515       SetCC = SetCC.getOperand(OpIdx);
45516       truncatedToBoolWithAnd = true;
45517     } else
45518       SetCC = SetCC.getOperand(0);
45519   }
45520 
45521   switch (SetCC.getOpcode()) {
45522   case X86ISD::SETCC_CARRY:
45523     // Since SETCC_CARRY gives output based on R = CF ? ~0 : 0, it's unsafe to
45524     // simplify it if the result of SETCC_CARRY is not canonicalized to 0 or 1,
45525     // i.e. it's a comparison against true but the result of SETCC_CARRY is not
45526     // truncated to i1 using 'and'.
45527     if (checkAgainstTrue && !truncatedToBoolWithAnd)
45528       break;
45529     assert(X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B &&
45530            "Invalid use of SETCC_CARRY!");
45531     [[fallthrough]];
45532   case X86ISD::SETCC:
45533     // Set the condition code or opposite one if necessary.
45534     CC = X86::CondCode(SetCC.getConstantOperandVal(0));
45535     if (needOppositeCond)
45536       CC = X86::GetOppositeBranchCondition(CC);
45537     return SetCC.getOperand(1);
45538   case X86ISD::CMOV: {
45539     // Check whether false/true value has canonical one, i.e. 0 or 1.
45540     ConstantSDNode *FVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(0));
45541     ConstantSDNode *TVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(1));
45542     // Quit if true value is not a constant.
45543     if (!TVal)
45544       return SDValue();
45545     // Quit if false value is not a constant.
45546     if (!FVal) {
45547       SDValue Op = SetCC.getOperand(0);
45548       // Skip 'zext' or 'trunc' node.
45549       if (Op.getOpcode() == ISD::ZERO_EXTEND ||
45550           Op.getOpcode() == ISD::TRUNCATE)
45551         Op = Op.getOperand(0);
45552       // A special case for rdrand/rdseed, where 0 is set if false cond is
45553       // found.
45554       if ((Op.getOpcode() != X86ISD::RDRAND &&
45555            Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
45556         return SDValue();
45557     }
45558     // Quit if false value is not the constant 0 or 1.
45559     bool FValIsFalse = true;
45560     if (FVal && FVal->getZExtValue() != 0) {
45561       if (FVal->getZExtValue() != 1)
45562         return SDValue();
45563       // If FVal is 1, opposite cond is needed.
45564       needOppositeCond = !needOppositeCond;
45565       FValIsFalse = false;
45566     }
45567     // Quit if TVal is not the constant opposite of FVal.
45568     if (FValIsFalse && TVal->getZExtValue() != 1)
45569       return SDValue();
45570     if (!FValIsFalse && TVal->getZExtValue() != 0)
45571       return SDValue();
45572     CC = X86::CondCode(SetCC.getConstantOperandVal(2));
45573     if (needOppositeCond)
45574       CC = X86::GetOppositeBranchCondition(CC);
45575     return SetCC.getOperand(3);
45576   }
45577   }
45578 
45579   return SDValue();
45580 }
45581 
45582 /// Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
45583 /// Match:
45584 ///   (X86or (X86setcc) (X86setcc))
45585 ///   (X86cmp (and (X86setcc) (X86setcc)), 0)
45586 static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
45587                                            X86::CondCode &CC1, SDValue &Flags,
45588                                            bool &isAnd) {
45589   if (Cond->getOpcode() == X86ISD::CMP) {
45590     if (!isNullConstant(Cond->getOperand(1)))
45591       return false;
45592 
45593     Cond = Cond->getOperand(0);
45594   }
45595 
45596   isAnd = false;
45597 
45598   SDValue SetCC0, SetCC1;
45599   switch (Cond->getOpcode()) {
45600   default: return false;
45601   case ISD::AND:
45602   case X86ISD::AND:
45603     isAnd = true;
45604     [[fallthrough]];
45605   case ISD::OR:
45606   case X86ISD::OR:
45607     SetCC0 = Cond->getOperand(0);
45608     SetCC1 = Cond->getOperand(1);
45609     break;
45610   };
45611 
45612   // Make sure we have SETCC nodes, using the same flags value.
45613   if (SetCC0.getOpcode() != X86ISD::SETCC ||
45614       SetCC1.getOpcode() != X86ISD::SETCC ||
45615       SetCC0->getOperand(1) != SetCC1->getOperand(1))
45616     return false;
45617 
45618   CC0 = (X86::CondCode)SetCC0->getConstantOperandVal(0);
45619   CC1 = (X86::CondCode)SetCC1->getConstantOperandVal(0);
45620   Flags = SetCC0->getOperand(1);
45621   return true;
45622 }
45623 
45624 // When legalizing carry, we create carries via add X, -1
45625 // If that comes from an actual carry, via setcc, we use the
45626 // carry directly.
45627 static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
45628   if (EFLAGS.getOpcode() == X86ISD::ADD) {
45629     if (isAllOnesConstant(EFLAGS.getOperand(1))) {
45630       bool FoundAndLSB = false;
45631       SDValue Carry = EFLAGS.getOperand(0);
45632       while (Carry.getOpcode() == ISD::TRUNCATE ||
45633              Carry.getOpcode() == ISD::ZERO_EXTEND ||
45634              (Carry.getOpcode() == ISD::AND &&
45635               isOneConstant(Carry.getOperand(1)))) {
45636         FoundAndLSB |= Carry.getOpcode() == ISD::AND;
45637         Carry = Carry.getOperand(0);
45638       }
45639       if (Carry.getOpcode() == X86ISD::SETCC ||
45640           Carry.getOpcode() == X86ISD::SETCC_CARRY) {
45641         // TODO: Merge this code with equivalent in combineAddOrSubToADCOrSBB?
45642         uint64_t CarryCC = Carry.getConstantOperandVal(0);
45643         SDValue CarryOp1 = Carry.getOperand(1);
45644         if (CarryCC == X86::COND_B)
45645           return CarryOp1;
45646         if (CarryCC == X86::COND_A) {
45647           // Try to convert COND_A into COND_B in an attempt to facilitate
45648           // materializing "setb reg".
45649           //
45650           // Do not flip "e > c", where "c" is a constant, because Cmp
45651           // instruction cannot take an immediate as its first operand.
45652           //
45653           if (CarryOp1.getOpcode() == X86ISD::SUB &&
45654               CarryOp1.getNode()->hasOneUse() &&
45655               CarryOp1.getValueType().isInteger() &&
45656               !isa<ConstantSDNode>(CarryOp1.getOperand(1))) {
45657             SDValue SubCommute =
45658                 DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(),
45659                             CarryOp1.getOperand(1), CarryOp1.getOperand(0));
45660             return SDValue(SubCommute.getNode(), CarryOp1.getResNo());
45661           }
45662         }
45663         // If this is a check of the z flag of an add with 1, switch to the
45664         // C flag.
45665         if (CarryCC == X86::COND_E &&
45666             CarryOp1.getOpcode() == X86ISD::ADD &&
45667             isOneConstant(CarryOp1.getOperand(1)))
45668           return CarryOp1;
45669       } else if (FoundAndLSB) {
45670         SDLoc DL(Carry);
45671         SDValue BitNo = DAG.getConstant(0, DL, Carry.getValueType());
45672         if (Carry.getOpcode() == ISD::SRL) {
45673           BitNo = Carry.getOperand(1);
45674           Carry = Carry.getOperand(0);
45675         }
45676         return getBT(Carry, BitNo, DL, DAG);
45677       }
45678     }
45679   }
45680 
45681   return SDValue();
45682 }
45683 
45684 /// If we are inverting an PTEST/TESTP operand, attempt to adjust the CC
45685 /// to avoid the inversion.
45686 static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC,
45687                               SelectionDAG &DAG,
45688                               const X86Subtarget &Subtarget) {
45689   // TODO: Handle X86ISD::KTEST/X86ISD::KORTEST.
45690   if (EFLAGS.getOpcode() != X86ISD::PTEST &&
45691       EFLAGS.getOpcode() != X86ISD::TESTP)
45692     return SDValue();
45693 
45694   // PTEST/TESTP sets EFLAGS as:
45695   // TESTZ: ZF = (Op0 & Op1) == 0
45696   // TESTC: CF = (~Op0 & Op1) == 0
45697   // TESTNZC: ZF == 0 && CF == 0
45698   MVT VT = EFLAGS.getSimpleValueType();
45699   SDValue Op0 = EFLAGS.getOperand(0);
45700   SDValue Op1 = EFLAGS.getOperand(1);
45701   MVT OpVT = Op0.getSimpleValueType();
45702 
45703   // TEST*(~X,Y) == TEST*(X,Y)
45704   if (SDValue NotOp0 = IsNOT(Op0, DAG)) {
45705     X86::CondCode InvCC;
45706     switch (CC) {
45707     case X86::COND_B:
45708       // testc -> testz.
45709       InvCC = X86::COND_E;
45710       break;
45711     case X86::COND_AE:
45712       // !testc -> !testz.
45713       InvCC = X86::COND_NE;
45714       break;
45715     case X86::COND_E:
45716       // testz -> testc.
45717       InvCC = X86::COND_B;
45718       break;
45719     case X86::COND_NE:
45720       // !testz -> !testc.
45721       InvCC = X86::COND_AE;
45722       break;
45723     case X86::COND_A:
45724     case X86::COND_BE:
45725       // testnzc -> testnzc (no change).
45726       InvCC = CC;
45727       break;
45728     default:
45729       InvCC = X86::COND_INVALID;
45730       break;
45731     }
45732 
45733     if (InvCC != X86::COND_INVALID) {
45734       CC = InvCC;
45735       return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45736                          DAG.getBitcast(OpVT, NotOp0), Op1);
45737     }
45738   }
45739 
45740   if (CC == X86::COND_B || CC == X86::COND_AE) {
45741     // TESTC(X,~X) == TESTC(X,-1)
45742     if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
45743       if (peekThroughBitcasts(NotOp1) == peekThroughBitcasts(Op0)) {
45744         SDLoc DL(EFLAGS);
45745         return DAG.getNode(
45746             EFLAGS.getOpcode(), DL, VT, DAG.getBitcast(OpVT, NotOp1),
45747             DAG.getBitcast(OpVT,
45748                            DAG.getAllOnesConstant(DL, NotOp1.getValueType())));
45749       }
45750     }
45751   }
45752 
45753   if (CC == X86::COND_E || CC == X86::COND_NE) {
45754     // TESTZ(X,~Y) == TESTC(Y,X)
45755     if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
45756       CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
45757       return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45758                          DAG.getBitcast(OpVT, NotOp1), Op0);
45759     }
45760 
45761     if (Op0 == Op1) {
45762       SDValue BC = peekThroughBitcasts(Op0);
45763       EVT BCVT = BC.getValueType();
45764 
45765       // TESTZ(AND(X,Y),AND(X,Y)) == TESTZ(X,Y)
45766       if (BC.getOpcode() == ISD::AND || BC.getOpcode() == X86ISD::FAND) {
45767         return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45768                            DAG.getBitcast(OpVT, BC.getOperand(0)),
45769                            DAG.getBitcast(OpVT, BC.getOperand(1)));
45770       }
45771 
45772       // TESTZ(AND(~X,Y),AND(~X,Y)) == TESTC(X,Y)
45773       if (BC.getOpcode() == X86ISD::ANDNP || BC.getOpcode() == X86ISD::FANDN) {
45774         CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
45775         return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45776                            DAG.getBitcast(OpVT, BC.getOperand(0)),
45777                            DAG.getBitcast(OpVT, BC.getOperand(1)));
45778       }
45779 
45780       // If every element is an all-sign value, see if we can use TESTP/MOVMSK
45781       // to more efficiently extract the sign bits and compare that.
45782       // TODO: Handle TESTC with comparison inversion.
45783       // TODO: Can we remove SimplifyMultipleUseDemandedBits and rely on
45784       // TESTP/MOVMSK combines to make sure its never worse than PTEST?
45785       if (BCVT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(BCVT)) {
45786         unsigned EltBits = BCVT.getScalarSizeInBits();
45787         if (DAG.ComputeNumSignBits(BC) == EltBits) {
45788           assert(VT == MVT::i32 && "Expected i32 EFLAGS comparison result");
45789           APInt SignMask = APInt::getSignMask(EltBits);
45790           const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45791           if (SDValue Res =
45792                   TLI.SimplifyMultipleUseDemandedBits(BC, SignMask, DAG)) {
45793             // For vXi16 cases we need to use pmovmksb and extract every other
45794             // sign bit.
45795             SDLoc DL(EFLAGS);
45796             if ((EltBits == 32 || EltBits == 64) && Subtarget.hasAVX()) {
45797               MVT FloatSVT = MVT::getFloatingPointVT(EltBits);
45798               MVT FloatVT =
45799                   MVT::getVectorVT(FloatSVT, OpVT.getSizeInBits() / EltBits);
45800               Res = DAG.getBitcast(FloatVT, Res);
45801               return DAG.getNode(X86ISD::TESTP, SDLoc(EFLAGS), VT, Res, Res);
45802             } else if (EltBits == 16) {
45803               MVT MovmskVT = BCVT.is128BitVector() ? MVT::v16i8 : MVT::v32i8;
45804               Res = DAG.getBitcast(MovmskVT, Res);
45805               Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
45806               Res = DAG.getNode(ISD::AND, DL, MVT::i32, Res,
45807                                 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
45808             } else {
45809               Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
45810             }
45811             return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Res,
45812                                DAG.getConstant(0, DL, MVT::i32));
45813           }
45814         }
45815       }
45816     }
45817 
45818     // TESTZ(-1,X) == TESTZ(X,X)
45819     if (ISD::isBuildVectorAllOnes(Op0.getNode()))
45820       return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1);
45821 
45822     // TESTZ(X,-1) == TESTZ(X,X)
45823     if (ISD::isBuildVectorAllOnes(Op1.getNode()))
45824       return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0);
45825 
45826     // TESTZ(OR(LO(X),HI(X)),OR(LO(Y),HI(Y))) -> TESTZ(X,Y)
45827     // TODO: Add COND_NE handling?
45828     if (CC == X86::COND_E && OpVT.is128BitVector() && Subtarget.hasAVX()) {
45829       SDValue Src0 = peekThroughBitcasts(Op0);
45830       SDValue Src1 = peekThroughBitcasts(Op1);
45831       if (Src0.getOpcode() == ISD::OR && Src1.getOpcode() == ISD::OR) {
45832         Src0 = getSplitVectorSrc(peekThroughBitcasts(Src0.getOperand(0)),
45833                                  peekThroughBitcasts(Src0.getOperand(1)), true);
45834         Src1 = getSplitVectorSrc(peekThroughBitcasts(Src1.getOperand(0)),
45835                                  peekThroughBitcasts(Src1.getOperand(1)), true);
45836         if (Src0 && Src1) {
45837           MVT OpVT2 = OpVT.getDoubleNumVectorElementsVT();
45838           return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
45839                              DAG.getBitcast(OpVT2, Src0),
45840                              DAG.getBitcast(OpVT2, Src1));
45841         }
45842       }
45843     }
45844   }
45845 
45846   return SDValue();
45847 }
45848 
45849 // Attempt to simplify the MOVMSK input based on the comparison type.
45850 static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
45851                                   SelectionDAG &DAG,
45852                                   const X86Subtarget &Subtarget) {
45853   // Handle eq/ne against zero (any_of).
45854   // Handle eq/ne against -1 (all_of).
45855   if (!(CC == X86::COND_E || CC == X86::COND_NE))
45856     return SDValue();
45857   if (EFLAGS.getValueType() != MVT::i32)
45858     return SDValue();
45859   unsigned CmpOpcode = EFLAGS.getOpcode();
45860   if (CmpOpcode != X86ISD::CMP && CmpOpcode != X86ISD::SUB)
45861     return SDValue();
45862   auto *CmpConstant = dyn_cast<ConstantSDNode>(EFLAGS.getOperand(1));
45863   if (!CmpConstant)
45864     return SDValue();
45865   const APInt &CmpVal = CmpConstant->getAPIntValue();
45866 
45867   SDValue CmpOp = EFLAGS.getOperand(0);
45868   unsigned CmpBits = CmpOp.getValueSizeInBits();
45869   assert(CmpBits == CmpVal.getBitWidth() && "Value size mismatch");
45870 
45871   // Peek through any truncate.
45872   if (CmpOp.getOpcode() == ISD::TRUNCATE)
45873     CmpOp = CmpOp.getOperand(0);
45874 
45875   // Bail if we don't find a MOVMSK.
45876   if (CmpOp.getOpcode() != X86ISD::MOVMSK)
45877     return SDValue();
45878 
45879   SDValue Vec = CmpOp.getOperand(0);
45880   MVT VecVT = Vec.getSimpleValueType();
45881   assert((VecVT.is128BitVector() || VecVT.is256BitVector()) &&
45882          "Unexpected MOVMSK operand");
45883   unsigned NumElts = VecVT.getVectorNumElements();
45884   unsigned NumEltBits = VecVT.getScalarSizeInBits();
45885 
45886   bool IsAnyOf = CmpOpcode == X86ISD::CMP && CmpVal.isZero();
45887   bool IsAllOf = (CmpOpcode == X86ISD::SUB || CmpOpcode == X86ISD::CMP) &&
45888                  NumElts <= CmpBits && CmpVal.isMask(NumElts);
45889   if (!IsAnyOf && !IsAllOf)
45890     return SDValue();
45891 
45892   // TODO: Check more combining cases for me.
45893   // Here we check the cmp use number to decide do combining or not.
45894   // Currently we only get 2 tests about combining "MOVMSK(CONCAT(..))"
45895   // and "MOVMSK(PCMPEQ(..))" are fit to use this constraint.
45896   bool IsOneUse = CmpOp.getNode()->hasOneUse();
45897 
45898   // See if we can peek through to a vector with a wider element type, if the
45899   // signbits extend down to all the sub-elements as well.
45900   // Calling MOVMSK with the wider type, avoiding the bitcast, helps expose
45901   // potential SimplifyDemandedBits/Elts cases.
45902   // If we looked through a truncate that discard bits, we can't do this
45903   // transform.
45904   // FIXME: We could do this transform for truncates that discarded bits by
45905   // inserting an AND mask between the new MOVMSK and the CMP.
45906   if (Vec.getOpcode() == ISD::BITCAST && NumElts <= CmpBits) {
45907     SDValue BC = peekThroughBitcasts(Vec);
45908     MVT BCVT = BC.getSimpleValueType();
45909     unsigned BCNumElts = BCVT.getVectorNumElements();
45910     unsigned BCNumEltBits = BCVT.getScalarSizeInBits();
45911     if ((BCNumEltBits == 32 || BCNumEltBits == 64) &&
45912         BCNumEltBits > NumEltBits &&
45913         DAG.ComputeNumSignBits(BC) > (BCNumEltBits - NumEltBits)) {
45914       SDLoc DL(EFLAGS);
45915       APInt CmpMask = APInt::getLowBitsSet(32, IsAnyOf ? 0 : BCNumElts);
45916       return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
45917                          DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, BC),
45918                          DAG.getConstant(CmpMask, DL, MVT::i32));
45919     }
45920   }
45921 
45922   // MOVMSK(CONCAT(X,Y)) == 0 ->  MOVMSK(OR(X,Y)).
45923   // MOVMSK(CONCAT(X,Y)) != 0 ->  MOVMSK(OR(X,Y)).
45924   // MOVMSK(CONCAT(X,Y)) == -1 ->  MOVMSK(AND(X,Y)).
45925   // MOVMSK(CONCAT(X,Y)) != -1 ->  MOVMSK(AND(X,Y)).
45926   if (VecVT.is256BitVector() && NumElts <= CmpBits && IsOneUse) {
45927     SmallVector<SDValue> Ops;
45928     if (collectConcatOps(peekThroughBitcasts(Vec).getNode(), Ops, DAG) &&
45929         Ops.size() == 2) {
45930       SDLoc DL(EFLAGS);
45931       EVT SubVT = Ops[0].getValueType().changeTypeToInteger();
45932       APInt CmpMask = APInt::getLowBitsSet(32, IsAnyOf ? 0 : NumElts / 2);
45933       SDValue V = DAG.getNode(IsAnyOf ? ISD::OR : ISD::AND, DL, SubVT,
45934                               DAG.getBitcast(SubVT, Ops[0]),
45935                               DAG.getBitcast(SubVT, Ops[1]));
45936       V = DAG.getBitcast(VecVT.getHalfNumVectorElementsVT(), V);
45937       return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
45938                          DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V),
45939                          DAG.getConstant(CmpMask, DL, MVT::i32));
45940     }
45941   }
45942 
45943   // MOVMSK(PCMPEQ(X,0)) == -1 -> PTESTZ(X,X).
45944   // MOVMSK(PCMPEQ(X,0)) != -1 -> !PTESTZ(X,X).
45945   // MOVMSK(PCMPEQ(X,Y)) == -1 -> PTESTZ(XOR(X,Y),XOR(X,Y)).
45946   // MOVMSK(PCMPEQ(X,Y)) != -1 -> !PTESTZ(XOR(X,Y),XOR(X,Y)).
45947   if (IsAllOf && Subtarget.hasSSE41() && IsOneUse) {
45948     MVT TestVT = VecVT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
45949     SDValue BC = peekThroughBitcasts(Vec);
45950     // Ensure MOVMSK was testing every signbit of BC.
45951     if (BC.getValueType().getVectorNumElements() <= NumElts) {
45952       if (BC.getOpcode() == X86ISD::PCMPEQ) {
45953         SDValue V = DAG.getNode(ISD::XOR, SDLoc(BC), BC.getValueType(),
45954                                 BC.getOperand(0), BC.getOperand(1));
45955         V = DAG.getBitcast(TestVT, V);
45956         return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
45957       }
45958       // Check for 256-bit split vector cases.
45959       if (BC.getOpcode() == ISD::AND &&
45960           BC.getOperand(0).getOpcode() == X86ISD::PCMPEQ &&
45961           BC.getOperand(1).getOpcode() == X86ISD::PCMPEQ) {
45962         SDValue LHS = BC.getOperand(0);
45963         SDValue RHS = BC.getOperand(1);
45964         LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), LHS.getValueType(),
45965                           LHS.getOperand(0), LHS.getOperand(1));
45966         RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), RHS.getValueType(),
45967                           RHS.getOperand(0), RHS.getOperand(1));
45968         LHS = DAG.getBitcast(TestVT, LHS);
45969         RHS = DAG.getBitcast(TestVT, RHS);
45970         SDValue V = DAG.getNode(ISD::OR, SDLoc(EFLAGS), TestVT, LHS, RHS);
45971         return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
45972       }
45973     }
45974   }
45975 
45976   // See if we can avoid a PACKSS by calling MOVMSK on the sources.
45977   // For vXi16 cases we can use a v2Xi8 PMOVMSKB. We must mask out
45978   // sign bits prior to the comparison with zero unless we know that
45979   // the vXi16 splats the sign bit down to the lower i8 half.
45980   // TODO: Handle all_of patterns.
45981   if (Vec.getOpcode() == X86ISD::PACKSS && VecVT == MVT::v16i8) {
45982     SDValue VecOp0 = Vec.getOperand(0);
45983     SDValue VecOp1 = Vec.getOperand(1);
45984     bool SignExt0 = DAG.ComputeNumSignBits(VecOp0) > 8;
45985     bool SignExt1 = DAG.ComputeNumSignBits(VecOp1) > 8;
45986     // PMOVMSKB(PACKSSBW(X, undef)) -> PMOVMSKB(BITCAST_v16i8(X)) & 0xAAAA.
45987     if (IsAnyOf && CmpBits == 8 && VecOp1.isUndef()) {
45988       SDLoc DL(EFLAGS);
45989       SDValue Result = DAG.getBitcast(MVT::v16i8, VecOp0);
45990       Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
45991       Result = DAG.getZExtOrTrunc(Result, DL, MVT::i16);
45992       if (!SignExt0) {
45993         Result = DAG.getNode(ISD::AND, DL, MVT::i16, Result,
45994                              DAG.getConstant(0xAAAA, DL, MVT::i16));
45995       }
45996       return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
45997                          DAG.getConstant(0, DL, MVT::i16));
45998     }
45999     // PMOVMSKB(PACKSSBW(LO(X), HI(X)))
46000     // -> PMOVMSKB(BITCAST_v32i8(X)) & 0xAAAAAAAA.
46001     if (CmpBits >= 16 && Subtarget.hasInt256() &&
46002         (IsAnyOf || (SignExt0 && SignExt1))) {
46003       if (SDValue Src = getSplitVectorSrc(VecOp0, VecOp1, true)) {
46004         SDLoc DL(EFLAGS);
46005         SDValue Result = peekThroughBitcasts(Src);
46006         if (IsAllOf && Result.getOpcode() == X86ISD::PCMPEQ &&
46007             Result.getValueType().getVectorNumElements() <= NumElts) {
46008           SDValue V = DAG.getNode(ISD::XOR, DL, Result.getValueType(),
46009                                   Result.getOperand(0), Result.getOperand(1));
46010           V = DAG.getBitcast(MVT::v4i64, V);
46011           return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
46012         }
46013         Result = DAG.getBitcast(MVT::v32i8, Result);
46014         Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
46015         unsigned CmpMask = IsAnyOf ? 0 : 0xFFFFFFFF;
46016         if (!SignExt0 || !SignExt1) {
46017           assert(IsAnyOf &&
46018                  "Only perform v16i16 signmasks for any_of patterns");
46019           Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result,
46020                                DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
46021         }
46022         return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
46023                            DAG.getConstant(CmpMask, DL, MVT::i32));
46024       }
46025     }
46026   }
46027 
46028   // MOVMSK(SHUFFLE(X,u)) -> MOVMSK(X) iff every element is referenced.
46029   // Since we peek through a bitcast, we need to be careful if the base vector
46030   // type has smaller elements than the MOVMSK type.  In that case, even if
46031   // all the elements are demanded by the shuffle mask, only the "high"
46032   // elements which have highbits that align with highbits in the MOVMSK vec
46033   // elements are actually demanded. A simplification of spurious operations
46034   // on the "low" elements take place during other simplifications.
46035   //
46036   // For example:
46037   // MOVMSK64(BITCAST(SHUF32 X, (1,0,3,2))) even though all the elements are
46038   // demanded, because we are swapping around the result can change.
46039   //
46040   // To address this, we check that we can scale the shuffle mask to MOVMSK
46041   // element width (this will ensure "high" elements match). Its slightly overly
46042   // conservative, but fine for an edge case fold.
46043   SmallVector<int, 32> ShuffleMask, ScaledMaskUnused;
46044   SmallVector<SDValue, 2> ShuffleInputs;
46045   if (NumElts <= CmpBits &&
46046       getTargetShuffleInputs(peekThroughBitcasts(Vec), ShuffleInputs,
46047                              ShuffleMask, DAG) &&
46048       ShuffleInputs.size() == 1 && !isAnyZeroOrUndef(ShuffleMask) &&
46049       ShuffleInputs[0].getValueSizeInBits() == VecVT.getSizeInBits() &&
46050       scaleShuffleElements(ShuffleMask, NumElts, ScaledMaskUnused)) {
46051     unsigned NumShuffleElts = ShuffleMask.size();
46052     APInt DemandedElts = APInt::getZero(NumShuffleElts);
46053     for (int M : ShuffleMask) {
46054       assert(0 <= M && M < (int)NumShuffleElts && "Bad unary shuffle index");
46055       DemandedElts.setBit(M);
46056     }
46057     if (DemandedElts.isAllOnes()) {
46058       SDLoc DL(EFLAGS);
46059       SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
46060       Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
46061       Result =
46062           DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType());
46063       return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
46064                          EFLAGS.getOperand(1));
46065     }
46066   }
46067 
46068   // MOVMSKPS(V) !=/== 0 -> TESTPS(V,V)
46069   // MOVMSKPD(V) !=/== 0 -> TESTPD(V,V)
46070   // MOVMSKPS(V) !=/== -1 -> TESTPS(V,V)
46071   // MOVMSKPD(V) !=/== -1 -> TESTPD(V,V)
46072   // iff every element is referenced.
46073   if (NumElts <= CmpBits && Subtarget.hasAVX() &&
46074       !Subtarget.preferMovmskOverVTest() && IsOneUse &&
46075       (NumEltBits == 32 || NumEltBits == 64)) {
46076     SDLoc DL(EFLAGS);
46077     MVT FloatSVT = MVT::getFloatingPointVT(NumEltBits);
46078     MVT FloatVT = MVT::getVectorVT(FloatSVT, NumElts);
46079     MVT IntVT = FloatVT.changeVectorElementTypeToInteger();
46080     SDValue LHS = Vec;
46081     SDValue RHS = IsAnyOf ? Vec : DAG.getAllOnesConstant(DL, IntVT);
46082     CC = IsAnyOf ? CC : (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
46083     return DAG.getNode(X86ISD::TESTP, DL, MVT::i32,
46084                        DAG.getBitcast(FloatVT, LHS),
46085                        DAG.getBitcast(FloatVT, RHS));
46086   }
46087 
46088   return SDValue();
46089 }
46090 
46091 /// Optimize an EFLAGS definition used according to the condition code \p CC
46092 /// into a simpler EFLAGS value, potentially returning a new \p CC and replacing
46093 /// uses of chain values.
46094 static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
46095                                   SelectionDAG &DAG,
46096                                   const X86Subtarget &Subtarget) {
46097   if (CC == X86::COND_B)
46098     if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
46099       return Flags;
46100 
46101   if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
46102     return R;
46103 
46104   if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG, Subtarget))
46105     return R;
46106 
46107   if (SDValue R = combineSetCCMOVMSK(EFLAGS, CC, DAG, Subtarget))
46108     return R;
46109 
46110   return combineSetCCAtomicArith(EFLAGS, CC, DAG, Subtarget);
46111 }
46112 
46113 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
46114 static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
46115                            TargetLowering::DAGCombinerInfo &DCI,
46116                            const X86Subtarget &Subtarget) {
46117   SDLoc DL(N);
46118 
46119   SDValue FalseOp = N->getOperand(0);
46120   SDValue TrueOp = N->getOperand(1);
46121   X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
46122   SDValue Cond = N->getOperand(3);
46123 
46124   // cmov X, X, ?, ? --> X
46125   if (TrueOp == FalseOp)
46126     return TrueOp;
46127 
46128   // Try to simplify the EFLAGS and condition code operands.
46129   // We can't always do this as FCMOV only supports a subset of X86 cond.
46130   if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
46131     if (!(FalseOp.getValueType() == MVT::f80 ||
46132           (FalseOp.getValueType() == MVT::f64 && !Subtarget.hasSSE2()) ||
46133           (FalseOp.getValueType() == MVT::f32 && !Subtarget.hasSSE1())) ||
46134         !Subtarget.canUseCMOV() || hasFPCMov(CC)) {
46135       SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
46136                        Flags};
46137       return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
46138     }
46139   }
46140 
46141   // If this is a select between two integer constants, try to do some
46142   // optimizations.  Note that the operands are ordered the opposite of SELECT
46143   // operands.
46144   if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) {
46145     if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) {
46146       // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
46147       // larger than FalseC (the false value).
46148       if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
46149         CC = X86::GetOppositeBranchCondition(CC);
46150         std::swap(TrueC, FalseC);
46151         std::swap(TrueOp, FalseOp);
46152       }
46153 
46154       // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3.  Likewise for any pow2/0.
46155       // This is efficient for any integer data type (including i8/i16) and
46156       // shift amount.
46157       if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
46158         Cond = getSETCC(CC, Cond, DL, DAG);
46159 
46160         // Zero extend the condition if needed.
46161         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
46162 
46163         unsigned ShAmt = TrueC->getAPIntValue().logBase2();
46164         Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
46165                            DAG.getConstant(ShAmt, DL, MVT::i8));
46166         return Cond;
46167       }
46168 
46169       // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.  This is efficient
46170       // for any integer data type, including i8/i16.
46171       if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
46172         Cond = getSETCC(CC, Cond, DL, DAG);
46173 
46174         // Zero extend the condition if needed.
46175         Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
46176                            FalseC->getValueType(0), Cond);
46177         Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
46178                            SDValue(FalseC, 0));
46179         return Cond;
46180       }
46181 
46182       // Optimize cases that will turn into an LEA instruction.  This requires
46183       // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
46184       if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
46185         APInt Diff = TrueC->getAPIntValue() - FalseC->getAPIntValue();
46186         assert(Diff.getBitWidth() == N->getValueType(0).getSizeInBits() &&
46187                "Implicit constant truncation");
46188 
46189         bool isFastMultiplier = false;
46190         if (Diff.ult(10)) {
46191           switch (Diff.getZExtValue()) {
46192           default: break;
46193           case 1:  // result = add base, cond
46194           case 2:  // result = lea base(    , cond*2)
46195           case 3:  // result = lea base(cond, cond*2)
46196           case 4:  // result = lea base(    , cond*4)
46197           case 5:  // result = lea base(cond, cond*4)
46198           case 8:  // result = lea base(    , cond*8)
46199           case 9:  // result = lea base(cond, cond*8)
46200             isFastMultiplier = true;
46201             break;
46202           }
46203         }
46204 
46205         if (isFastMultiplier) {
46206           Cond = getSETCC(CC, Cond, DL ,DAG);
46207           // Zero extend the condition if needed.
46208           Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
46209                              Cond);
46210           // Scale the condition by the difference.
46211           if (Diff != 1)
46212             Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
46213                                DAG.getConstant(Diff, DL, Cond.getValueType()));
46214 
46215           // Add the base if non-zero.
46216           if (FalseC->getAPIntValue() != 0)
46217             Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
46218                                SDValue(FalseC, 0));
46219           return Cond;
46220         }
46221       }
46222     }
46223   }
46224 
46225   // Handle these cases:
46226   //   (select (x != c), e, c) -> select (x != c), e, x),
46227   //   (select (x == c), c, e) -> select (x == c), x, e)
46228   // where the c is an integer constant, and the "select" is the combination
46229   // of CMOV and CMP.
46230   //
46231   // The rationale for this change is that the conditional-move from a constant
46232   // needs two instructions, however, conditional-move from a register needs
46233   // only one instruction.
46234   //
46235   // CAVEAT: By replacing a constant with a symbolic value, it may obscure
46236   //  some instruction-combining opportunities. This opt needs to be
46237   //  postponed as late as possible.
46238   //
46239   if (!DCI.isBeforeLegalize() && !DCI.isBeforeLegalizeOps()) {
46240     // the DCI.xxxx conditions are provided to postpone the optimization as
46241     // late as possible.
46242 
46243     ConstantSDNode *CmpAgainst = nullptr;
46244     if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
46245         (CmpAgainst = dyn_cast<ConstantSDNode>(Cond.getOperand(1))) &&
46246         !isa<ConstantSDNode>(Cond.getOperand(0))) {
46247 
46248       if (CC == X86::COND_NE &&
46249           CmpAgainst == dyn_cast<ConstantSDNode>(FalseOp)) {
46250         CC = X86::GetOppositeBranchCondition(CC);
46251         std::swap(TrueOp, FalseOp);
46252       }
46253 
46254       if (CC == X86::COND_E &&
46255           CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
46256         SDValue Ops[] = {FalseOp, Cond.getOperand(0),
46257                          DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
46258         return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
46259       }
46260     }
46261   }
46262 
46263   // Transform:
46264   //
46265   //   (cmov 1 T (uge T 2))
46266   //
46267   // to:
46268   //
46269   //   (adc T 0 (sub T 1))
46270   if (CC == X86::COND_AE && isOneConstant(FalseOp) &&
46271       Cond.getOpcode() == X86ISD::SUB && Cond->hasOneUse()) {
46272     SDValue Cond0 = Cond.getOperand(0);
46273     if (Cond0.getOpcode() == ISD::TRUNCATE)
46274       Cond0 = Cond0.getOperand(0);
46275     auto *Sub1C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
46276     if (Cond0 == TrueOp && Sub1C && Sub1C->getZExtValue() == 2) {
46277       EVT CondVT = Cond->getValueType(0);
46278       EVT OuterVT = N->getValueType(0);
46279       // Subtract 1 and generate a carry.
46280       SDValue NewSub =
46281           DAG.getNode(X86ISD::SUB, DL, Cond->getVTList(), Cond.getOperand(0),
46282                       DAG.getConstant(1, DL, CondVT));
46283       SDValue EFLAGS(NewSub.getNode(), 1);
46284       return DAG.getNode(X86ISD::ADC, DL, DAG.getVTList(OuterVT, MVT::i32),
46285                          TrueOp, DAG.getConstant(0, DL, OuterVT), EFLAGS);
46286     }
46287   }
46288 
46289   // Fold and/or of setcc's to double CMOV:
46290   //   (CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
46291   //   (CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
46292   //
46293   // This combine lets us generate:
46294   //   cmovcc1 (jcc1 if we don't have CMOV)
46295   //   cmovcc2 (same)
46296   // instead of:
46297   //   setcc1
46298   //   setcc2
46299   //   and/or
46300   //   cmovne (jne if we don't have CMOV)
46301   // When we can't use the CMOV instruction, it might increase branch
46302   // mispredicts.
46303   // When we can use CMOV, or when there is no mispredict, this improves
46304   // throughput and reduces register pressure.
46305   //
46306   if (CC == X86::COND_NE) {
46307     SDValue Flags;
46308     X86::CondCode CC0, CC1;
46309     bool isAndSetCC;
46310     if (checkBoolTestAndOrSetCCCombine(Cond, CC0, CC1, Flags, isAndSetCC)) {
46311       if (isAndSetCC) {
46312         std::swap(FalseOp, TrueOp);
46313         CC0 = X86::GetOppositeBranchCondition(CC0);
46314         CC1 = X86::GetOppositeBranchCondition(CC1);
46315       }
46316 
46317       SDValue LOps[] = {FalseOp, TrueOp,
46318                         DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
46319       SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
46320       SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
46321                        Flags};
46322       SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
46323       return CMOV;
46324     }
46325   }
46326 
46327   // Fold (CMOV C1, (ADD (CTTZ X), C2), (X != 0)) ->
46328   //      (ADD (CMOV C1-C2, (CTTZ X), (X != 0)), C2)
46329   // Or (CMOV (ADD (CTTZ X), C2), C1, (X == 0)) ->
46330   //    (ADD (CMOV (CTTZ X), C1-C2, (X == 0)), C2)
46331   if ((CC == X86::COND_NE || CC == X86::COND_E) &&
46332       Cond.getOpcode() == X86ISD::CMP && isNullConstant(Cond.getOperand(1))) {
46333     SDValue Add = TrueOp;
46334     SDValue Const = FalseOp;
46335     // Canonicalize the condition code for easier matching and output.
46336     if (CC == X86::COND_E)
46337       std::swap(Add, Const);
46338 
46339     // We might have replaced the constant in the cmov with the LHS of the
46340     // compare. If so change it to the RHS of the compare.
46341     if (Const == Cond.getOperand(0))
46342       Const = Cond.getOperand(1);
46343 
46344     // Ok, now make sure that Add is (add (cttz X), C2) and Const is a constant.
46345     if (isa<ConstantSDNode>(Const) && Add.getOpcode() == ISD::ADD &&
46346         Add.hasOneUse() && isa<ConstantSDNode>(Add.getOperand(1)) &&
46347         (Add.getOperand(0).getOpcode() == ISD::CTTZ_ZERO_UNDEF ||
46348          Add.getOperand(0).getOpcode() == ISD::CTTZ) &&
46349         Add.getOperand(0).getOperand(0) == Cond.getOperand(0)) {
46350       EVT VT = N->getValueType(0);
46351       // This should constant fold.
46352       SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
46353       SDValue CMov =
46354           DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
46355                       DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
46356       return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));
46357     }
46358   }
46359 
46360   return SDValue();
46361 }
46362 
46363 /// Different mul shrinking modes.
46364 enum class ShrinkMode { MULS8, MULU8, MULS16, MULU16 };
46365 
46366 static bool canReduceVMulWidth(SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) {
46367   EVT VT = N->getOperand(0).getValueType();
46368   if (VT.getScalarSizeInBits() != 32)
46369     return false;
46370 
46371   assert(N->getNumOperands() == 2 && "NumOperands of Mul are 2");
46372   unsigned SignBits[2] = {1, 1};
46373   bool IsPositive[2] = {false, false};
46374   for (unsigned i = 0; i < 2; i++) {
46375     SDValue Opd = N->getOperand(i);
46376 
46377     SignBits[i] = DAG.ComputeNumSignBits(Opd);
46378     IsPositive[i] = DAG.SignBitIsZero(Opd);
46379   }
46380 
46381   bool AllPositive = IsPositive[0] && IsPositive[1];
46382   unsigned MinSignBits = std::min(SignBits[0], SignBits[1]);
46383   // When ranges are from -128 ~ 127, use MULS8 mode.
46384   if (MinSignBits >= 25)
46385     Mode = ShrinkMode::MULS8;
46386   // When ranges are from 0 ~ 255, use MULU8 mode.
46387   else if (AllPositive && MinSignBits >= 24)
46388     Mode = ShrinkMode::MULU8;
46389   // When ranges are from -32768 ~ 32767, use MULS16 mode.
46390   else if (MinSignBits >= 17)
46391     Mode = ShrinkMode::MULS16;
46392   // When ranges are from 0 ~ 65535, use MULU16 mode.
46393   else if (AllPositive && MinSignBits >= 16)
46394     Mode = ShrinkMode::MULU16;
46395   else
46396     return false;
46397   return true;
46398 }
46399 
46400 /// When the operands of vector mul are extended from smaller size values,
46401 /// like i8 and i16, the type of mul may be shrinked to generate more
46402 /// efficient code. Two typical patterns are handled:
46403 /// Pattern1:
46404 ///     %2 = sext/zext <N x i8> %1 to <N x i32>
46405 ///     %4 = sext/zext <N x i8> %3 to <N x i32>
46406 //   or %4 = build_vector <N x i32> %C1, ..., %CN (%C1..%CN are constants)
46407 ///     %5 = mul <N x i32> %2, %4
46408 ///
46409 /// Pattern2:
46410 ///     %2 = zext/sext <N x i16> %1 to <N x i32>
46411 ///     %4 = zext/sext <N x i16> %3 to <N x i32>
46412 ///  or %4 = build_vector <N x i32> %C1, ..., %CN (%C1..%CN are constants)
46413 ///     %5 = mul <N x i32> %2, %4
46414 ///
46415 /// There are four mul shrinking modes:
46416 /// If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is
46417 /// -128 to 128, and the scalar value range of %4 is also -128 to 128,
46418 /// generate pmullw+sext32 for it (MULS8 mode).
46419 /// If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is
46420 /// 0 to 255, and the scalar value range of %4 is also 0 to 255,
46421 /// generate pmullw+zext32 for it (MULU8 mode).
46422 /// If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is
46423 /// -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767,
46424 /// generate pmullw+pmulhw for it (MULS16 mode).
46425 /// If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is
46426 /// 0 to 65535, and the scalar value range of %4 is also 0 to 65535,
46427 /// generate pmullw+pmulhuw for it (MULU16 mode).
46428 static SDValue reduceVMULWidth(SDNode *N, SelectionDAG &DAG,
46429                                const X86Subtarget &Subtarget) {
46430   // Check for legality
46431   // pmullw/pmulhw are not supported by SSE.
46432   if (!Subtarget.hasSSE2())
46433     return SDValue();
46434 
46435   // Check for profitability
46436   // pmulld is supported since SSE41. It is better to use pmulld
46437   // instead of pmullw+pmulhw, except for subtargets where pmulld is slower than
46438   // the expansion.
46439   bool OptForMinSize = DAG.getMachineFunction().getFunction().hasMinSize();
46440   if (Subtarget.hasSSE41() && (OptForMinSize || !Subtarget.isPMULLDSlow()))
46441     return SDValue();
46442 
46443   ShrinkMode Mode;
46444   if (!canReduceVMulWidth(N, DAG, Mode))
46445     return SDValue();
46446 
46447   SDLoc DL(N);
46448   SDValue N0 = N->getOperand(0);
46449   SDValue N1 = N->getOperand(1);
46450   EVT VT = N->getOperand(0).getValueType();
46451   unsigned NumElts = VT.getVectorNumElements();
46452   if ((NumElts % 2) != 0)
46453     return SDValue();
46454 
46455   EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts);
46456 
46457   // Shrink the operands of mul.
46458   SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
46459   SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
46460 
46461   // Generate the lower part of mul: pmullw. For MULU8/MULS8, only the
46462   // lower part is needed.
46463   SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
46464   if (Mode == ShrinkMode::MULU8 || Mode == ShrinkMode::MULS8)
46465     return DAG.getNode((Mode == ShrinkMode::MULU8) ? ISD::ZERO_EXTEND
46466                                                    : ISD::SIGN_EXTEND,
46467                        DL, VT, MulLo);
46468 
46469   EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts / 2);
46470   // Generate the higher part of mul: pmulhw/pmulhuw. For MULU16/MULS16,
46471   // the higher part is also needed.
46472   SDValue MulHi =
46473       DAG.getNode(Mode == ShrinkMode::MULS16 ? ISD::MULHS : ISD::MULHU, DL,
46474                   ReducedVT, NewN0, NewN1);
46475 
46476   // Repack the lower part and higher part result of mul into a wider
46477   // result.
46478   // Generate shuffle functioning as punpcklwd.
46479   SmallVector<int, 16> ShuffleMask(NumElts);
46480   for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
46481     ShuffleMask[2 * i] = i;
46482     ShuffleMask[2 * i + 1] = i + NumElts;
46483   }
46484   SDValue ResLo =
46485       DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
46486   ResLo = DAG.getBitcast(ResVT, ResLo);
46487   // Generate shuffle functioning as punpckhwd.
46488   for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
46489     ShuffleMask[2 * i] = i + NumElts / 2;
46490     ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2;
46491   }
46492   SDValue ResHi =
46493       DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
46494   ResHi = DAG.getBitcast(ResVT, ResHi);
46495   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
46496 }
46497 
46498 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
46499                                  EVT VT, const SDLoc &DL) {
46500 
46501   auto combineMulShlAddOrSub = [&](int Mult, int Shift, bool isAdd) {
46502     SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
46503                                  DAG.getConstant(Mult, DL, VT));
46504     Result = DAG.getNode(ISD::SHL, DL, VT, Result,
46505                          DAG.getConstant(Shift, DL, MVT::i8));
46506     Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
46507                          N->getOperand(0));
46508     return Result;
46509   };
46510 
46511   auto combineMulMulAddOrSub = [&](int Mul1, int Mul2, bool isAdd) {
46512     SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
46513                                  DAG.getConstant(Mul1, DL, VT));
46514     Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result,
46515                          DAG.getConstant(Mul2, DL, VT));
46516     Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
46517                          N->getOperand(0));
46518     return Result;
46519   };
46520 
46521   switch (MulAmt) {
46522   default:
46523     break;
46524   case 11:
46525     // mul x, 11 => add ((shl (mul x, 5), 1), x)
46526     return combineMulShlAddOrSub(5, 1, /*isAdd*/ true);
46527   case 21:
46528     // mul x, 21 => add ((shl (mul x, 5), 2), x)
46529     return combineMulShlAddOrSub(5, 2, /*isAdd*/ true);
46530   case 41:
46531     // mul x, 41 => add ((shl (mul x, 5), 3), x)
46532     return combineMulShlAddOrSub(5, 3, /*isAdd*/ true);
46533   case 22:
46534     // mul x, 22 => add (add ((shl (mul x, 5), 2), x), x)
46535     return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
46536                        combineMulShlAddOrSub(5, 2, /*isAdd*/ true));
46537   case 19:
46538     // mul x, 19 => add ((shl (mul x, 9), 1), x)
46539     return combineMulShlAddOrSub(9, 1, /*isAdd*/ true);
46540   case 37:
46541     // mul x, 37 => add ((shl (mul x, 9), 2), x)
46542     return combineMulShlAddOrSub(9, 2, /*isAdd*/ true);
46543   case 73:
46544     // mul x, 73 => add ((shl (mul x, 9), 3), x)
46545     return combineMulShlAddOrSub(9, 3, /*isAdd*/ true);
46546   case 13:
46547     // mul x, 13 => add ((shl (mul x, 3), 2), x)
46548     return combineMulShlAddOrSub(3, 2, /*isAdd*/ true);
46549   case 23:
46550     // mul x, 23 => sub ((shl (mul x, 3), 3), x)
46551     return combineMulShlAddOrSub(3, 3, /*isAdd*/ false);
46552   case 26:
46553     // mul x, 26 => add ((mul (mul x, 5), 5), x)
46554     return combineMulMulAddOrSub(5, 5, /*isAdd*/ true);
46555   case 28:
46556     // mul x, 28 => add ((mul (mul x, 9), 3), x)
46557     return combineMulMulAddOrSub(9, 3, /*isAdd*/ true);
46558   case 29:
46559     // mul x, 29 => add (add ((mul (mul x, 9), 3), x), x)
46560     return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
46561                        combineMulMulAddOrSub(9, 3, /*isAdd*/ true));
46562   }
46563 
46564   // Another trick. If this is a power 2 + 2/4/8, we can use a shift followed
46565   // by a single LEA.
46566   // First check if this a sum of two power of 2s because that's easy. Then
46567   // count how many zeros are up to the first bit.
46568   // TODO: We can do this even without LEA at a cost of two shifts and an add.
46569   if (isPowerOf2_64(MulAmt & (MulAmt - 1))) {
46570     unsigned ScaleShift = llvm::countr_zero(MulAmt);
46571     if (ScaleShift >= 1 && ScaleShift < 4) {
46572       unsigned ShiftAmt = Log2_64((MulAmt & (MulAmt - 1)));
46573       SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46574                                    DAG.getConstant(ShiftAmt, DL, MVT::i8));
46575       SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46576                                    DAG.getConstant(ScaleShift, DL, MVT::i8));
46577       return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);
46578     }
46579   }
46580 
46581   return SDValue();
46582 }
46583 
46584 // If the upper 17 bits of either element are zero and the other element are
46585 // zero/sign bits then we can use PMADDWD, which is always at least as quick as
46586 // PMULLD, except on KNL.
46587 static SDValue combineMulToPMADDWD(SDNode *N, SelectionDAG &DAG,
46588                                    const X86Subtarget &Subtarget) {
46589   if (!Subtarget.hasSSE2())
46590     return SDValue();
46591 
46592   if (Subtarget.isPMADDWDSlow())
46593     return SDValue();
46594 
46595   EVT VT = N->getValueType(0);
46596 
46597   // Only support vXi32 vectors.
46598   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32)
46599     return SDValue();
46600 
46601   // Make sure the type is legal or can split/widen to a legal type.
46602   // With AVX512 but without BWI, we would need to split v32i16.
46603   unsigned NumElts = VT.getVectorNumElements();
46604   if (NumElts == 1 || !isPowerOf2_32(NumElts))
46605     return SDValue();
46606 
46607   // With AVX512 but without BWI, we would need to split v32i16.
46608   if (32 <= (2 * NumElts) && Subtarget.hasAVX512() && !Subtarget.hasBWI())
46609     return SDValue();
46610 
46611   SDValue N0 = N->getOperand(0);
46612   SDValue N1 = N->getOperand(1);
46613 
46614   // If we are zero/sign extending two steps without SSE4.1, its better to
46615   // reduce the vmul width instead.
46616   if (!Subtarget.hasSSE41() &&
46617       (((N0.getOpcode() == ISD::ZERO_EXTEND &&
46618          N0.getOperand(0).getScalarValueSizeInBits() <= 8) &&
46619         (N1.getOpcode() == ISD::ZERO_EXTEND &&
46620          N1.getOperand(0).getScalarValueSizeInBits() <= 8)) ||
46621        ((N0.getOpcode() == ISD::SIGN_EXTEND &&
46622          N0.getOperand(0).getScalarValueSizeInBits() <= 8) &&
46623         (N1.getOpcode() == ISD::SIGN_EXTEND &&
46624          N1.getOperand(0).getScalarValueSizeInBits() <= 8))))
46625     return SDValue();
46626 
46627   // If we are sign extending a wide vector without SSE4.1, its better to reduce
46628   // the vmul width instead.
46629   if (!Subtarget.hasSSE41() &&
46630       (N0.getOpcode() == ISD::SIGN_EXTEND &&
46631        N0.getOperand(0).getValueSizeInBits() > 128) &&
46632       (N1.getOpcode() == ISD::SIGN_EXTEND &&
46633        N1.getOperand(0).getValueSizeInBits() > 128))
46634     return SDValue();
46635 
46636   // Sign bits must extend down to the lowest i16.
46637   if (DAG.ComputeMaxSignificantBits(N1) > 16 ||
46638       DAG.ComputeMaxSignificantBits(N0) > 16)
46639     return SDValue();
46640 
46641   // At least one of the elements must be zero in the upper 17 bits, or can be
46642   // safely made zero without altering the final result.
46643   auto GetZeroableOp = [&](SDValue Op) {
46644     APInt Mask17 = APInt::getHighBitsSet(32, 17);
46645     if (DAG.MaskedValueIsZero(Op, Mask17))
46646       return Op;
46647     // Mask off upper 16-bits of sign-extended constants.
46648     if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()))
46649       return DAG.getNode(ISD::AND, SDLoc(N), VT, Op,
46650                          DAG.getConstant(0xFFFF, SDLoc(N), VT));
46651     if (Op.getOpcode() == ISD::SIGN_EXTEND && N->isOnlyUserOf(Op.getNode())) {
46652       SDValue Src = Op.getOperand(0);
46653       // Convert sext(vXi16) to zext(vXi16).
46654       if (Src.getScalarValueSizeInBits() == 16 && VT.getSizeInBits() <= 128)
46655         return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Src);
46656       // Convert sext(vXi8) to zext(vXi16 sext(vXi8)) on pre-SSE41 targets
46657       // which will expand the extension.
46658       if (Src.getScalarValueSizeInBits() < 16 && !Subtarget.hasSSE41()) {
46659         EVT ExtVT = VT.changeVectorElementType(MVT::i16);
46660         Src = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), ExtVT, Src);
46661         return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Src);
46662       }
46663     }
46664     // Convert SIGN_EXTEND_VECTOR_INREG to ZEXT_EXTEND_VECTOR_INREG.
46665     if (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG &&
46666         N->isOnlyUserOf(Op.getNode())) {
46667       SDValue Src = Op.getOperand(0);
46668       if (Src.getScalarValueSizeInBits() == 16)
46669         return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(N), VT, Src);
46670     }
46671     // Convert VSRAI(Op, 16) to VSRLI(Op, 16).
46672     if (Op.getOpcode() == X86ISD::VSRAI && Op.getConstantOperandVal(1) == 16 &&
46673         N->isOnlyUserOf(Op.getNode())) {
46674       return DAG.getNode(X86ISD::VSRLI, SDLoc(N), VT, Op.getOperand(0),
46675                          Op.getOperand(1));
46676     }
46677     return SDValue();
46678   };
46679   SDValue ZeroN0 = GetZeroableOp(N0);
46680   SDValue ZeroN1 = GetZeroableOp(N1);
46681   if (!ZeroN0 && !ZeroN1)
46682     return SDValue();
46683   N0 = ZeroN0 ? ZeroN0 : N0;
46684   N1 = ZeroN1 ? ZeroN1 : N1;
46685 
46686   // Use SplitOpsAndApply to handle AVX splitting.
46687   auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
46688                            ArrayRef<SDValue> Ops) {
46689     MVT ResVT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
46690     MVT OpVT = MVT::getVectorVT(MVT::i16, Ops[0].getValueSizeInBits() / 16);
46691     return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT,
46692                        DAG.getBitcast(OpVT, Ops[0]),
46693                        DAG.getBitcast(OpVT, Ops[1]));
46694   };
46695   return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {N0, N1},
46696                           PMADDWDBuilder);
46697 }
46698 
46699 static SDValue combineMulToPMULDQ(SDNode *N, SelectionDAG &DAG,
46700                                   const X86Subtarget &Subtarget) {
46701   if (!Subtarget.hasSSE2())
46702     return SDValue();
46703 
46704   EVT VT = N->getValueType(0);
46705 
46706   // Only support vXi64 vectors.
46707   if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 ||
46708       VT.getVectorNumElements() < 2 ||
46709       !isPowerOf2_32(VT.getVectorNumElements()))
46710     return SDValue();
46711 
46712   SDValue N0 = N->getOperand(0);
46713   SDValue N1 = N->getOperand(1);
46714 
46715   // MULDQ returns the 64-bit result of the signed multiplication of the lower
46716   // 32-bits. We can lower with this if the sign bits stretch that far.
46717   if (Subtarget.hasSSE41() && DAG.ComputeNumSignBits(N0) > 32 &&
46718       DAG.ComputeNumSignBits(N1) > 32) {
46719     auto PMULDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
46720                             ArrayRef<SDValue> Ops) {
46721       return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops);
46722     };
46723     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 },
46724                             PMULDQBuilder, /*CheckBWI*/false);
46725   }
46726 
46727   // If the upper bits are zero we can use a single pmuludq.
46728   APInt Mask = APInt::getHighBitsSet(64, 32);
46729   if (DAG.MaskedValueIsZero(N0, Mask) && DAG.MaskedValueIsZero(N1, Mask)) {
46730     auto PMULUDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
46731                              ArrayRef<SDValue> Ops) {
46732       return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops);
46733     };
46734     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 },
46735                             PMULUDQBuilder, /*CheckBWI*/false);
46736   }
46737 
46738   return SDValue();
46739 }
46740 
46741 static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
46742                           TargetLowering::DAGCombinerInfo &DCI,
46743                           const X86Subtarget &Subtarget) {
46744   EVT VT = N->getValueType(0);
46745 
46746   if (SDValue V = combineMulToPMADDWD(N, DAG, Subtarget))
46747     return V;
46748 
46749   if (SDValue V = combineMulToPMULDQ(N, DAG, Subtarget))
46750     return V;
46751 
46752   if (DCI.isBeforeLegalize() && VT.isVector())
46753     return reduceVMULWidth(N, DAG, Subtarget);
46754 
46755   // Optimize a single multiply with constant into two operations in order to
46756   // implement it with two cheaper instructions, e.g. LEA + SHL, LEA + LEA.
46757   if (!MulConstantOptimization)
46758     return SDValue();
46759 
46760   // An imul is usually smaller than the alternative sequence.
46761   if (DAG.getMachineFunction().getFunction().hasMinSize())
46762     return SDValue();
46763 
46764   if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
46765     return SDValue();
46766 
46767   if (VT != MVT::i64 && VT != MVT::i32 &&
46768       (!VT.isVector() || !VT.isSimple() || !VT.isInteger()))
46769     return SDValue();
46770 
46771   ConstantSDNode *CNode = isConstOrConstSplat(
46772       N->getOperand(1), /*AllowUndefs*/ true, /*AllowTrunc*/ false);
46773   const APInt *C = nullptr;
46774   if (!CNode) {
46775     if (VT.isVector())
46776       if (auto *RawC = getTargetConstantFromNode(N->getOperand(1)))
46777         if (auto *SplatC = RawC->getSplatValue())
46778           C = &(SplatC->getUniqueInteger());
46779 
46780     if (!C || C->getBitWidth() != VT.getScalarSizeInBits())
46781       return SDValue();
46782   } else {
46783     C = &(CNode->getAPIntValue());
46784   }
46785 
46786   if (isPowerOf2_64(C->getZExtValue()))
46787     return SDValue();
46788 
46789   int64_t SignMulAmt = C->getSExtValue();
46790   assert(SignMulAmt != INT64_MIN && "Int min should have been handled!");
46791   uint64_t AbsMulAmt = SignMulAmt < 0 ? -SignMulAmt : SignMulAmt;
46792 
46793   SDLoc DL(N);
46794   SDValue NewMul = SDValue();
46795   if (VT == MVT::i64 || VT == MVT::i32) {
46796     if (AbsMulAmt == 3 || AbsMulAmt == 5 || AbsMulAmt == 9) {
46797       NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
46798                            DAG.getConstant(AbsMulAmt, DL, VT));
46799       if (SignMulAmt < 0)
46800         NewMul =
46801             DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), NewMul);
46802 
46803       return NewMul;
46804     }
46805 
46806     uint64_t MulAmt1 = 0;
46807     uint64_t MulAmt2 = 0;
46808     if ((AbsMulAmt % 9) == 0) {
46809       MulAmt1 = 9;
46810       MulAmt2 = AbsMulAmt / 9;
46811     } else if ((AbsMulAmt % 5) == 0) {
46812       MulAmt1 = 5;
46813       MulAmt2 = AbsMulAmt / 5;
46814     } else if ((AbsMulAmt % 3) == 0) {
46815       MulAmt1 = 3;
46816       MulAmt2 = AbsMulAmt / 3;
46817     }
46818 
46819     // For negative multiply amounts, only allow MulAmt2 to be a power of 2.
46820     if (MulAmt2 &&
46821         (isPowerOf2_64(MulAmt2) ||
46822          (SignMulAmt >= 0 && (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)))) {
46823 
46824       if (isPowerOf2_64(MulAmt2) && !(SignMulAmt >= 0 && N->hasOneUse() &&
46825                                       N->use_begin()->getOpcode() == ISD::ADD))
46826         // If second multiplifer is pow2, issue it first. We want the multiply
46827         // by 3, 5, or 9 to be folded into the addressing mode unless the lone
46828         // use is an add. Only do this for positive multiply amounts since the
46829         // negate would prevent it from being used as an address mode anyway.
46830         std::swap(MulAmt1, MulAmt2);
46831 
46832       if (isPowerOf2_64(MulAmt1))
46833         NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46834                              DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8));
46835       else
46836         NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
46837                              DAG.getConstant(MulAmt1, DL, VT));
46838 
46839       if (isPowerOf2_64(MulAmt2))
46840         NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
46841                              DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8));
46842       else
46843         NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
46844                              DAG.getConstant(MulAmt2, DL, VT));
46845 
46846       // Negate the result.
46847       if (SignMulAmt < 0)
46848         NewMul =
46849             DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), NewMul);
46850     } else if (!Subtarget.slowLEA())
46851       NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL);
46852   }
46853   if (!NewMul) {
46854     EVT ShiftVT = VT.isVector() ? VT : MVT::i8;
46855     assert(C->getZExtValue() != 0 &&
46856            C->getZExtValue() != maxUIntN(VT.getScalarSizeInBits()) &&
46857            "Both cases that could cause potential overflows should have "
46858            "already been handled.");
46859     if (isPowerOf2_64(AbsMulAmt - 1)) {
46860       // (mul x, 2^N + 1) => (add (shl x, N), x)
46861       NewMul = DAG.getNode(
46862           ISD::ADD, DL, VT, N->getOperand(0),
46863           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46864                       DAG.getConstant(Log2_64(AbsMulAmt - 1), DL, ShiftVT)));
46865       // To negate, subtract the number from zero
46866       if (SignMulAmt < 0)
46867         NewMul =
46868             DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), NewMul);
46869     } else if (isPowerOf2_64(AbsMulAmt + 1)) {
46870       // (mul x, 2^N - 1) => (sub (shl x, N), x)
46871       NewMul =
46872           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46873                       DAG.getConstant(Log2_64(AbsMulAmt + 1), DL, ShiftVT));
46874       // To negate, reverse the operands of the subtract.
46875       if (SignMulAmt < 0)
46876         NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul);
46877       else
46878         NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
46879     } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt - 2) &&
46880                (!VT.isVector() || Subtarget.fastImmVectorShift())) {
46881       // (mul x, 2^N + 2) => (add (shl x, N), (add x, x))
46882       NewMul =
46883           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46884                       DAG.getConstant(Log2_64(AbsMulAmt - 2), DL, ShiftVT));
46885       NewMul = DAG.getNode(
46886           ISD::ADD, DL, VT, NewMul,
46887           DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0)));
46888     } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt + 2) &&
46889                (!VT.isVector() || Subtarget.fastImmVectorShift())) {
46890       // (mul x, 2^N - 2) => (sub (shl x, N), (add x, x))
46891       NewMul =
46892           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46893                       DAG.getConstant(Log2_64(AbsMulAmt + 2), DL, ShiftVT));
46894       NewMul = DAG.getNode(
46895           ISD::SUB, DL, VT, NewMul,
46896           DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0)));
46897     } else if (SignMulAmt >= 0 && VT.isVector() &&
46898                Subtarget.fastImmVectorShift()) {
46899       uint64_t AbsMulAmtLowBit = AbsMulAmt & (-AbsMulAmt);
46900       uint64_t ShiftAmt1;
46901       std::optional<unsigned> Opc;
46902       if (isPowerOf2_64(AbsMulAmt - AbsMulAmtLowBit)) {
46903         ShiftAmt1 = AbsMulAmt - AbsMulAmtLowBit;
46904         Opc = ISD::ADD;
46905       } else if (isPowerOf2_64(AbsMulAmt + AbsMulAmtLowBit)) {
46906         ShiftAmt1 = AbsMulAmt + AbsMulAmtLowBit;
46907         Opc = ISD::SUB;
46908       }
46909 
46910       if (Opc) {
46911         SDValue Shift1 =
46912             DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46913                         DAG.getConstant(Log2_64(ShiftAmt1), DL, ShiftVT));
46914         SDValue Shift2 =
46915             DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
46916                         DAG.getConstant(Log2_64(AbsMulAmtLowBit), DL, ShiftVT));
46917         NewMul = DAG.getNode(*Opc, DL, VT, Shift1, Shift2);
46918       }
46919     }
46920   }
46921 
46922   return NewMul;
46923 }
46924 
46925 // Try to form a MULHU or MULHS node by looking for
46926 // (srl (mul ext, ext), 16)
46927 // TODO: This is X86 specific because we want to be able to handle wide types
46928 // before type legalization. But we can only do it if the vector will be
46929 // legalized via widening/splitting. Type legalization can't handle promotion
46930 // of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
46931 // combiner.
46932 static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG,
46933                                    const X86Subtarget &Subtarget) {
46934   assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
46935            "SRL or SRA node is required here!");
46936   SDLoc DL(N);
46937 
46938   if (!Subtarget.hasSSE2())
46939     return SDValue();
46940 
46941   // The operation feeding into the shift must be a multiply.
46942   SDValue ShiftOperand = N->getOperand(0);
46943   if (ShiftOperand.getOpcode() != ISD::MUL || !ShiftOperand.hasOneUse())
46944     return SDValue();
46945 
46946   // Input type should be at least vXi32.
46947   EVT VT = N->getValueType(0);
46948   if (!VT.isVector() || VT.getVectorElementType().getSizeInBits() < 32)
46949     return SDValue();
46950 
46951   // Need a shift by 16.
46952   APInt ShiftAmt;
46953   if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), ShiftAmt) ||
46954       ShiftAmt != 16)
46955     return SDValue();
46956 
46957   SDValue LHS = ShiftOperand.getOperand(0);
46958   SDValue RHS = ShiftOperand.getOperand(1);
46959 
46960   unsigned ExtOpc = LHS.getOpcode();
46961   if ((ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND) ||
46962       RHS.getOpcode() != ExtOpc)
46963     return SDValue();
46964 
46965   // Peek through the extends.
46966   LHS = LHS.getOperand(0);
46967   RHS = RHS.getOperand(0);
46968 
46969   // Ensure the input types match.
46970   EVT MulVT = LHS.getValueType();
46971   if (MulVT.getVectorElementType() != MVT::i16 || RHS.getValueType() != MulVT)
46972     return SDValue();
46973 
46974   unsigned Opc = ExtOpc == ISD::SIGN_EXTEND ? ISD::MULHS : ISD::MULHU;
46975   SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS);
46976 
46977   ExtOpc = N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
46978   return DAG.getNode(ExtOpc, DL, VT, Mulh);
46979 }
46980 
46981 static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG) {
46982   SDValue N0 = N->getOperand(0);
46983   SDValue N1 = N->getOperand(1);
46984   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
46985   EVT VT = N0.getValueType();
46986 
46987   // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
46988   // since the result of setcc_c is all zero's or all ones.
46989   if (VT.isInteger() && !VT.isVector() &&
46990       N1C && N0.getOpcode() == ISD::AND &&
46991       N0.getOperand(1).getOpcode() == ISD::Constant) {
46992     SDValue N00 = N0.getOperand(0);
46993     APInt Mask = N0.getConstantOperandAPInt(1);
46994     Mask <<= N1C->getAPIntValue();
46995     bool MaskOK = false;
46996     // We can handle cases concerning bit-widening nodes containing setcc_c if
46997     // we carefully interrogate the mask to make sure we are semantics
46998     // preserving.
46999     // The transform is not safe if the result of C1 << C2 exceeds the bitwidth
47000     // of the underlying setcc_c operation if the setcc_c was zero extended.
47001     // Consider the following example:
47002     //   zext(setcc_c)                 -> i32 0x0000FFFF
47003     //   c1                            -> i32 0x0000FFFF
47004     //   c2                            -> i32 0x00000001
47005     //   (shl (and (setcc_c), c1), c2) -> i32 0x0001FFFE
47006     //   (and setcc_c, (c1 << c2))     -> i32 0x0000FFFE
47007     if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
47008       MaskOK = true;
47009     } else if (N00.getOpcode() == ISD::SIGN_EXTEND &&
47010                N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
47011       MaskOK = true;
47012     } else if ((N00.getOpcode() == ISD::ZERO_EXTEND ||
47013                 N00.getOpcode() == ISD::ANY_EXTEND) &&
47014                N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
47015       MaskOK = Mask.isIntN(N00.getOperand(0).getValueSizeInBits());
47016     }
47017     if (MaskOK && Mask != 0) {
47018       SDLoc DL(N);
47019       return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT));
47020     }
47021   }
47022 
47023   return SDValue();
47024 }
47025 
47026 static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
47027                                            const X86Subtarget &Subtarget) {
47028   SDValue N0 = N->getOperand(0);
47029   SDValue N1 = N->getOperand(1);
47030   EVT VT = N0.getValueType();
47031   unsigned Size = VT.getSizeInBits();
47032 
47033   if (SDValue V = combineShiftToPMULH(N, DAG, Subtarget))
47034     return V;
47035 
47036   // fold (ashr (shl, a, [56,48,32,24,16]), SarConst)
47037   // into (shl, (sext (a), [56,48,32,24,16] - SarConst)) or
47038   // into (lshr, (sext (a), SarConst - [56,48,32,24,16]))
47039   // depending on sign of (SarConst - [56,48,32,24,16])
47040 
47041   // sexts in X86 are MOVs. The MOVs have the same code size
47042   // as above SHIFTs (only SHIFT on 1 has lower code size).
47043   // However the MOVs have 2 advantages to a SHIFT:
47044   // 1. MOVs can write to a register that differs from source
47045   // 2. MOVs accept memory operands
47046 
47047   if (VT.isVector() || N1.getOpcode() != ISD::Constant ||
47048       N0.getOpcode() != ISD::SHL || !N0.hasOneUse() ||
47049       N0.getOperand(1).getOpcode() != ISD::Constant)
47050     return SDValue();
47051 
47052   SDValue N00 = N0.getOperand(0);
47053   SDValue N01 = N0.getOperand(1);
47054   APInt ShlConst = N01->getAsAPIntVal();
47055   APInt SarConst = N1->getAsAPIntVal();
47056   EVT CVT = N1.getValueType();
47057 
47058   if (SarConst.isNegative())
47059     return SDValue();
47060 
47061   for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) {
47062     unsigned ShiftSize = SVT.getSizeInBits();
47063     // skipping types without corresponding sext/zext and
47064     // ShlConst that is not one of [56,48,32,24,16]
47065     if (ShiftSize >= Size || ShlConst != Size - ShiftSize)
47066       continue;
47067     SDLoc DL(N);
47068     SDValue NN =
47069         DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT));
47070     SarConst = SarConst - (Size - ShiftSize);
47071     if (SarConst == 0)
47072       return NN;
47073     if (SarConst.isNegative())
47074       return DAG.getNode(ISD::SHL, DL, VT, NN,
47075                          DAG.getConstant(-SarConst, DL, CVT));
47076     return DAG.getNode(ISD::SRA, DL, VT, NN,
47077                        DAG.getConstant(SarConst, DL, CVT));
47078   }
47079   return SDValue();
47080 }
47081 
47082 static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
47083                                         TargetLowering::DAGCombinerInfo &DCI,
47084                                         const X86Subtarget &Subtarget) {
47085   SDValue N0 = N->getOperand(0);
47086   SDValue N1 = N->getOperand(1);
47087   EVT VT = N0.getValueType();
47088 
47089   if (SDValue V = combineShiftToPMULH(N, DAG, Subtarget))
47090     return V;
47091 
47092   // Only do this on the last DAG combine as it can interfere with other
47093   // combines.
47094   if (!DCI.isAfterLegalizeDAG())
47095     return SDValue();
47096 
47097   // Try to improve a sequence of srl (and X, C1), C2 by inverting the order.
47098   // TODO: This is a generic DAG combine that became an x86-only combine to
47099   // avoid shortcomings in other folds such as bswap, bit-test ('bt'), and
47100   // and-not ('andn').
47101   if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
47102     return SDValue();
47103 
47104   auto *ShiftC = dyn_cast<ConstantSDNode>(N1);
47105   auto *AndC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
47106   if (!ShiftC || !AndC)
47107     return SDValue();
47108 
47109   // If we can shrink the constant mask below 8-bits or 32-bits, then this
47110   // transform should reduce code size. It may also enable secondary transforms
47111   // from improved known-bits analysis or instruction selection.
47112   APInt MaskVal = AndC->getAPIntValue();
47113 
47114   // If this can be matched by a zero extend, don't optimize.
47115   if (MaskVal.isMask()) {
47116     unsigned TO = MaskVal.countr_one();
47117     if (TO >= 8 && isPowerOf2_32(TO))
47118       return SDValue();
47119   }
47120 
47121   APInt NewMaskVal = MaskVal.lshr(ShiftC->getAPIntValue());
47122   unsigned OldMaskSize = MaskVal.getSignificantBits();
47123   unsigned NewMaskSize = NewMaskVal.getSignificantBits();
47124   if ((OldMaskSize > 8 && NewMaskSize <= 8) ||
47125       (OldMaskSize > 32 && NewMaskSize <= 32)) {
47126     // srl (and X, AndC), ShiftC --> and (srl X, ShiftC), (AndC >> ShiftC)
47127     SDLoc DL(N);
47128     SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
47129     SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
47130     return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask);
47131   }
47132   return SDValue();
47133 }
47134 
47135 static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG,
47136                                          const X86Subtarget &Subtarget) {
47137   unsigned Opcode = N->getOpcode();
47138   assert(isHorizOp(Opcode) && "Unexpected hadd/hsub/pack opcode");
47139 
47140   SDLoc DL(N);
47141   EVT VT = N->getValueType(0);
47142   SDValue N0 = N->getOperand(0);
47143   SDValue N1 = N->getOperand(1);
47144   EVT SrcVT = N0.getValueType();
47145 
47146   SDValue BC0 =
47147       N->isOnlyUserOf(N0.getNode()) ? peekThroughOneUseBitcasts(N0) : N0;
47148   SDValue BC1 =
47149       N->isOnlyUserOf(N1.getNode()) ? peekThroughOneUseBitcasts(N1) : N1;
47150 
47151   // Attempt to fold HOP(LOSUBVECTOR(SHUFFLE(X)),HISUBVECTOR(SHUFFLE(X)))
47152   // to SHUFFLE(HOP(LOSUBVECTOR(X),HISUBVECTOR(X))), this is mainly for
47153   // truncation trees that help us avoid lane crossing shuffles.
47154   // TODO: There's a lot more we can do for PACK/HADD style shuffle combines.
47155   // TODO: We don't handle vXf64 shuffles yet.
47156   if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) {
47157     if (SDValue BCSrc = getSplitVectorSrc(BC0, BC1, false)) {
47158       SmallVector<SDValue> ShuffleOps;
47159       SmallVector<int> ShuffleMask, ScaledMask;
47160       SDValue Vec = peekThroughBitcasts(BCSrc);
47161       if (getTargetShuffleInputs(Vec, ShuffleOps, ShuffleMask, DAG)) {
47162         resolveTargetShuffleInputsAndMask(ShuffleOps, ShuffleMask);
47163         // To keep the HOP LHS/RHS coherency, we must be able to scale the unary
47164         // shuffle to a v4X64 width - we can probably relax this in the future.
47165         if (!isAnyZero(ShuffleMask) && ShuffleOps.size() == 1 &&
47166             ShuffleOps[0].getValueType().is256BitVector() &&
47167             scaleShuffleElements(ShuffleMask, 4, ScaledMask)) {
47168           SDValue Lo, Hi;
47169           MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;
47170           std::tie(Lo, Hi) = DAG.SplitVector(ShuffleOps[0], DL);
47171           Lo = DAG.getBitcast(SrcVT, Lo);
47172           Hi = DAG.getBitcast(SrcVT, Hi);
47173           SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
47174           Res = DAG.getBitcast(ShufVT, Res);
47175           Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ScaledMask);
47176           return DAG.getBitcast(VT, Res);
47177         }
47178       }
47179     }
47180   }
47181 
47182   // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(Z,W)) -> SHUFFLE(HOP()).
47183   if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) {
47184     // If either/both ops are a shuffle that can scale to v2x64,
47185     // then see if we can perform this as a v4x32 post shuffle.
47186     SmallVector<SDValue> Ops0, Ops1;
47187     SmallVector<int> Mask0, Mask1, ScaledMask0, ScaledMask1;
47188     bool IsShuf0 =
47189         getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) &&
47190         scaleShuffleElements(Mask0, 2, ScaledMask0) &&
47191         all_of(Ops0, [](SDValue Op) { return Op.getValueSizeInBits() == 128; });
47192     bool IsShuf1 =
47193         getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) &&
47194         scaleShuffleElements(Mask1, 2, ScaledMask1) &&
47195         all_of(Ops1, [](SDValue Op) { return Op.getValueSizeInBits() == 128; });
47196     if (IsShuf0 || IsShuf1) {
47197       if (!IsShuf0) {
47198         Ops0.assign({BC0});
47199         ScaledMask0.assign({0, 1});
47200       }
47201       if (!IsShuf1) {
47202         Ops1.assign({BC1});
47203         ScaledMask1.assign({0, 1});
47204       }
47205 
47206       SDValue LHS, RHS;
47207       int PostShuffle[4] = {-1, -1, -1, -1};
47208       auto FindShuffleOpAndIdx = [&](int M, int &Idx, ArrayRef<SDValue> Ops) {
47209         if (M < 0)
47210           return true;
47211         Idx = M % 2;
47212         SDValue Src = Ops[M / 2];
47213         if (!LHS || LHS == Src) {
47214           LHS = Src;
47215           return true;
47216         }
47217         if (!RHS || RHS == Src) {
47218           Idx += 2;
47219           RHS = Src;
47220           return true;
47221         }
47222         return false;
47223       };
47224       if (FindShuffleOpAndIdx(ScaledMask0[0], PostShuffle[0], Ops0) &&
47225           FindShuffleOpAndIdx(ScaledMask0[1], PostShuffle[1], Ops0) &&
47226           FindShuffleOpAndIdx(ScaledMask1[0], PostShuffle[2], Ops1) &&
47227           FindShuffleOpAndIdx(ScaledMask1[1], PostShuffle[3], Ops1)) {
47228         LHS = DAG.getBitcast(SrcVT, LHS);
47229         RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS);
47230         MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;
47231         SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
47232         Res = DAG.getBitcast(ShufVT, Res);
47233         Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, PostShuffle);
47234         return DAG.getBitcast(VT, Res);
47235       }
47236     }
47237   }
47238 
47239   // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(X,Y)) -> SHUFFLE(HOP(X,Y)).
47240   if (VT.is256BitVector() && Subtarget.hasInt256()) {
47241     SmallVector<int> Mask0, Mask1;
47242     SmallVector<SDValue> Ops0, Ops1;
47243     SmallVector<int, 2> ScaledMask0, ScaledMask1;
47244     if (getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) &&
47245         getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) &&
47246         !Ops0.empty() && !Ops1.empty() &&
47247         all_of(Ops0,
47248                [](SDValue Op) { return Op.getValueType().is256BitVector(); }) &&
47249         all_of(Ops1,
47250                [](SDValue Op) { return Op.getValueType().is256BitVector(); }) &&
47251         scaleShuffleElements(Mask0, 2, ScaledMask0) &&
47252         scaleShuffleElements(Mask1, 2, ScaledMask1)) {
47253       SDValue Op00 = peekThroughBitcasts(Ops0.front());
47254       SDValue Op10 = peekThroughBitcasts(Ops1.front());
47255       SDValue Op01 = peekThroughBitcasts(Ops0.back());
47256       SDValue Op11 = peekThroughBitcasts(Ops1.back());
47257       if ((Op00 == Op11) && (Op01 == Op10)) {
47258         std::swap(Op10, Op11);
47259         ShuffleVectorSDNode::commuteMask(ScaledMask1);
47260       }
47261       if ((Op00 == Op10) && (Op01 == Op11)) {
47262         const int Map[4] = {0, 2, 1, 3};
47263         SmallVector<int, 4> ShuffleMask(
47264             {Map[ScaledMask0[0]], Map[ScaledMask1[0]], Map[ScaledMask0[1]],
47265              Map[ScaledMask1[1]]});
47266         MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;
47267         SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00),
47268                                   DAG.getBitcast(SrcVT, Op01));
47269         Res = DAG.getBitcast(ShufVT, Res);
47270         Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ShuffleMask);
47271         return DAG.getBitcast(VT, Res);
47272       }
47273     }
47274   }
47275 
47276   return SDValue();
47277 }
47278 
47279 static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
47280                                  TargetLowering::DAGCombinerInfo &DCI,
47281                                  const X86Subtarget &Subtarget) {
47282   unsigned Opcode = N->getOpcode();
47283   assert((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&
47284          "Unexpected pack opcode");
47285 
47286   EVT VT = N->getValueType(0);
47287   SDValue N0 = N->getOperand(0);
47288   SDValue N1 = N->getOperand(1);
47289   unsigned NumDstElts = VT.getVectorNumElements();
47290   unsigned DstBitsPerElt = VT.getScalarSizeInBits();
47291   unsigned SrcBitsPerElt = 2 * DstBitsPerElt;
47292   assert(N0.getScalarValueSizeInBits() == SrcBitsPerElt &&
47293          N1.getScalarValueSizeInBits() == SrcBitsPerElt &&
47294          "Unexpected PACKSS/PACKUS input type");
47295 
47296   bool IsSigned = (X86ISD::PACKSS == Opcode);
47297 
47298   // Constant Folding.
47299   APInt UndefElts0, UndefElts1;
47300   SmallVector<APInt, 32> EltBits0, EltBits1;
47301   if ((N0.isUndef() || N->isOnlyUserOf(N0.getNode())) &&
47302       (N1.isUndef() || N->isOnlyUserOf(N1.getNode())) &&
47303       getTargetConstantBitsFromNode(N0, SrcBitsPerElt, UndefElts0, EltBits0) &&
47304       getTargetConstantBitsFromNode(N1, SrcBitsPerElt, UndefElts1, EltBits1)) {
47305     unsigned NumLanes = VT.getSizeInBits() / 128;
47306     unsigned NumSrcElts = NumDstElts / 2;
47307     unsigned NumDstEltsPerLane = NumDstElts / NumLanes;
47308     unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
47309 
47310     APInt Undefs(NumDstElts, 0);
47311     SmallVector<APInt, 32> Bits(NumDstElts, APInt::getZero(DstBitsPerElt));
47312     for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
47313       for (unsigned Elt = 0; Elt != NumDstEltsPerLane; ++Elt) {
47314         unsigned SrcIdx = Lane * NumSrcEltsPerLane + Elt % NumSrcEltsPerLane;
47315         auto &UndefElts = (Elt >= NumSrcEltsPerLane ? UndefElts1 : UndefElts0);
47316         auto &EltBits = (Elt >= NumSrcEltsPerLane ? EltBits1 : EltBits0);
47317 
47318         if (UndefElts[SrcIdx]) {
47319           Undefs.setBit(Lane * NumDstEltsPerLane + Elt);
47320           continue;
47321         }
47322 
47323         APInt &Val = EltBits[SrcIdx];
47324         if (IsSigned) {
47325           // PACKSS: Truncate signed value with signed saturation.
47326           // Source values less than dst minint are saturated to minint.
47327           // Source values greater than dst maxint are saturated to maxint.
47328           if (Val.isSignedIntN(DstBitsPerElt))
47329             Val = Val.trunc(DstBitsPerElt);
47330           else if (Val.isNegative())
47331             Val = APInt::getSignedMinValue(DstBitsPerElt);
47332           else
47333             Val = APInt::getSignedMaxValue(DstBitsPerElt);
47334         } else {
47335           // PACKUS: Truncate signed value with unsigned saturation.
47336           // Source values less than zero are saturated to zero.
47337           // Source values greater than dst maxuint are saturated to maxuint.
47338           if (Val.isIntN(DstBitsPerElt))
47339             Val = Val.trunc(DstBitsPerElt);
47340           else if (Val.isNegative())
47341             Val = APInt::getZero(DstBitsPerElt);
47342           else
47343             Val = APInt::getAllOnes(DstBitsPerElt);
47344         }
47345         Bits[Lane * NumDstEltsPerLane + Elt] = Val;
47346       }
47347     }
47348 
47349     return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N));
47350   }
47351 
47352   // Try to fold PACK(SHUFFLE(),SHUFFLE()) -> SHUFFLE(PACK()).
47353   if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
47354     return V;
47355 
47356   // Try to fold PACKSS(NOT(X),NOT(Y)) -> NOT(PACKSS(X,Y)).
47357   // Currently limit this to allsignbits cases only.
47358   if (IsSigned &&
47359       (N0.isUndef() || DAG.ComputeNumSignBits(N0) == SrcBitsPerElt) &&
47360       (N1.isUndef() || DAG.ComputeNumSignBits(N1) == SrcBitsPerElt)) {
47361     SDValue Not0 = N0.isUndef() ? N0 : IsNOT(N0, DAG);
47362     SDValue Not1 = N1.isUndef() ? N1 : IsNOT(N1, DAG);
47363     if (Not0 && Not1) {
47364       SDLoc DL(N);
47365       MVT SrcVT = N0.getSimpleValueType();
47366       SDValue Pack =
47367           DAG.getNode(X86ISD::PACKSS, DL, VT, DAG.getBitcast(SrcVT, Not0),
47368                       DAG.getBitcast(SrcVT, Not1));
47369       return DAG.getNOT(DL, Pack, VT);
47370     }
47371   }
47372 
47373   // Try to combine a PACKUSWB/PACKSSWB implemented truncate with a regular
47374   // truncate to create a larger truncate.
47375   if (Subtarget.hasAVX512() &&
47376       N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 &&
47377       N0.getOperand(0).getValueType() == MVT::v8i32) {
47378     if ((IsSigned && DAG.ComputeNumSignBits(N0) > 8) ||
47379         (!IsSigned &&
47380          DAG.MaskedValueIsZero(N0, APInt::getHighBitsSet(16, 8)))) {
47381       if (Subtarget.hasVLX())
47382         return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0));
47383 
47384       // Widen input to v16i32 so we can truncate that.
47385       SDLoc dl(N);
47386       SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,
47387                                    N0.getOperand(0), DAG.getUNDEF(MVT::v8i32));
47388       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat);
47389     }
47390   }
47391 
47392   // Try to fold PACK(EXTEND(X),EXTEND(Y)) -> CONCAT(X,Y) subvectors.
47393   if (VT.is128BitVector()) {
47394     unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
47395     SDValue Src0, Src1;
47396     if (N0.getOpcode() == ExtOpc &&
47397         N0.getOperand(0).getValueType().is64BitVector() &&
47398         N0.getOperand(0).getScalarValueSizeInBits() == DstBitsPerElt) {
47399       Src0 = N0.getOperand(0);
47400     }
47401     if (N1.getOpcode() == ExtOpc &&
47402         N1.getOperand(0).getValueType().is64BitVector() &&
47403         N1.getOperand(0).getScalarValueSizeInBits() == DstBitsPerElt) {
47404       Src1 = N1.getOperand(0);
47405     }
47406     if ((Src0 || N0.isUndef()) && (Src1 || N1.isUndef())) {
47407       assert((Src0 || Src1) && "Found PACK(UNDEF,UNDEF)");
47408       Src0 = Src0 ? Src0 : DAG.getUNDEF(Src1.getValueType());
47409       Src1 = Src1 ? Src1 : DAG.getUNDEF(Src0.getValueType());
47410       return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1);
47411     }
47412 
47413     // Try again with pack(*_extend_vector_inreg, undef).
47414     unsigned VecInRegOpc = IsSigned ? ISD::SIGN_EXTEND_VECTOR_INREG
47415                                     : ISD::ZERO_EXTEND_VECTOR_INREG;
47416     if (N0.getOpcode() == VecInRegOpc && N1.isUndef() &&
47417         N0.getOperand(0).getScalarValueSizeInBits() < DstBitsPerElt)
47418       return getEXTEND_VECTOR_INREG(ExtOpc, SDLoc(N), VT, N0.getOperand(0),
47419                                     DAG);
47420   }
47421 
47422   // Attempt to combine as shuffle.
47423   SDValue Op(N, 0);
47424   if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
47425     return Res;
47426 
47427   return SDValue();
47428 }
47429 
47430 static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG,
47431                                     TargetLowering::DAGCombinerInfo &DCI,
47432                                     const X86Subtarget &Subtarget) {
47433   assert((X86ISD::HADD == N->getOpcode() || X86ISD::FHADD == N->getOpcode() ||
47434           X86ISD::HSUB == N->getOpcode() || X86ISD::FHSUB == N->getOpcode()) &&
47435          "Unexpected horizontal add/sub opcode");
47436 
47437   if (!shouldUseHorizontalOp(true, DAG, Subtarget)) {
47438     MVT VT = N->getSimpleValueType(0);
47439     SDValue LHS = N->getOperand(0);
47440     SDValue RHS = N->getOperand(1);
47441 
47442     // HOP(HOP'(X,X),HOP'(Y,Y)) -> HOP(PERMUTE(HOP'(X,Y)),PERMUTE(HOP'(X,Y)).
47443     if (LHS != RHS && LHS.getOpcode() == N->getOpcode() &&
47444         LHS.getOpcode() == RHS.getOpcode() &&
47445         LHS.getValueType() == RHS.getValueType() &&
47446         N->isOnlyUserOf(LHS.getNode()) && N->isOnlyUserOf(RHS.getNode())) {
47447       SDValue LHS0 = LHS.getOperand(0);
47448       SDValue LHS1 = LHS.getOperand(1);
47449       SDValue RHS0 = RHS.getOperand(0);
47450       SDValue RHS1 = RHS.getOperand(1);
47451       if ((LHS0 == LHS1 || LHS0.isUndef() || LHS1.isUndef()) &&
47452           (RHS0 == RHS1 || RHS0.isUndef() || RHS1.isUndef())) {
47453         SDLoc DL(N);
47454         SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(),
47455                                   LHS0.isUndef() ? LHS1 : LHS0,
47456                                   RHS0.isUndef() ? RHS1 : RHS0);
47457         MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
47458         Res = DAG.getBitcast(ShufVT, Res);
47459         SDValue NewLHS =
47460             DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
47461                         getV4X86ShuffleImm8ForMask({0, 1, 0, 1}, DL, DAG));
47462         SDValue NewRHS =
47463             DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
47464                         getV4X86ShuffleImm8ForMask({2, 3, 2, 3}, DL, DAG));
47465         return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS),
47466                            DAG.getBitcast(VT, NewRHS));
47467       }
47468     }
47469   }
47470 
47471   // Try to fold HOP(SHUFFLE(),SHUFFLE()) -> SHUFFLE(HOP()).
47472   if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
47473     return V;
47474 
47475   return SDValue();
47476 }
47477 
47478 static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG,
47479                                      TargetLowering::DAGCombinerInfo &DCI,
47480                                      const X86Subtarget &Subtarget) {
47481   assert((X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->getOpcode() ||
47482           X86ISD::VSRL == N->getOpcode()) &&
47483          "Unexpected shift opcode");
47484   EVT VT = N->getValueType(0);
47485   SDValue N0 = N->getOperand(0);
47486   SDValue N1 = N->getOperand(1);
47487 
47488   // Shift zero -> zero.
47489   if (ISD::isBuildVectorAllZeros(N0.getNode()))
47490     return DAG.getConstant(0, SDLoc(N), VT);
47491 
47492   // Detect constant shift amounts.
47493   APInt UndefElts;
47494   SmallVector<APInt, 32> EltBits;
47495   if (getTargetConstantBitsFromNode(N1, 64, UndefElts, EltBits, true, false)) {
47496     unsigned X86Opc = getTargetVShiftUniformOpcode(N->getOpcode(), false);
47497     return getTargetVShiftByConstNode(X86Opc, SDLoc(N), VT.getSimpleVT(), N0,
47498                                       EltBits[0].getZExtValue(), DAG);
47499   }
47500 
47501   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47502   APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
47503   if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
47504     return SDValue(N, 0);
47505 
47506   return SDValue();
47507 }
47508 
47509 static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
47510                                      TargetLowering::DAGCombinerInfo &DCI,
47511                                      const X86Subtarget &Subtarget) {
47512   unsigned Opcode = N->getOpcode();
47513   assert((X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode ||
47514           X86ISD::VSRLI == Opcode) &&
47515          "Unexpected shift opcode");
47516   bool LogicalShift = X86ISD::VSHLI == Opcode || X86ISD::VSRLI == Opcode;
47517   EVT VT = N->getValueType(0);
47518   SDValue N0 = N->getOperand(0);
47519   SDValue N1 = N->getOperand(1);
47520   unsigned NumBitsPerElt = VT.getScalarSizeInBits();
47521   assert(VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 &&
47522          "Unexpected value type");
47523   assert(N1.getValueType() == MVT::i8 && "Unexpected shift amount type");
47524 
47525   // (shift undef, X) -> 0
47526   if (N0.isUndef())
47527     return DAG.getConstant(0, SDLoc(N), VT);
47528 
47529   // Out of range logical bit shifts are guaranteed to be zero.
47530   // Out of range arithmetic bit shifts splat the sign bit.
47531   unsigned ShiftVal = N->getConstantOperandVal(1);
47532   if (ShiftVal >= NumBitsPerElt) {
47533     if (LogicalShift)
47534       return DAG.getConstant(0, SDLoc(N), VT);
47535     ShiftVal = NumBitsPerElt - 1;
47536   }
47537 
47538   // (shift X, 0) -> X
47539   if (!ShiftVal)
47540     return N0;
47541 
47542   // (shift 0, C) -> 0
47543   if (ISD::isBuildVectorAllZeros(N0.getNode()))
47544     // N0 is all zeros or undef. We guarantee that the bits shifted into the
47545     // result are all zeros, not undef.
47546     return DAG.getConstant(0, SDLoc(N), VT);
47547 
47548   // (VSRAI -1, C) -> -1
47549   if (!LogicalShift && ISD::isBuildVectorAllOnes(N0.getNode()))
47550     // N0 is all ones or undef. We guarantee that the bits shifted into the
47551     // result are all ones, not undef.
47552     return DAG.getConstant(-1, SDLoc(N), VT);
47553 
47554   auto MergeShifts = [&](SDValue X, uint64_t Amt0, uint64_t Amt1) {
47555     unsigned NewShiftVal = Amt0 + Amt1;
47556     if (NewShiftVal >= NumBitsPerElt) {
47557       // Out of range logical bit shifts are guaranteed to be zero.
47558       // Out of range arithmetic bit shifts splat the sign bit.
47559       if (LogicalShift)
47560         return DAG.getConstant(0, SDLoc(N), VT);
47561       NewShiftVal = NumBitsPerElt - 1;
47562     }
47563     return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0),
47564                        DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8));
47565   };
47566 
47567   // (shift (shift X, C2), C1) -> (shift X, (C1 + C2))
47568   if (Opcode == N0.getOpcode())
47569     return MergeShifts(N0.getOperand(0), ShiftVal, N0.getConstantOperandVal(1));
47570 
47571   // (shl (add X, X), C) -> (shl X, (C + 1))
47572   if (Opcode == X86ISD::VSHLI && N0.getOpcode() == ISD::ADD &&
47573       N0.getOperand(0) == N0.getOperand(1))
47574     return MergeShifts(N0.getOperand(0), ShiftVal, 1);
47575 
47576   // We can decode 'whole byte' logical bit shifts as shuffles.
47577   if (LogicalShift && (ShiftVal % 8) == 0) {
47578     SDValue Op(N, 0);
47579     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
47580       return Res;
47581   }
47582 
47583   // Attempt to detect an expanded vXi64 SIGN_EXTEND_INREG vXi1 pattern, and
47584   // convert to a splatted v2Xi32 SIGN_EXTEND_INREG pattern:
47585   // psrad(pshufd(psllq(X,63),1,1,3,3),31) ->
47586   // pshufd(psrad(pslld(X,31),31),0,0,2,2).
47587   if (Opcode == X86ISD::VSRAI && NumBitsPerElt == 32 && ShiftVal == 31 &&
47588       N0.getOpcode() == X86ISD::PSHUFD &&
47589       N0.getConstantOperandVal(1) == getV4X86ShuffleImm({1, 1, 3, 3}) &&
47590       N0->hasOneUse()) {
47591     SDValue BC = peekThroughOneUseBitcasts(N0.getOperand(0));
47592     if (BC.getOpcode() == X86ISD::VSHLI &&
47593         BC.getScalarValueSizeInBits() == 64 &&
47594         BC.getConstantOperandVal(1) == 63) {
47595       SDLoc DL(N);
47596       SDValue Src = BC.getOperand(0);
47597       Src = DAG.getBitcast(VT, Src);
47598       Src = DAG.getNode(X86ISD::PSHUFD, DL, VT, Src,
47599                         getV4X86ShuffleImm8ForMask({0, 0, 2, 2}, DL, DAG));
47600       Src = DAG.getNode(X86ISD::VSHLI, DL, VT, Src, N1);
47601       Src = DAG.getNode(X86ISD::VSRAI, DL, VT, Src, N1);
47602       return Src;
47603     }
47604   }
47605 
47606   auto TryConstantFold = [&](SDValue V) {
47607     APInt UndefElts;
47608     SmallVector<APInt, 32> EltBits;
47609     if (!getTargetConstantBitsFromNode(V, NumBitsPerElt, UndefElts, EltBits))
47610       return SDValue();
47611     assert(EltBits.size() == VT.getVectorNumElements() &&
47612            "Unexpected shift value type");
47613     // Undef elements need to fold to 0. It's possible SimplifyDemandedBits
47614     // created an undef input due to no input bits being demanded, but user
47615     // still expects 0 in other bits.
47616     for (unsigned i = 0, e = EltBits.size(); i != e; ++i) {
47617       APInt &Elt = EltBits[i];
47618       if (UndefElts[i])
47619         Elt = 0;
47620       else if (X86ISD::VSHLI == Opcode)
47621         Elt <<= ShiftVal;
47622       else if (X86ISD::VSRAI == Opcode)
47623         Elt.ashrInPlace(ShiftVal);
47624       else
47625         Elt.lshrInPlace(ShiftVal);
47626     }
47627     // Reset undef elements since they were zeroed above.
47628     UndefElts = 0;
47629     return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N));
47630   };
47631 
47632   // Constant Folding.
47633   if (N->isOnlyUserOf(N0.getNode())) {
47634     if (SDValue C = TryConstantFold(N0))
47635       return C;
47636 
47637     // Fold (shift (logic X, C2), C1) -> (logic (shift X, C1), (shift C2, C1))
47638     // Don't break NOT patterns.
47639     SDValue BC = peekThroughOneUseBitcasts(N0);
47640     if (ISD::isBitwiseLogicOp(BC.getOpcode()) &&
47641         BC->isOnlyUserOf(BC.getOperand(1).getNode()) &&
47642         !ISD::isBuildVectorAllOnes(BC.getOperand(1).getNode())) {
47643       if (SDValue RHS = TryConstantFold(BC.getOperand(1))) {
47644         SDLoc DL(N);
47645         SDValue LHS = DAG.getNode(Opcode, DL, VT,
47646                                   DAG.getBitcast(VT, BC.getOperand(0)), N1);
47647         return DAG.getNode(BC.getOpcode(), DL, VT, LHS, RHS);
47648       }
47649     }
47650   }
47651 
47652   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47653   if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumBitsPerElt),
47654                                DCI))
47655     return SDValue(N, 0);
47656 
47657   return SDValue();
47658 }
47659 
47660 static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
47661                                    TargetLowering::DAGCombinerInfo &DCI,
47662                                    const X86Subtarget &Subtarget) {
47663   EVT VT = N->getValueType(0);
47664   unsigned Opcode = N->getOpcode();
47665   assert(((Opcode == X86ISD::PINSRB && VT == MVT::v16i8) ||
47666           (Opcode == X86ISD::PINSRW && VT == MVT::v8i16) ||
47667           Opcode == ISD::INSERT_VECTOR_ELT) &&
47668          "Unexpected vector insertion");
47669 
47670   SDValue Vec = N->getOperand(0);
47671   SDValue Scl = N->getOperand(1);
47672   SDValue Idx = N->getOperand(2);
47673 
47674   // Fold insert_vector_elt(undef, elt, 0) --> scalar_to_vector(elt).
47675   if (Opcode == ISD::INSERT_VECTOR_ELT && Vec.isUndef() && isNullConstant(Idx))
47676     return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Scl);
47677 
47678   if (Opcode == X86ISD::PINSRB || Opcode == X86ISD::PINSRW) {
47679     unsigned NumBitsPerElt = VT.getScalarSizeInBits();
47680     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47681     if (TLI.SimplifyDemandedBits(SDValue(N, 0),
47682                                  APInt::getAllOnes(NumBitsPerElt), DCI))
47683       return SDValue(N, 0);
47684   }
47685 
47686   // Attempt to combine insertion patterns to a shuffle.
47687   if (VT.isSimple() && DCI.isAfterLegalizeDAG()) {
47688     SDValue Op(N, 0);
47689     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
47690       return Res;
47691   }
47692 
47693   return SDValue();
47694 }
47695 
47696 /// Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs
47697 /// reference the same FP CMP, and rewrite for CMPEQSS and friends. Likewise for
47698 /// OR -> CMPNEQSS.
47699 static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
47700                                    TargetLowering::DAGCombinerInfo &DCI,
47701                                    const X86Subtarget &Subtarget) {
47702   unsigned opcode;
47703 
47704   // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but
47705   // we're requiring SSE2 for both.
47706   if (Subtarget.hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
47707     SDValue N0 = N->getOperand(0);
47708     SDValue N1 = N->getOperand(1);
47709     SDValue CMP0 = N0.getOperand(1);
47710     SDValue CMP1 = N1.getOperand(1);
47711     SDLoc DL(N);
47712 
47713     // The SETCCs should both refer to the same CMP.
47714     if (CMP0.getOpcode() != X86ISD::FCMP || CMP0 != CMP1)
47715       return SDValue();
47716 
47717     SDValue CMP00 = CMP0->getOperand(0);
47718     SDValue CMP01 = CMP0->getOperand(1);
47719     EVT     VT    = CMP00.getValueType();
47720 
47721     if (VT == MVT::f32 || VT == MVT::f64 ||
47722         (VT == MVT::f16 && Subtarget.hasFP16())) {
47723       bool ExpectingFlags = false;
47724       // Check for any users that want flags:
47725       for (const SDNode *U : N->uses()) {
47726         if (ExpectingFlags)
47727           break;
47728 
47729         switch (U->getOpcode()) {
47730         default:
47731         case ISD::BR_CC:
47732         case ISD::BRCOND:
47733         case ISD::SELECT:
47734           ExpectingFlags = true;
47735           break;
47736         case ISD::CopyToReg:
47737         case ISD::SIGN_EXTEND:
47738         case ISD::ZERO_EXTEND:
47739         case ISD::ANY_EXTEND:
47740           break;
47741         }
47742       }
47743 
47744       if (!ExpectingFlags) {
47745         enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
47746         enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
47747 
47748         if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
47749           X86::CondCode tmp = cc0;
47750           cc0 = cc1;
47751           cc1 = tmp;
47752         }
47753 
47754         if ((cc0 == X86::COND_E  && cc1 == X86::COND_NP) ||
47755             (cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
47756           // FIXME: need symbolic constants for these magic numbers.
47757           // See X86ATTInstPrinter.cpp:printSSECC().
47758           unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
47759           if (Subtarget.hasAVX512()) {
47760             SDValue FSetCC =
47761                 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
47762                             DAG.getTargetConstant(x86cc, DL, MVT::i8));
47763             // Need to fill with zeros to ensure the bitcast will produce zeroes
47764             // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
47765             SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
47766                                       DAG.getConstant(0, DL, MVT::v16i1),
47767                                       FSetCC, DAG.getIntPtrConstant(0, DL));
47768             return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL,
47769                                       N->getSimpleValueType(0));
47770           }
47771           SDValue OnesOrZeroesF =
47772               DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00,
47773                           CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8));
47774 
47775           bool is64BitFP = (CMP00.getValueType() == MVT::f64);
47776           MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
47777 
47778           if (is64BitFP && !Subtarget.is64Bit()) {
47779             // On a 32-bit target, we cannot bitcast the 64-bit float to a
47780             // 64-bit integer, since that's not a legal type. Since
47781             // OnesOrZeroesF is all ones or all zeroes, we don't need all the
47782             // bits, but can do this little dance to extract the lowest 32 bits
47783             // and work with those going forward.
47784             SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
47785                                            OnesOrZeroesF);
47786             SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
47787             OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
47788                                         Vector32, DAG.getIntPtrConstant(0, DL));
47789             IntVT = MVT::i32;
47790           }
47791 
47792           SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
47793           SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
47794                                       DAG.getConstant(1, DL, IntVT));
47795           SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
47796                                               ANDed);
47797           return OneBitOfTruth;
47798         }
47799       }
47800     }
47801   }
47802   return SDValue();
47803 }
47804 
47805 /// Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
47806 static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) {
47807   assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
47808 
47809   MVT VT = N->getSimpleValueType(0);
47810   if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector())
47811     return SDValue();
47812 
47813   SDValue X, Y;
47814   SDValue N0 = N->getOperand(0);
47815   SDValue N1 = N->getOperand(1);
47816 
47817   if (SDValue Not = IsNOT(N0, DAG)) {
47818     X = Not;
47819     Y = N1;
47820   } else if (SDValue Not = IsNOT(N1, DAG)) {
47821     X = Not;
47822     Y = N0;
47823   } else
47824     return SDValue();
47825 
47826   X = DAG.getBitcast(VT, X);
47827   Y = DAG.getBitcast(VT, Y);
47828   return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
47829 }
47830 
47831 /// Try to fold:
47832 ///   and (vector_shuffle<Z,...,Z>
47833 ///            (insert_vector_elt undef, (xor X, -1), Z), undef), Y
47834 ///   ->
47835 ///   andnp (vector_shuffle<Z,...,Z>
47836 ///              (insert_vector_elt undef, X, Z), undef), Y
47837 static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG,
47838                                     const X86Subtarget &Subtarget) {
47839   assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
47840 
47841   EVT VT = N->getValueType(0);
47842   // Do not split 256 and 512 bit vectors with SSE2 as they overwrite original
47843   // value and require extra moves.
47844   if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||
47845         ((VT.is256BitVector() || VT.is512BitVector()) && Subtarget.hasAVX())))
47846     return SDValue();
47847 
47848   auto GetNot = [&DAG](SDValue V) {
47849     auto *SVN = dyn_cast<ShuffleVectorSDNode>(peekThroughOneUseBitcasts(V));
47850     // TODO: SVN->hasOneUse() is a strong condition. It can be relaxed if all
47851     // end-users are ISD::AND including cases
47852     // (and(extract_vector_element(SVN), Y)).
47853     if (!SVN || !SVN->hasOneUse() || !SVN->isSplat() ||
47854         !SVN->getOperand(1).isUndef()) {
47855       return SDValue();
47856     }
47857     SDValue IVEN = SVN->getOperand(0);
47858     if (IVEN.getOpcode() != ISD::INSERT_VECTOR_ELT ||
47859         !IVEN.getOperand(0).isUndef() || !IVEN.hasOneUse())
47860       return SDValue();
47861     if (!isa<ConstantSDNode>(IVEN.getOperand(2)) ||
47862         IVEN.getConstantOperandAPInt(2) != SVN->getSplatIndex())
47863       return SDValue();
47864     SDValue Src = IVEN.getOperand(1);
47865     if (SDValue Not = IsNOT(Src, DAG)) {
47866       SDValue NotSrc = DAG.getBitcast(Src.getValueType(), Not);
47867       SDValue NotIVEN =
47868           DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(IVEN), IVEN.getValueType(),
47869                       IVEN.getOperand(0), NotSrc, IVEN.getOperand(2));
47870       return DAG.getVectorShuffle(SVN->getValueType(0), SDLoc(SVN), NotIVEN,
47871                                   SVN->getOperand(1), SVN->getMask());
47872     }
47873     return SDValue();
47874   };
47875 
47876   SDValue X, Y;
47877   SDValue N0 = N->getOperand(0);
47878   SDValue N1 = N->getOperand(1);
47879 
47880   if (SDValue Not = GetNot(N0)) {
47881     X = Not;
47882     Y = N1;
47883   } else if (SDValue Not = GetNot(N1)) {
47884     X = Not;
47885     Y = N0;
47886   } else
47887     return SDValue();
47888 
47889   X = DAG.getBitcast(VT, X);
47890   Y = DAG.getBitcast(VT, Y);
47891   SDLoc DL(N);
47892   // We do not split for SSE at all, but we need to split vectors for AVX1 and
47893   // AVX2.
47894   if (!Subtarget.useAVX512Regs() && VT.is512BitVector()) {
47895     SDValue LoX, HiX;
47896     std::tie(LoX, HiX) = splitVector(X, DAG, DL);
47897     SDValue LoY, HiY;
47898     std::tie(LoY, HiY) = splitVector(Y, DAG, DL);
47899     EVT SplitVT = LoX.getValueType();
47900     SDValue LoV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {LoX, LoY});
47901     SDValue HiV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {HiX, HiY});
47902     return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, {LoV, HiV});
47903   }
47904   return DAG.getNode(X86ISD::ANDNP, DL, VT, {X, Y});
47905 }
47906 
47907 // Try to widen AND, OR and XOR nodes to VT in order to remove casts around
47908 // logical operations, like in the example below.
47909 //   or (and (truncate x, truncate y)),
47910 //      (xor (truncate z, build_vector (constants)))
47911 // Given a target type \p VT, we generate
47912 //   or (and x, y), (xor z, zext(build_vector (constants)))
47913 // given x, y and z are of type \p VT. We can do so, if operands are either
47914 // truncates from VT types, the second operand is a vector of constants or can
47915 // be recursively promoted.
47916 static SDValue PromoteMaskArithmetic(SDNode *N, EVT VT, SelectionDAG &DAG,
47917                                      unsigned Depth) {
47918   // Limit recursion to avoid excessive compile times.
47919   if (Depth >= SelectionDAG::MaxRecursionDepth)
47920     return SDValue();
47921 
47922   if (N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND &&
47923       N->getOpcode() != ISD::OR)
47924     return SDValue();
47925 
47926   SDValue N0 = N->getOperand(0);
47927   SDValue N1 = N->getOperand(1);
47928   SDLoc DL(N);
47929 
47930   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
47931   if (!TLI.isOperationLegalOrPromote(N->getOpcode(), VT))
47932     return SDValue();
47933 
47934   if (SDValue NN0 = PromoteMaskArithmetic(N0.getNode(), VT, DAG, Depth + 1))
47935     N0 = NN0;
47936   else {
47937     // The Left side has to be a trunc.
47938     if (N0.getOpcode() != ISD::TRUNCATE)
47939       return SDValue();
47940 
47941     // The type of the truncated inputs.
47942     if (N0.getOperand(0).getValueType() != VT)
47943       return SDValue();
47944 
47945     N0 = N0.getOperand(0);
47946   }
47947 
47948   if (SDValue NN1 = PromoteMaskArithmetic(N1.getNode(), VT, DAG, Depth + 1))
47949     N1 = NN1;
47950   else {
47951     // The right side has to be a 'trunc' or a constant vector.
47952     bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
47953                     N1.getOperand(0).getValueType() == VT;
47954     if (!RHSTrunc && !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()))
47955       return SDValue();
47956 
47957     if (RHSTrunc)
47958       N1 = N1.getOperand(0);
47959     else
47960       N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N1);
47961   }
47962 
47963   return DAG.getNode(N->getOpcode(), DL, VT, N0, N1);
47964 }
47965 
47966 // On AVX/AVX2 the type v8i1 is legalized to v8i16, which is an XMM sized
47967 // register. In most cases we actually compare or select YMM-sized registers
47968 // and mixing the two types creates horrible code. This method optimizes
47969 // some of the transition sequences.
47970 // Even with AVX-512 this is still useful for removing casts around logical
47971 // operations on vXi1 mask types.
47972 static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
47973                                      const X86Subtarget &Subtarget) {
47974   EVT VT = N->getValueType(0);
47975   assert(VT.isVector() && "Expected vector type");
47976 
47977   SDLoc DL(N);
47978   assert((N->getOpcode() == ISD::ANY_EXTEND ||
47979           N->getOpcode() == ISD::ZERO_EXTEND ||
47980           N->getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node");
47981 
47982   SDValue Narrow = N->getOperand(0);
47983   EVT NarrowVT = Narrow.getValueType();
47984 
47985   // Generate the wide operation.
47986   SDValue Op = PromoteMaskArithmetic(Narrow.getNode(), VT, DAG, 0);
47987   if (!Op)
47988     return SDValue();
47989   switch (N->getOpcode()) {
47990   default: llvm_unreachable("Unexpected opcode");
47991   case ISD::ANY_EXTEND:
47992     return Op;
47993   case ISD::ZERO_EXTEND:
47994     return DAG.getZeroExtendInReg(Op, DL, NarrowVT);
47995   case ISD::SIGN_EXTEND:
47996     return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
47997                        Op, DAG.getValueType(NarrowVT));
47998   }
47999 }
48000 
48001 static unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode) {
48002   unsigned FPOpcode;
48003   switch (Opcode) {
48004   default: llvm_unreachable("Unexpected input node for FP logic conversion");
48005   case ISD::AND: FPOpcode = X86ISD::FAND; break;
48006   case ISD::OR:  FPOpcode = X86ISD::FOR;  break;
48007   case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
48008   }
48009   return FPOpcode;
48010 }
48011 
48012 /// If both input operands of a logic op are being cast from floating-point
48013 /// types or FP compares, try to convert this into a floating-point logic node
48014 /// to avoid unnecessary moves from SSE to integer registers.
48015 static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG,
48016                                         TargetLowering::DAGCombinerInfo &DCI,
48017                                         const X86Subtarget &Subtarget) {
48018   EVT VT = N->getValueType(0);
48019   SDValue N0 = N->getOperand(0);
48020   SDValue N1 = N->getOperand(1);
48021   SDLoc DL(N);
48022 
48023   if (!((N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) ||
48024         (N0.getOpcode() == ISD::SETCC && N1.getOpcode() == ISD::SETCC)))
48025     return SDValue();
48026 
48027   SDValue N00 = N0.getOperand(0);
48028   SDValue N10 = N1.getOperand(0);
48029   EVT N00Type = N00.getValueType();
48030   EVT N10Type = N10.getValueType();
48031 
48032   // Ensure that both types are the same and are legal scalar fp types.
48033   if (N00Type != N10Type || !((Subtarget.hasSSE1() && N00Type == MVT::f32) ||
48034                               (Subtarget.hasSSE2() && N00Type == MVT::f64) ||
48035                               (Subtarget.hasFP16() && N00Type == MVT::f16)))
48036     return SDValue();
48037 
48038   if (N0.getOpcode() == ISD::BITCAST && !DCI.isBeforeLegalizeOps()) {
48039     unsigned FPOpcode = convertIntLogicToFPLogicOpcode(N->getOpcode());
48040     SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
48041     return DAG.getBitcast(VT, FPLogic);
48042   }
48043 
48044   if (VT != MVT::i1 || N0.getOpcode() != ISD::SETCC || !N0.hasOneUse() ||
48045       !N1.hasOneUse())
48046     return SDValue();
48047 
48048   ISD::CondCode CC0 = cast<CondCodeSDNode>(N0.getOperand(2))->get();
48049   ISD::CondCode CC1 = cast<CondCodeSDNode>(N1.getOperand(2))->get();
48050 
48051   // The vector ISA for FP predicates is incomplete before AVX, so converting
48052   // COMIS* to CMPS* may not be a win before AVX.
48053   if (!Subtarget.hasAVX() &&
48054       !(cheapX86FSETCC_SSE(CC0) && cheapX86FSETCC_SSE(CC1)))
48055     return SDValue();
48056 
48057   // Convert scalar FP compares and logic to vector compares (COMIS* to CMPS*)
48058   // and vector logic:
48059   // logic (setcc N00, N01), (setcc N10, N11) -->
48060   // extelt (logic (setcc (s2v N00), (s2v N01)), setcc (s2v N10), (s2v N11))), 0
48061   unsigned NumElts = 128 / N00Type.getSizeInBits();
48062   EVT VecVT = EVT::getVectorVT(*DAG.getContext(), N00Type, NumElts);
48063   EVT BoolVecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
48064   SDValue ZeroIndex = DAG.getVectorIdxConstant(0, DL);
48065   SDValue N01 = N0.getOperand(1);
48066   SDValue N11 = N1.getOperand(1);
48067   SDValue Vec00 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N00);
48068   SDValue Vec01 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N01);
48069   SDValue Vec10 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N10);
48070   SDValue Vec11 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N11);
48071   SDValue Setcc0 = DAG.getSetCC(DL, BoolVecVT, Vec00, Vec01, CC0);
48072   SDValue Setcc1 = DAG.getSetCC(DL, BoolVecVT, Vec10, Vec11, CC1);
48073   SDValue Logic = DAG.getNode(N->getOpcode(), DL, BoolVecVT, Setcc0, Setcc1);
48074   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Logic, ZeroIndex);
48075 }
48076 
48077 // Attempt to fold BITOP(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(BITOP(X,Y))
48078 // to reduce XMM->GPR traffic.
48079 static SDValue combineBitOpWithMOVMSK(SDNode *N, SelectionDAG &DAG) {
48080   unsigned Opc = N->getOpcode();
48081   assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
48082          "Unexpected bit opcode");
48083 
48084   SDValue N0 = N->getOperand(0);
48085   SDValue N1 = N->getOperand(1);
48086 
48087   // Both operands must be single use MOVMSK.
48088   if (N0.getOpcode() != X86ISD::MOVMSK || !N0.hasOneUse() ||
48089       N1.getOpcode() != X86ISD::MOVMSK || !N1.hasOneUse())
48090     return SDValue();
48091 
48092   SDValue Vec0 = N0.getOperand(0);
48093   SDValue Vec1 = N1.getOperand(0);
48094   EVT VecVT0 = Vec0.getValueType();
48095   EVT VecVT1 = Vec1.getValueType();
48096 
48097   // Both MOVMSK operands must be from vectors of the same size and same element
48098   // size, but its OK for a fp/int diff.
48099   if (VecVT0.getSizeInBits() != VecVT1.getSizeInBits() ||
48100       VecVT0.getScalarSizeInBits() != VecVT1.getScalarSizeInBits())
48101     return SDValue();
48102 
48103   SDLoc DL(N);
48104   unsigned VecOpc =
48105       VecVT0.isFloatingPoint() ? convertIntLogicToFPLogicOpcode(Opc) : Opc;
48106   SDValue Result =
48107       DAG.getNode(VecOpc, DL, VecVT0, Vec0, DAG.getBitcast(VecVT0, Vec1));
48108   return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
48109 }
48110 
48111 // Attempt to fold BITOP(SHIFT(X,Z),SHIFT(Y,Z)) -> SHIFT(BITOP(X,Y),Z).
48112 // NOTE: This is a very limited case of what SimplifyUsingDistributiveLaws
48113 // handles in InstCombine.
48114 static SDValue combineBitOpWithShift(SDNode *N, SelectionDAG &DAG) {
48115   unsigned Opc = N->getOpcode();
48116   assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
48117          "Unexpected bit opcode");
48118 
48119   SDValue N0 = N->getOperand(0);
48120   SDValue N1 = N->getOperand(1);
48121   EVT VT = N->getValueType(0);
48122 
48123   // Both operands must be single use.
48124   if (!N0.hasOneUse() || !N1.hasOneUse())
48125     return SDValue();
48126 
48127   // Search for matching shifts.
48128   SDValue BC0 = peekThroughOneUseBitcasts(N0);
48129   SDValue BC1 = peekThroughOneUseBitcasts(N1);
48130 
48131   unsigned BCOpc = BC0.getOpcode();
48132   EVT BCVT = BC0.getValueType();
48133   if (BCOpc != BC1->getOpcode() || BCVT != BC1.getValueType())
48134     return SDValue();
48135 
48136   switch (BCOpc) {
48137   case X86ISD::VSHLI:
48138   case X86ISD::VSRLI:
48139   case X86ISD::VSRAI: {
48140     if (BC0.getOperand(1) != BC1.getOperand(1))
48141       return SDValue();
48142 
48143     SDLoc DL(N);
48144     SDValue BitOp =
48145         DAG.getNode(Opc, DL, BCVT, BC0.getOperand(0), BC1.getOperand(0));
48146     SDValue Shift = DAG.getNode(BCOpc, DL, BCVT, BitOp, BC0.getOperand(1));
48147     return DAG.getBitcast(VT, Shift);
48148   }
48149   }
48150 
48151   return SDValue();
48152 }
48153 
48154 // Attempt to fold:
48155 // BITOP(PACKSS(X,Z),PACKSS(Y,W)) --> PACKSS(BITOP(X,Y),BITOP(Z,W)).
48156 // TODO: Handle PACKUS handling.
48157 static SDValue combineBitOpWithPACK(SDNode *N, SelectionDAG &DAG) {
48158   unsigned Opc = N->getOpcode();
48159   assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
48160          "Unexpected bit opcode");
48161 
48162   SDValue N0 = N->getOperand(0);
48163   SDValue N1 = N->getOperand(1);
48164   EVT VT = N->getValueType(0);
48165 
48166   // Both operands must be single use.
48167   if (!N0.hasOneUse() || !N1.hasOneUse())
48168     return SDValue();
48169 
48170   // Search for matching packs.
48171   N0 = peekThroughOneUseBitcasts(N0);
48172   N1 = peekThroughOneUseBitcasts(N1);
48173 
48174   if (N0.getOpcode() != X86ISD::PACKSS || N1.getOpcode() != X86ISD::PACKSS)
48175     return SDValue();
48176 
48177   MVT DstVT = N0.getSimpleValueType();
48178   if (DstVT != N1.getSimpleValueType())
48179     return SDValue();
48180 
48181   MVT SrcVT = N0.getOperand(0).getSimpleValueType();
48182   unsigned NumSrcBits = SrcVT.getScalarSizeInBits();
48183 
48184   // Limit to allsignbits packing.
48185   if (DAG.ComputeNumSignBits(N0.getOperand(0)) != NumSrcBits ||
48186       DAG.ComputeNumSignBits(N0.getOperand(1)) != NumSrcBits ||
48187       DAG.ComputeNumSignBits(N1.getOperand(0)) != NumSrcBits ||
48188       DAG.ComputeNumSignBits(N1.getOperand(1)) != NumSrcBits)
48189     return SDValue();
48190 
48191   SDLoc DL(N);
48192   SDValue LHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(0), N1.getOperand(0));
48193   SDValue RHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(1), N1.getOperand(1));
48194   return DAG.getBitcast(VT, DAG.getNode(X86ISD::PACKSS, DL, DstVT, LHS, RHS));
48195 }
48196 
48197 /// If this is a zero/all-bits result that is bitwise-anded with a low bits
48198 /// mask. (Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and'
48199 /// with a shift-right to eliminate loading the vector constant mask value.
48200 static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
48201                                      const X86Subtarget &Subtarget) {
48202   SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
48203   SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
48204   EVT VT = Op0.getValueType();
48205   if (VT != Op1.getValueType() || !VT.isSimple() || !VT.isInteger())
48206     return SDValue();
48207 
48208   // Try to convert an "is positive" signbit masking operation into arithmetic
48209   // shift and "andn". This saves a materialization of a -1 vector constant.
48210   // The "is negative" variant should be handled more generally because it only
48211   // requires "and" rather than "andn":
48212   // and (pcmpgt X, -1), Y --> pandn (vsrai X, BitWidth - 1), Y
48213   //
48214   // This is limited to the original type to avoid producing even more bitcasts.
48215   // If the bitcasts can't be eliminated, then it is unlikely that this fold
48216   // will be profitable.
48217   if (N->getValueType(0) == VT &&
48218       supportedVectorShiftWithImm(VT, Subtarget, ISD::SRA)) {
48219     SDValue X, Y;
48220     if (Op1.getOpcode() == X86ISD::PCMPGT &&
48221         isAllOnesOrAllOnesSplat(Op1.getOperand(1)) && Op1.hasOneUse()) {
48222       X = Op1.getOperand(0);
48223       Y = Op0;
48224     } else if (Op0.getOpcode() == X86ISD::PCMPGT &&
48225                isAllOnesOrAllOnesSplat(Op0.getOperand(1)) && Op0.hasOneUse()) {
48226       X = Op0.getOperand(0);
48227       Y = Op1;
48228     }
48229     if (X && Y) {
48230       SDLoc DL(N);
48231       SDValue Sra =
48232           getTargetVShiftByConstNode(X86ISD::VSRAI, DL, VT.getSimpleVT(), X,
48233                                      VT.getScalarSizeInBits() - 1, DAG);
48234       return DAG.getNode(X86ISD::ANDNP, DL, VT, Sra, Y);
48235     }
48236   }
48237 
48238   APInt SplatVal;
48239   if (!X86::isConstantSplat(Op1, SplatVal, false) || !SplatVal.isMask())
48240     return SDValue();
48241 
48242   // Don't prevent creation of ANDN.
48243   if (isBitwiseNot(Op0))
48244     return SDValue();
48245 
48246   if (!supportedVectorShiftWithImm(VT, Subtarget, ISD::SRL))
48247     return SDValue();
48248 
48249   unsigned EltBitWidth = VT.getScalarSizeInBits();
48250   if (EltBitWidth != DAG.ComputeNumSignBits(Op0))
48251     return SDValue();
48252 
48253   SDLoc DL(N);
48254   unsigned ShiftVal = SplatVal.countr_one();
48255   SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
48256   SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt);
48257   return DAG.getBitcast(N->getValueType(0), Shift);
48258 }
48259 
48260 // Get the index node from the lowered DAG of a GEP IR instruction with one
48261 // indexing dimension.
48262 static SDValue getIndexFromUnindexedLoad(LoadSDNode *Ld) {
48263   if (Ld->isIndexed())
48264     return SDValue();
48265 
48266   SDValue Base = Ld->getBasePtr();
48267 
48268   if (Base.getOpcode() != ISD::ADD)
48269     return SDValue();
48270 
48271   SDValue ShiftedIndex = Base.getOperand(0);
48272 
48273   if (ShiftedIndex.getOpcode() != ISD::SHL)
48274     return SDValue();
48275 
48276   return ShiftedIndex.getOperand(0);
48277 
48278 }
48279 
48280 static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) {
48281   if (Subtarget.hasBMI2() && VT.isScalarInteger()) {
48282     switch (VT.getSizeInBits()) {
48283     default: return false;
48284     case 64: return Subtarget.is64Bit() ? true : false;
48285     case 32: return true;
48286     }
48287   }
48288   return false;
48289 }
48290 
48291 // This function recognizes cases where X86 bzhi instruction can replace and
48292 // 'and-load' sequence.
48293 // In case of loading integer value from an array of constants which is defined
48294 // as follows:
48295 //
48296 //   int array[SIZE] = {0x0, 0x1, 0x3, 0x7, 0xF ..., 2^(SIZE-1) - 1}
48297 //
48298 // then applying a bitwise and on the result with another input.
48299 // It's equivalent to performing bzhi (zero high bits) on the input, with the
48300 // same index of the load.
48301 static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
48302                                     const X86Subtarget &Subtarget) {
48303   MVT VT = Node->getSimpleValueType(0);
48304   SDLoc dl(Node);
48305 
48306   // Check if subtarget has BZHI instruction for the node's type
48307   if (!hasBZHI(Subtarget, VT))
48308     return SDValue();
48309 
48310   // Try matching the pattern for both operands.
48311   for (unsigned i = 0; i < 2; i++) {
48312     SDValue N = Node->getOperand(i);
48313     LoadSDNode *Ld = dyn_cast<LoadSDNode>(N.getNode());
48314 
48315      // continue if the operand is not a load instruction
48316     if (!Ld)
48317       return SDValue();
48318 
48319     const Value *MemOp = Ld->getMemOperand()->getValue();
48320 
48321     if (!MemOp)
48322       return SDValue();
48323 
48324     if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(MemOp)) {
48325       if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) {
48326         if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
48327 
48328           Constant *Init = GV->getInitializer();
48329           Type *Ty = Init->getType();
48330           if (!isa<ConstantDataArray>(Init) ||
48331               !Ty->getArrayElementType()->isIntegerTy() ||
48332               Ty->getArrayElementType()->getScalarSizeInBits() !=
48333                   VT.getSizeInBits() ||
48334               Ty->getArrayNumElements() >
48335                   Ty->getArrayElementType()->getScalarSizeInBits())
48336             continue;
48337 
48338           // Check if the array's constant elements are suitable to our case.
48339           uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
48340           bool ConstantsMatch = true;
48341           for (uint64_t j = 0; j < ArrayElementCount; j++) {
48342             auto *Elem = cast<ConstantInt>(Init->getAggregateElement(j));
48343             if (Elem->getZExtValue() != (((uint64_t)1 << j) - 1)) {
48344               ConstantsMatch = false;
48345               break;
48346             }
48347           }
48348           if (!ConstantsMatch)
48349             continue;
48350 
48351           // Do the transformation (For 32-bit type):
48352           // -> (and (load arr[idx]), inp)
48353           // <- (and (srl 0xFFFFFFFF, (sub 32, idx)))
48354           //    that will be replaced with one bzhi instruction.
48355           SDValue Inp = (i == 0) ? Node->getOperand(1) : Node->getOperand(0);
48356           SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);
48357 
48358           // Get the Node which indexes into the array.
48359           SDValue Index = getIndexFromUnindexedLoad(Ld);
48360           if (!Index)
48361             return SDValue();
48362           Index = DAG.getZExtOrTrunc(Index, dl, MVT::i32);
48363 
48364           SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);
48365           Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub);
48366 
48367           SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
48368           SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);
48369 
48370           return DAG.getNode(ISD::AND, dl, VT, Inp, LShr);
48371         }
48372       }
48373     }
48374   }
48375   return SDValue();
48376 }
48377 
48378 // Look for (and (bitcast (vXi1 (concat_vectors (vYi1 setcc), undef,))), C)
48379 // Where C is a mask containing the same number of bits as the setcc and
48380 // where the setcc will freely 0 upper bits of k-register. We can replace the
48381 // undef in the concat with 0s and remove the AND. This mainly helps with
48382 // v2i1/v4i1 setcc being casted to scalar.
48383 static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
48384                                              const X86Subtarget &Subtarget) {
48385   assert(N->getOpcode() == ISD::AND && "Unexpected opcode!");
48386 
48387   EVT VT = N->getValueType(0);
48388 
48389   // Make sure this is an AND with constant. We will check the value of the
48390   // constant later.
48391   auto *C1 = dyn_cast<ConstantSDNode>(N->getOperand(1));
48392   if (!C1)
48393     return SDValue();
48394 
48395   // This is implied by the ConstantSDNode.
48396   assert(!VT.isVector() && "Expected scalar VT!");
48397 
48398   SDValue Src = N->getOperand(0);
48399   if (!Src.hasOneUse())
48400     return SDValue();
48401 
48402   // (Optionally) peek through any_extend().
48403   if (Src.getOpcode() == ISD::ANY_EXTEND) {
48404     if (!Src.getOperand(0).hasOneUse())
48405       return SDValue();
48406     Src = Src.getOperand(0);
48407   }
48408 
48409   if (Src.getOpcode() != ISD::BITCAST || !Src.getOperand(0).hasOneUse())
48410     return SDValue();
48411 
48412   Src = Src.getOperand(0);
48413   EVT SrcVT = Src.getValueType();
48414 
48415   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
48416   if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::i1 ||
48417       !TLI.isTypeLegal(SrcVT))
48418     return SDValue();
48419 
48420   if (Src.getOpcode() != ISD::CONCAT_VECTORS)
48421     return SDValue();
48422 
48423   // We only care about the first subvector of the concat, we expect the
48424   // other subvectors to be ignored due to the AND if we make the change.
48425   SDValue SubVec = Src.getOperand(0);
48426   EVT SubVecVT = SubVec.getValueType();
48427 
48428   // The RHS of the AND should be a mask with as many bits as SubVec.
48429   if (!TLI.isTypeLegal(SubVecVT) ||
48430       !C1->getAPIntValue().isMask(SubVecVT.getVectorNumElements()))
48431     return SDValue();
48432 
48433   // First subvector should be a setcc with a legal result type or a
48434   // AND containing at least one setcc with a legal result type.
48435   auto IsLegalSetCC = [&](SDValue V) {
48436     if (V.getOpcode() != ISD::SETCC)
48437       return false;
48438     EVT SetccVT = V.getOperand(0).getValueType();
48439     if (!TLI.isTypeLegal(SetccVT) ||
48440         !(Subtarget.hasVLX() || SetccVT.is512BitVector()))
48441       return false;
48442     if (!(Subtarget.hasBWI() || SetccVT.getScalarSizeInBits() >= 32))
48443       return false;
48444     return true;
48445   };
48446   if (!(IsLegalSetCC(SubVec) || (SubVec.getOpcode() == ISD::AND &&
48447                                  (IsLegalSetCC(SubVec.getOperand(0)) ||
48448                                   IsLegalSetCC(SubVec.getOperand(1))))))
48449     return SDValue();
48450 
48451   // We passed all the checks. Rebuild the concat_vectors with zeroes
48452   // and cast it back to VT.
48453   SDLoc dl(N);
48454   SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
48455                               DAG.getConstant(0, dl, SubVecVT));
48456   Ops[0] = SubVec;
48457   SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
48458                                Ops);
48459   EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits());
48460   return DAG.getZExtOrTrunc(DAG.getBitcast(IntVT, Concat), dl, VT);
48461 }
48462 
48463 static SDValue getBMIMatchingOp(unsigned Opc, SelectionDAG &DAG,
48464                                 SDValue OpMustEq, SDValue Op, unsigned Depth) {
48465   // We don't want to go crazy with the recursion here. This isn't a super
48466   // important optimization.
48467   static constexpr unsigned kMaxDepth = 2;
48468 
48469   // Only do this re-ordering if op has one use.
48470   if (!Op.hasOneUse())
48471     return SDValue();
48472 
48473   SDLoc DL(Op);
48474   // If we hit another assosiative op, recurse further.
48475   if (Op.getOpcode() == Opc) {
48476     // Done recursing.
48477     if (Depth++ >= kMaxDepth)
48478       return SDValue();
48479 
48480     for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx)
48481       if (SDValue R =
48482               getBMIMatchingOp(Opc, DAG, OpMustEq, Op.getOperand(OpIdx), Depth))
48483         return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), R,
48484                            Op.getOperand(1 - OpIdx));
48485 
48486   } else if (Op.getOpcode() == ISD::SUB) {
48487     if (Opc == ISD::AND) {
48488       // BLSI: (and x, (sub 0, x))
48489       if (isNullConstant(Op.getOperand(0)) && Op.getOperand(1) == OpMustEq)
48490         return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
48491     }
48492     // Opc must be ISD::AND or ISD::XOR
48493     // BLSR: (and x, (sub x, 1))
48494     // BLSMSK: (xor x, (sub x, 1))
48495     if (isOneConstant(Op.getOperand(1)) && Op.getOperand(0) == OpMustEq)
48496       return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
48497 
48498   } else if (Op.getOpcode() == ISD::ADD) {
48499     // Opc must be ISD::AND or ISD::XOR
48500     // BLSR: (and x, (add x, -1))
48501     // BLSMSK: (xor x, (add x, -1))
48502     if (isAllOnesConstant(Op.getOperand(1)) && Op.getOperand(0) == OpMustEq)
48503       return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
48504   }
48505   return SDValue();
48506 }
48507 
48508 static SDValue combineBMILogicOp(SDNode *N, SelectionDAG &DAG,
48509                                  const X86Subtarget &Subtarget) {
48510   EVT VT = N->getValueType(0);
48511   // Make sure this node is a candidate for BMI instructions.
48512   if (!Subtarget.hasBMI() || !VT.isScalarInteger() ||
48513       (VT != MVT::i32 && VT != MVT::i64))
48514     return SDValue();
48515 
48516   assert(N->getOpcode() == ISD::AND || N->getOpcode() == ISD::XOR);
48517 
48518   // Try and match LHS and RHS.
48519   for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx)
48520     if (SDValue OpMatch =
48521             getBMIMatchingOp(N->getOpcode(), DAG, N->getOperand(OpIdx),
48522                              N->getOperand(1 - OpIdx), 0))
48523       return OpMatch;
48524   return SDValue();
48525 }
48526 
48527 static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
48528                           TargetLowering::DAGCombinerInfo &DCI,
48529                           const X86Subtarget &Subtarget) {
48530   SDValue N0 = N->getOperand(0);
48531   SDValue N1 = N->getOperand(1);
48532   EVT VT = N->getValueType(0);
48533   SDLoc dl(N);
48534   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
48535 
48536   // If this is SSE1 only convert to FAND to avoid scalarization.
48537   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
48538     return DAG.getBitcast(MVT::v4i32,
48539                           DAG.getNode(X86ISD::FAND, dl, MVT::v4f32,
48540                                       DAG.getBitcast(MVT::v4f32, N0),
48541                                       DAG.getBitcast(MVT::v4f32, N1)));
48542   }
48543 
48544   // Use a 32-bit and+zext if upper bits known zero.
48545   if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) {
48546     APInt HiMask = APInt::getHighBitsSet(64, 32);
48547     if (DAG.MaskedValueIsZero(N1, HiMask) ||
48548         DAG.MaskedValueIsZero(N0, HiMask)) {
48549       SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0);
48550       SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1);
48551       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64,
48552                          DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS));
48553     }
48554   }
48555 
48556   // Match all-of bool scalar reductions into a bitcast/movmsk + cmp.
48557   // TODO: Support multiple SrcOps.
48558   if (VT == MVT::i1) {
48559     SmallVector<SDValue, 2> SrcOps;
48560     SmallVector<APInt, 2> SrcPartials;
48561     if (matchScalarReduction(SDValue(N, 0), ISD::AND, SrcOps, &SrcPartials) &&
48562         SrcOps.size() == 1) {
48563       unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
48564       EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
48565       SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
48566       if (!Mask && TLI.isTypeLegal(SrcOps[0].getValueType()))
48567         Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
48568       if (Mask) {
48569         assert(SrcPartials[0].getBitWidth() == NumElts &&
48570                "Unexpected partial reduction mask");
48571         SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
48572         Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits);
48573         return DAG.getSetCC(dl, MVT::i1, Mask, PartialBits, ISD::SETEQ);
48574       }
48575     }
48576   }
48577 
48578   // InstCombine converts:
48579   //    `(-x << C0) & C1`
48580   // to
48581   //    `(x * (Pow2_Ceil(C1) - (1 << C0))) & C1`
48582   // This saves an IR instruction but on x86 the neg/shift version is preferable
48583   // so undo the transform.
48584 
48585   if (N0.getOpcode() == ISD::MUL && N0.hasOneUse()) {
48586     // TODO: We don't actually need a splat for this, we just need the checks to
48587     // hold for each element.
48588     ConstantSDNode *N1C = isConstOrConstSplat(N1, /*AllowUndefs*/ true,
48589                                               /*AllowTruncation*/ false);
48590     ConstantSDNode *N01C =
48591         isConstOrConstSplat(N0.getOperand(1), /*AllowUndefs*/ true,
48592                             /*AllowTruncation*/ false);
48593     if (N1C && N01C) {
48594       const APInt &MulC = N01C->getAPIntValue();
48595       const APInt &AndC = N1C->getAPIntValue();
48596       APInt MulCLowBit = MulC & (-MulC);
48597       if (MulC.uge(AndC) && !MulC.isPowerOf2() &&
48598           (MulCLowBit + MulC).isPowerOf2()) {
48599         SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
48600                                   N0.getOperand(0));
48601         int32_t MulCLowBitLog = MulCLowBit.exactLogBase2();
48602         assert(MulCLowBitLog != -1 &&
48603                "Isolated lowbit is somehow not a power of 2!");
48604         SDValue Shift = DAG.getNode(ISD::SHL, dl, VT, Neg,
48605                                     DAG.getConstant(MulCLowBitLog, dl, VT));
48606         return DAG.getNode(ISD::AND, dl, VT, Shift, N1);
48607       }
48608     }
48609   }
48610 
48611   if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
48612     return V;
48613 
48614   if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
48615     return R;
48616 
48617   if (SDValue R = combineBitOpWithShift(N, DAG))
48618     return R;
48619 
48620   if (SDValue R = combineBitOpWithPACK(N, DAG))
48621     return R;
48622 
48623   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
48624     return FPLogic;
48625 
48626   if (SDValue R = combineAndShuffleNot(N, DAG, Subtarget))
48627     return R;
48628 
48629   if (DCI.isBeforeLegalizeOps())
48630     return SDValue();
48631 
48632   if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
48633     return R;
48634 
48635   if (SDValue R = combineAndNotIntoANDNP(N, DAG))
48636     return R;
48637 
48638   if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
48639     return ShiftRight;
48640 
48641   if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
48642     return R;
48643 
48644   // fold (and (mul x, c1), c2) -> (mul x, (and c1, c2))
48645   // iff c2 is all/no bits mask - i.e. a select-with-zero mask.
48646   // TODO: Handle PMULDQ/PMULUDQ/VPMADDWD/VPMADDUBSW?
48647   if (VT.isVector() && getTargetConstantFromNode(N1)) {
48648     unsigned Opc0 = N0.getOpcode();
48649     if ((Opc0 == ISD::MUL || Opc0 == ISD::MULHU || Opc0 == ISD::MULHS) &&
48650         getTargetConstantFromNode(N0.getOperand(1)) &&
48651         DAG.ComputeNumSignBits(N1) == VT.getScalarSizeInBits() &&
48652         N0->hasOneUse() && N0.getOperand(1)->hasOneUse()) {
48653       SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1);
48654       return DAG.getNode(Opc0, dl, VT, N0.getOperand(0), MaskMul);
48655     }
48656   }
48657 
48658   // Fold AND(SRL(X,Y),1) -> SETCC(BT(X,Y), COND_B) iff Y is not a constant
48659   // avoids slow variable shift (moving shift amount to ECX etc.)
48660   if (isOneConstant(N1) && N0->hasOneUse()) {
48661     SDValue Src = N0;
48662     while ((Src.getOpcode() == ISD::ZERO_EXTEND ||
48663             Src.getOpcode() == ISD::TRUNCATE) &&
48664            Src.getOperand(0)->hasOneUse())
48665       Src = Src.getOperand(0);
48666     bool ContainsNOT = false;
48667     X86::CondCode X86CC = X86::COND_B;
48668     // Peek through AND(NOT(SRL(X,Y)),1).
48669     if (isBitwiseNot(Src)) {
48670       Src = Src.getOperand(0);
48671       X86CC = X86::COND_AE;
48672       ContainsNOT = true;
48673     }
48674     if (Src.getOpcode() == ISD::SRL &&
48675         !isa<ConstantSDNode>(Src.getOperand(1))) {
48676       SDValue BitNo = Src.getOperand(1);
48677       Src = Src.getOperand(0);
48678       // Peek through AND(SRL(NOT(X),Y),1).
48679       if (isBitwiseNot(Src)) {
48680         Src = Src.getOperand(0);
48681         X86CC = X86CC == X86::COND_AE ? X86::COND_B : X86::COND_AE;
48682         ContainsNOT = true;
48683       }
48684       // If we have BMI2 then SHRX should be faster for i32/i64 cases.
48685       if (!(Subtarget.hasBMI2() && !ContainsNOT && VT.getSizeInBits() >= 32))
48686         if (SDValue BT = getBT(Src, BitNo, dl, DAG))
48687           return DAG.getZExtOrTrunc(getSETCC(X86CC, BT, dl, DAG), dl, VT);
48688     }
48689   }
48690 
48691   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
48692     // Attempt to recursively combine a bitmask AND with shuffles.
48693     SDValue Op(N, 0);
48694     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
48695       return Res;
48696 
48697     // If either operand is a constant mask, then only the elements that aren't
48698     // zero are actually demanded by the other operand.
48699     auto GetDemandedMasks = [&](SDValue Op) {
48700       APInt UndefElts;
48701       SmallVector<APInt> EltBits;
48702       int NumElts = VT.getVectorNumElements();
48703       int EltSizeInBits = VT.getScalarSizeInBits();
48704       APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
48705       APInt DemandedElts = APInt::getAllOnes(NumElts);
48706       if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
48707                                         EltBits)) {
48708         DemandedBits.clearAllBits();
48709         DemandedElts.clearAllBits();
48710         for (int I = 0; I != NumElts; ++I) {
48711           if (UndefElts[I]) {
48712             // We can't assume an undef src element gives an undef dst - the
48713             // other src might be zero.
48714             DemandedBits.setAllBits();
48715             DemandedElts.setBit(I);
48716           } else if (!EltBits[I].isZero()) {
48717             DemandedBits |= EltBits[I];
48718             DemandedElts.setBit(I);
48719           }
48720         }
48721       }
48722       return std::make_pair(DemandedBits, DemandedElts);
48723     };
48724     APInt Bits0, Elts0;
48725     APInt Bits1, Elts1;
48726     std::tie(Bits0, Elts0) = GetDemandedMasks(N1);
48727     std::tie(Bits1, Elts1) = GetDemandedMasks(N0);
48728 
48729     if (TLI.SimplifyDemandedVectorElts(N0, Elts0, DCI) ||
48730         TLI.SimplifyDemandedVectorElts(N1, Elts1, DCI) ||
48731         TLI.SimplifyDemandedBits(N0, Bits0, Elts0, DCI) ||
48732         TLI.SimplifyDemandedBits(N1, Bits1, Elts1, DCI)) {
48733       if (N->getOpcode() != ISD::DELETED_NODE)
48734         DCI.AddToWorklist(N);
48735       return SDValue(N, 0);
48736     }
48737 
48738     SDValue NewN0 = TLI.SimplifyMultipleUseDemandedBits(N0, Bits0, Elts0, DAG);
48739     SDValue NewN1 = TLI.SimplifyMultipleUseDemandedBits(N1, Bits1, Elts1, DAG);
48740     if (NewN0 || NewN1)
48741       return DAG.getNode(ISD::AND, dl, VT, NewN0 ? NewN0 : N0,
48742                          NewN1 ? NewN1 : N1);
48743   }
48744 
48745   // Attempt to combine a scalar bitmask AND with an extracted shuffle.
48746   if ((VT.getScalarSizeInBits() % 8) == 0 &&
48747       N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
48748       isa<ConstantSDNode>(N0.getOperand(1)) && N0->hasOneUse()) {
48749     SDValue BitMask = N1;
48750     SDValue SrcVec = N0.getOperand(0);
48751     EVT SrcVecVT = SrcVec.getValueType();
48752 
48753     // Check that the constant bitmask masks whole bytes.
48754     APInt UndefElts;
48755     SmallVector<APInt, 64> EltBits;
48756     if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) &&
48757         getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) &&
48758         llvm::all_of(EltBits, [](const APInt &M) {
48759           return M.isZero() || M.isAllOnes();
48760         })) {
48761       unsigned NumElts = SrcVecVT.getVectorNumElements();
48762       unsigned Scale = SrcVecVT.getScalarSizeInBits() / 8;
48763       unsigned Idx = N0.getConstantOperandVal(1);
48764 
48765       // Create a root shuffle mask from the byte mask and the extracted index.
48766       SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef);
48767       for (unsigned i = 0; i != Scale; ++i) {
48768         if (UndefElts[i])
48769           continue;
48770         int VecIdx = Scale * Idx + i;
48771         ShuffleMask[VecIdx] = EltBits[i].isZero() ? SM_SentinelZero : VecIdx;
48772       }
48773 
48774       if (SDValue Shuffle = combineX86ShufflesRecursively(
48775               {SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1,
48776               X86::MaxShuffleCombineDepth,
48777               /*HasVarMask*/ false, /*AllowVarCrossLaneMask*/ true,
48778               /*AllowVarPerLaneMask*/ true, DAG, Subtarget))
48779         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
48780                            N0.getOperand(1));
48781     }
48782   }
48783 
48784   if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
48785     return R;
48786 
48787   return SDValue();
48788 }
48789 
48790 // Canonicalize OR(AND(X,C),AND(Y,~C)) -> OR(AND(X,C),ANDNP(C,Y))
48791 static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
48792                                      const X86Subtarget &Subtarget) {
48793   assert(N->getOpcode() == ISD::OR && "Unexpected Opcode");
48794 
48795   MVT VT = N->getSimpleValueType(0);
48796   unsigned EltSizeInBits = VT.getScalarSizeInBits();
48797   if (!VT.isVector() || (EltSizeInBits % 8) != 0)
48798     return SDValue();
48799 
48800   SDValue N0 = peekThroughBitcasts(N->getOperand(0));
48801   SDValue N1 = peekThroughBitcasts(N->getOperand(1));
48802   if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
48803     return SDValue();
48804 
48805   // On XOP we'll lower to PCMOV so accept one use. With AVX512, we can use
48806   // VPTERNLOG. Otherwise only do this if either mask has multiple uses already.
48807   if (!(Subtarget.hasXOP() || useVPTERNLOG(Subtarget, VT) ||
48808         !N0.getOperand(1).hasOneUse() || !N1.getOperand(1).hasOneUse()))
48809     return SDValue();
48810 
48811   // Attempt to extract constant byte masks.
48812   APInt UndefElts0, UndefElts1;
48813   SmallVector<APInt, 32> EltBits0, EltBits1;
48814   if (!getTargetConstantBitsFromNode(N0.getOperand(1), 8, UndefElts0, EltBits0,
48815                                      false, false))
48816     return SDValue();
48817   if (!getTargetConstantBitsFromNode(N1.getOperand(1), 8, UndefElts1, EltBits1,
48818                                      false, false))
48819     return SDValue();
48820 
48821   for (unsigned i = 0, e = EltBits0.size(); i != e; ++i) {
48822     // TODO - add UNDEF elts support.
48823     if (UndefElts0[i] || UndefElts1[i])
48824       return SDValue();
48825     if (EltBits0[i] != ~EltBits1[i])
48826       return SDValue();
48827   }
48828 
48829   SDLoc DL(N);
48830 
48831   if (useVPTERNLOG(Subtarget, VT)) {
48832     // Emit a VPTERNLOG node directly - 0xCA is the imm code for A?B:C.
48833     // VPTERNLOG is only available as vXi32/64-bit types.
48834     MVT OpSVT = EltSizeInBits <= 32 ? MVT::i32 : MVT::i64;
48835     MVT OpVT =
48836         MVT::getVectorVT(OpSVT, VT.getSizeInBits() / OpSVT.getSizeInBits());
48837     SDValue A = DAG.getBitcast(OpVT, N0.getOperand(1));
48838     SDValue B = DAG.getBitcast(OpVT, N0.getOperand(0));
48839     SDValue C = DAG.getBitcast(OpVT, N1.getOperand(0));
48840     SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);
48841     SDValue Res = getAVX512Node(X86ISD::VPTERNLOG, DL, OpVT, {A, B, C, Imm},
48842                                 DAG, Subtarget);
48843     return DAG.getBitcast(VT, Res);
48844   }
48845 
48846   SDValue X = N->getOperand(0);
48847   SDValue Y =
48848       DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)),
48849                   DAG.getBitcast(VT, N1.getOperand(0)));
48850   return DAG.getNode(ISD::OR, DL, VT, X, Y);
48851 }
48852 
48853 // Try to match OR(AND(~MASK,X),AND(MASK,Y)) logic pattern.
48854 static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
48855   if (N->getOpcode() != ISD::OR)
48856     return false;
48857 
48858   SDValue N0 = N->getOperand(0);
48859   SDValue N1 = N->getOperand(1);
48860 
48861   // Canonicalize AND to LHS.
48862   if (N1.getOpcode() == ISD::AND)
48863     std::swap(N0, N1);
48864 
48865   // Attempt to match OR(AND(M,Y),ANDNP(M,X)).
48866   if (N0.getOpcode() != ISD::AND || N1.getOpcode() != X86ISD::ANDNP)
48867     return false;
48868 
48869   Mask = N1.getOperand(0);
48870   X = N1.getOperand(1);
48871 
48872   // Check to see if the mask appeared in both the AND and ANDNP.
48873   if (N0.getOperand(0) == Mask)
48874     Y = N0.getOperand(1);
48875   else if (N0.getOperand(1) == Mask)
48876     Y = N0.getOperand(0);
48877   else
48878     return false;
48879 
48880   // TODO: Attempt to match against AND(XOR(-1,M),Y) as well, waiting for
48881   // ANDNP combine allows other combines to happen that prevent matching.
48882   return true;
48883 }
48884 
48885 // Try to fold:
48886 //   (or (and (m, y), (pandn m, x)))
48887 // into:
48888 //   (vselect m, x, y)
48889 // As a special case, try to fold:
48890 //   (or (and (m, (sub 0, x)), (pandn m, x)))
48891 // into:
48892 //   (sub (xor X, M), M)
48893 static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG,
48894                                             const X86Subtarget &Subtarget) {
48895   assert(N->getOpcode() == ISD::OR && "Unexpected Opcode");
48896 
48897   EVT VT = N->getValueType(0);
48898   if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||
48899         (VT.is256BitVector() && Subtarget.hasInt256())))
48900     return SDValue();
48901 
48902   SDValue X, Y, Mask;
48903   if (!matchLogicBlend(N, X, Y, Mask))
48904     return SDValue();
48905 
48906   // Validate that X, Y, and Mask are bitcasts, and see through them.
48907   Mask = peekThroughBitcasts(Mask);
48908   X = peekThroughBitcasts(X);
48909   Y = peekThroughBitcasts(Y);
48910 
48911   EVT MaskVT = Mask.getValueType();
48912   unsigned EltBits = MaskVT.getScalarSizeInBits();
48913 
48914   // TODO: Attempt to handle floating point cases as well?
48915   if (!MaskVT.isInteger() || DAG.ComputeNumSignBits(Mask) != EltBits)
48916     return SDValue();
48917 
48918   SDLoc DL(N);
48919 
48920   // Attempt to combine to conditional negate: (sub (xor X, M), M)
48921   if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL,
48922                                                            DAG, Subtarget))
48923     return Res;
48924 
48925   // PBLENDVB is only available on SSE 4.1.
48926   if (!Subtarget.hasSSE41())
48927     return SDValue();
48928 
48929   // If we have VPTERNLOG we should prefer that since PBLENDVB is multiple uops.
48930   if (Subtarget.hasVLX())
48931     return SDValue();
48932 
48933   MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8;
48934 
48935   X = DAG.getBitcast(BlendVT, X);
48936   Y = DAG.getBitcast(BlendVT, Y);
48937   Mask = DAG.getBitcast(BlendVT, Mask);
48938   Mask = DAG.getSelect(DL, BlendVT, Mask, Y, X);
48939   return DAG.getBitcast(VT, Mask);
48940 }
48941 
48942 // Helper function for combineOrCmpEqZeroToCtlzSrl
48943 // Transforms:
48944 //   seteq(cmp x, 0)
48945 //   into:
48946 //   srl(ctlz x), log2(bitsize(x))
48947 // Input pattern is checked by caller.
48948 static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) {
48949   SDValue Cmp = Op.getOperand(1);
48950   EVT VT = Cmp.getOperand(0).getValueType();
48951   unsigned Log2b = Log2_32(VT.getSizeInBits());
48952   SDLoc dl(Op);
48953   SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));
48954   // The result of the shift is true or false, and on X86, the 32-bit
48955   // encoding of shr and lzcnt is more desirable.
48956   SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
48957   SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
48958                             DAG.getConstant(Log2b, dl, MVT::i8));
48959   return Scc;
48960 }
48961 
48962 // Try to transform:
48963 //   zext(or(setcc(eq, (cmp x, 0)), setcc(eq, (cmp y, 0))))
48964 //   into:
48965 //   srl(or(ctlz(x), ctlz(y)), log2(bitsize(x))
48966 // Will also attempt to match more generic cases, eg:
48967 //   zext(or(or(setcc(eq, cmp 0), setcc(eq, cmp 0)), setcc(eq, cmp 0)))
48968 // Only applies if the target supports the FastLZCNT feature.
48969 static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
48970                                            TargetLowering::DAGCombinerInfo &DCI,
48971                                            const X86Subtarget &Subtarget) {
48972   if (DCI.isBeforeLegalize() || !Subtarget.getTargetLowering()->isCtlzFast())
48973     return SDValue();
48974 
48975   auto isORCandidate = [](SDValue N) {
48976     return (N->getOpcode() == ISD::OR && N->hasOneUse());
48977   };
48978 
48979   // Check the zero extend is extending to 32-bit or more. The code generated by
48980   // srl(ctlz) for 16-bit or less variants of the pattern would require extra
48981   // instructions to clear the upper bits.
48982   if (!N->hasOneUse() || !N->getSimpleValueType(0).bitsGE(MVT::i32) ||
48983       !isORCandidate(N->getOperand(0)))
48984     return SDValue();
48985 
48986   // Check the node matches: setcc(eq, cmp 0)
48987   auto isSetCCCandidate = [](SDValue N) {
48988     return N->getOpcode() == X86ISD::SETCC && N->hasOneUse() &&
48989            X86::CondCode(N->getConstantOperandVal(0)) == X86::COND_E &&
48990            N->getOperand(1).getOpcode() == X86ISD::CMP &&
48991            isNullConstant(N->getOperand(1).getOperand(1)) &&
48992            N->getOperand(1).getValueType().bitsGE(MVT::i32);
48993   };
48994 
48995   SDNode *OR = N->getOperand(0).getNode();
48996   SDValue LHS = OR->getOperand(0);
48997   SDValue RHS = OR->getOperand(1);
48998 
48999   // Save nodes matching or(or, setcc(eq, cmp 0)).
49000   SmallVector<SDNode *, 2> ORNodes;
49001   while (((isORCandidate(LHS) && isSetCCCandidate(RHS)) ||
49002           (isORCandidate(RHS) && isSetCCCandidate(LHS)))) {
49003     ORNodes.push_back(OR);
49004     OR = (LHS->getOpcode() == ISD::OR) ? LHS.getNode() : RHS.getNode();
49005     LHS = OR->getOperand(0);
49006     RHS = OR->getOperand(1);
49007   }
49008 
49009   // The last OR node should match or(setcc(eq, cmp 0), setcc(eq, cmp 0)).
49010   if (!(isSetCCCandidate(LHS) && isSetCCCandidate(RHS)) ||
49011       !isORCandidate(SDValue(OR, 0)))
49012     return SDValue();
49013 
49014   // We have a or(setcc(eq, cmp 0), setcc(eq, cmp 0)) pattern, try to lower it
49015   // to
49016   // or(srl(ctlz),srl(ctlz)).
49017   // The dag combiner can then fold it into:
49018   // srl(or(ctlz, ctlz)).
49019   SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, DAG);
49020   SDValue Ret, NewRHS;
49021   if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG)))
49022     Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, NewLHS, NewRHS);
49023 
49024   if (!Ret)
49025     return SDValue();
49026 
49027   // Try to lower nodes matching the or(or, setcc(eq, cmp 0)) pattern.
49028   while (!ORNodes.empty()) {
49029     OR = ORNodes.pop_back_val();
49030     LHS = OR->getOperand(0);
49031     RHS = OR->getOperand(1);
49032     // Swap rhs with lhs to match or(setcc(eq, cmp, 0), or).
49033     if (RHS->getOpcode() == ISD::OR)
49034       std::swap(LHS, RHS);
49035     NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG);
49036     if (!NewRHS)
49037       return SDValue();
49038     Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, Ret, NewRHS);
49039   }
49040 
49041   return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret);
49042 }
49043 
49044 static SDValue foldMaskedMergeImpl(SDValue And0_L, SDValue And0_R,
49045                                    SDValue And1_L, SDValue And1_R,
49046                                    const SDLoc &DL, SelectionDAG &DAG) {
49047   if (!isBitwiseNot(And0_L, true) || !And0_L->hasOneUse())
49048     return SDValue();
49049   SDValue NotOp = And0_L->getOperand(0);
49050   if (NotOp == And1_R)
49051     std::swap(And1_R, And1_L);
49052   if (NotOp != And1_L)
49053     return SDValue();
49054 
49055   // (~(NotOp) & And0_R) | (NotOp & And1_R)
49056   // --> ((And0_R ^ And1_R) & NotOp) ^ And1_R
49057   EVT VT = And1_L->getValueType(0);
49058   SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R);
49059   SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R);
49060   SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp);
49061   SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R);
49062   return Xor1;
49063 }
49064 
49065 /// Fold "masked merge" expressions like `(m & x) | (~m & y)` into the
49066 /// equivalent `((x ^ y) & m) ^ y)` pattern.
49067 /// This is typically a better representation for  targets without a fused
49068 /// "and-not" operation. This function is intended to be called from a
49069 /// `TargetLowering::PerformDAGCombine` callback on `ISD::OR` nodes.
49070 static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG) {
49071   // Note that masked-merge variants using XOR or ADD expressions are
49072   // normalized to OR by InstCombine so we only check for OR.
49073   assert(Node->getOpcode() == ISD::OR && "Must be called with ISD::OR node");
49074   SDValue N0 = Node->getOperand(0);
49075   if (N0->getOpcode() != ISD::AND || !N0->hasOneUse())
49076     return SDValue();
49077   SDValue N1 = Node->getOperand(1);
49078   if (N1->getOpcode() != ISD::AND || !N1->hasOneUse())
49079     return SDValue();
49080 
49081   SDLoc DL(Node);
49082   SDValue N00 = N0->getOperand(0);
49083   SDValue N01 = N0->getOperand(1);
49084   SDValue N10 = N1->getOperand(0);
49085   SDValue N11 = N1->getOperand(1);
49086   if (SDValue Result = foldMaskedMergeImpl(N00, N01, N10, N11, DL, DAG))
49087     return Result;
49088   if (SDValue Result = foldMaskedMergeImpl(N01, N00, N10, N11, DL, DAG))
49089     return Result;
49090   if (SDValue Result = foldMaskedMergeImpl(N10, N11, N00, N01, DL, DAG))
49091     return Result;
49092   if (SDValue Result = foldMaskedMergeImpl(N11, N10, N00, N01, DL, DAG))
49093     return Result;
49094   return SDValue();
49095 }
49096 
49097 /// If this is an add or subtract where one operand is produced by a cmp+setcc,
49098 /// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB}
49099 /// with CMP+{ADC, SBB}.
49100 /// Also try (ADD/SUB)+(AND(SRL,1)) bit extraction pattern with BT+{ADC, SBB}.
49101 static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
49102                                          SDValue X, SDValue Y,
49103                                          SelectionDAG &DAG,
49104                                          bool ZeroSecondOpOnly = false) {
49105   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
49106     return SDValue();
49107 
49108   // Look through a one-use zext.
49109   if (Y.getOpcode() == ISD::ZERO_EXTEND && Y.hasOneUse())
49110     Y = Y.getOperand(0);
49111 
49112   X86::CondCode CC;
49113   SDValue EFLAGS;
49114   if (Y.getOpcode() == X86ISD::SETCC && Y.hasOneUse()) {
49115     CC = (X86::CondCode)Y.getConstantOperandVal(0);
49116     EFLAGS = Y.getOperand(1);
49117   } else if (Y.getOpcode() == ISD::AND && isOneConstant(Y.getOperand(1)) &&
49118              Y.hasOneUse()) {
49119     EFLAGS = LowerAndToBT(Y, ISD::SETNE, DL, DAG, CC);
49120   }
49121 
49122   if (!EFLAGS)
49123     return SDValue();
49124 
49125   // If X is -1 or 0, then we have an opportunity to avoid constants required in
49126   // the general case below.
49127   auto *ConstantX = dyn_cast<ConstantSDNode>(X);
49128   if (ConstantX && !ZeroSecondOpOnly) {
49129     if ((!IsSub && CC == X86::COND_AE && ConstantX->isAllOnes()) ||
49130         (IsSub && CC == X86::COND_B && ConstantX->isZero())) {
49131       // This is a complicated way to get -1 or 0 from the carry flag:
49132       // -1 + SETAE --> -1 + (!CF) --> CF ? -1 : 0 --> SBB %eax, %eax
49133       //  0 - SETB  -->  0 -  (CF) --> CF ? -1 : 0 --> SBB %eax, %eax
49134       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
49135                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
49136                          EFLAGS);
49137     }
49138 
49139     if ((!IsSub && CC == X86::COND_BE && ConstantX->isAllOnes()) ||
49140         (IsSub && CC == X86::COND_A && ConstantX->isZero())) {
49141       if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
49142           EFLAGS.getValueType().isInteger() &&
49143           !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
49144         // Swap the operands of a SUB, and we have the same pattern as above.
49145         // -1 + SETBE (SUB A, B) --> -1 + SETAE (SUB B, A) --> SUB + SBB
49146         //  0 - SETA  (SUB A, B) -->  0 - SETB  (SUB B, A) --> SUB + SBB
49147         SDValue NewSub = DAG.getNode(
49148             X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
49149             EFLAGS.getOperand(1), EFLAGS.getOperand(0));
49150         SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
49151         return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
49152                            DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
49153                            NewEFLAGS);
49154       }
49155     }
49156   }
49157 
49158   if (CC == X86::COND_B) {
49159     // X + SETB Z --> adc X, 0
49160     // X - SETB Z --> sbb X, 0
49161     return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
49162                        DAG.getVTList(VT, MVT::i32), X,
49163                        DAG.getConstant(0, DL, VT), EFLAGS);
49164   }
49165 
49166   if (ZeroSecondOpOnly)
49167     return SDValue();
49168 
49169   if (CC == X86::COND_A) {
49170     // Try to convert COND_A into COND_B in an attempt to facilitate
49171     // materializing "setb reg".
49172     //
49173     // Do not flip "e > c", where "c" is a constant, because Cmp instruction
49174     // cannot take an immediate as its first operand.
49175     //
49176     if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
49177         EFLAGS.getValueType().isInteger() &&
49178         !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
49179       SDValue NewSub =
49180           DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
49181                       EFLAGS.getOperand(1), EFLAGS.getOperand(0));
49182       SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
49183       return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
49184                          DAG.getVTList(VT, MVT::i32), X,
49185                          DAG.getConstant(0, DL, VT), NewEFLAGS);
49186     }
49187   }
49188 
49189   if (CC == X86::COND_AE) {
49190     // X + SETAE --> sbb X, -1
49191     // X - SETAE --> adc X, -1
49192     return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
49193                        DAG.getVTList(VT, MVT::i32), X,
49194                        DAG.getConstant(-1, DL, VT), EFLAGS);
49195   }
49196 
49197   if (CC == X86::COND_BE) {
49198     // X + SETBE --> sbb X, -1
49199     // X - SETBE --> adc X, -1
49200     // Try to convert COND_BE into COND_AE in an attempt to facilitate
49201     // materializing "setae reg".
49202     //
49203     // Do not flip "e <= c", where "c" is a constant, because Cmp instruction
49204     // cannot take an immediate as its first operand.
49205     //
49206     if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
49207         EFLAGS.getValueType().isInteger() &&
49208         !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
49209       SDValue NewSub =
49210           DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
49211                       EFLAGS.getOperand(1), EFLAGS.getOperand(0));
49212       SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
49213       return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
49214                          DAG.getVTList(VT, MVT::i32), X,
49215                          DAG.getConstant(-1, DL, VT), NewEFLAGS);
49216     }
49217   }
49218 
49219   if (CC != X86::COND_E && CC != X86::COND_NE)
49220     return SDValue();
49221 
49222   if (EFLAGS.getOpcode() != X86ISD::CMP || !EFLAGS.hasOneUse() ||
49223       !X86::isZeroNode(EFLAGS.getOperand(1)) ||
49224       !EFLAGS.getOperand(0).getValueType().isInteger())
49225     return SDValue();
49226 
49227   SDValue Z = EFLAGS.getOperand(0);
49228   EVT ZVT = Z.getValueType();
49229 
49230   // If X is -1 or 0, then we have an opportunity to avoid constants required in
49231   // the general case below.
49232   if (ConstantX) {
49233     // 'neg' sets the carry flag when Z != 0, so create 0 or -1 using 'sbb' with
49234     // fake operands:
49235     //  0 - (Z != 0) --> sbb %eax, %eax, (neg Z)
49236     // -1 + (Z == 0) --> sbb %eax, %eax, (neg Z)
49237     if ((IsSub && CC == X86::COND_NE && ConstantX->isZero()) ||
49238         (!IsSub && CC == X86::COND_E && ConstantX->isAllOnes())) {
49239       SDValue Zero = DAG.getConstant(0, DL, ZVT);
49240       SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
49241       SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
49242       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
49243                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
49244                          SDValue(Neg.getNode(), 1));
49245     }
49246 
49247     // cmp with 1 sets the carry flag when Z == 0, so create 0 or -1 using 'sbb'
49248     // with fake operands:
49249     //  0 - (Z == 0) --> sbb %eax, %eax, (cmp Z, 1)
49250     // -1 + (Z != 0) --> sbb %eax, %eax, (cmp Z, 1)
49251     if ((IsSub && CC == X86::COND_E && ConstantX->isZero()) ||
49252         (!IsSub && CC == X86::COND_NE && ConstantX->isAllOnes())) {
49253       SDValue One = DAG.getConstant(1, DL, ZVT);
49254       SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
49255       SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
49256       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
49257                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
49258                          Cmp1.getValue(1));
49259     }
49260   }
49261 
49262   // (cmp Z, 1) sets the carry flag if Z is 0.
49263   SDValue One = DAG.getConstant(1, DL, ZVT);
49264   SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
49265   SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
49266 
49267   // Add the flags type for ADC/SBB nodes.
49268   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
49269 
49270   // X - (Z != 0) --> sub X, (zext(setne Z, 0)) --> adc X, -1, (cmp Z, 1)
49271   // X + (Z != 0) --> add X, (zext(setne Z, 0)) --> sbb X, -1, (cmp Z, 1)
49272   if (CC == X86::COND_NE)
49273     return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
49274                        DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1));
49275 
49276   // X - (Z == 0) --> sub X, (zext(sete  Z, 0)) --> sbb X, 0, (cmp Z, 1)
49277   // X + (Z == 0) --> add X, (zext(sete  Z, 0)) --> adc X, 0, (cmp Z, 1)
49278   return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X,
49279                      DAG.getConstant(0, DL, VT), Cmp1.getValue(1));
49280 }
49281 
49282 /// If this is an add or subtract where one operand is produced by a cmp+setcc,
49283 /// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB}
49284 /// with CMP+{ADC, SBB}.
49285 static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
49286   bool IsSub = N->getOpcode() == ISD::SUB;
49287   SDValue X = N->getOperand(0);
49288   SDValue Y = N->getOperand(1);
49289   EVT VT = N->getValueType(0);
49290   SDLoc DL(N);
49291 
49292   if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, X, Y, DAG))
49293     return ADCOrSBB;
49294 
49295   // Commute and try again (negate the result for subtracts).
49296   if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, Y, X, DAG)) {
49297     if (IsSub)
49298       ADCOrSBB =
49299           DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), ADCOrSBB);
49300     return ADCOrSBB;
49301   }
49302 
49303   return SDValue();
49304 }
49305 
49306 static SDValue combineOrXorWithSETCC(SDNode *N, SDValue N0, SDValue N1,
49307                                      SelectionDAG &DAG) {
49308   assert((N->getOpcode() == ISD::XOR || N->getOpcode() == ISD::OR) &&
49309          "Unexpected opcode");
49310 
49311   // Delegate to combineAddOrSubToADCOrSBB if we have:
49312   //
49313   //   (xor/or (zero_extend (setcc)) imm)
49314   //
49315   // where imm is odd if and only if we have xor, in which case the XOR/OR are
49316   // equivalent to a SUB/ADD, respectively.
49317   if (N0.getOpcode() == ISD::ZERO_EXTEND &&
49318       N0.getOperand(0).getOpcode() == X86ISD::SETCC && N0.hasOneUse()) {
49319     if (auto *N1C = dyn_cast<ConstantSDNode>(N1)) {
49320       bool IsSub = N->getOpcode() == ISD::XOR;
49321       bool N1COdd = N1C->getZExtValue() & 1;
49322       if (IsSub ? N1COdd : !N1COdd) {
49323         SDLoc DL(N);
49324         EVT VT = N->getValueType(0);
49325         if (SDValue R = combineAddOrSubToADCOrSBB(IsSub, DL, VT, N1, N0, DAG))
49326           return R;
49327       }
49328     }
49329   }
49330 
49331   return SDValue();
49332 }
49333 
49334 static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
49335                          TargetLowering::DAGCombinerInfo &DCI,
49336                          const X86Subtarget &Subtarget) {
49337   SDValue N0 = N->getOperand(0);
49338   SDValue N1 = N->getOperand(1);
49339   EVT VT = N->getValueType(0);
49340   SDLoc dl(N);
49341   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49342 
49343   // If this is SSE1 only convert to FOR to avoid scalarization.
49344   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
49345     return DAG.getBitcast(MVT::v4i32,
49346                           DAG.getNode(X86ISD::FOR, dl, MVT::v4f32,
49347                                       DAG.getBitcast(MVT::v4f32, N0),
49348                                       DAG.getBitcast(MVT::v4f32, N1)));
49349   }
49350 
49351   // Match any-of bool scalar reductions into a bitcast/movmsk + cmp.
49352   // TODO: Support multiple SrcOps.
49353   if (VT == MVT::i1) {
49354     SmallVector<SDValue, 2> SrcOps;
49355     SmallVector<APInt, 2> SrcPartials;
49356     if (matchScalarReduction(SDValue(N, 0), ISD::OR, SrcOps, &SrcPartials) &&
49357         SrcOps.size() == 1) {
49358       unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
49359       EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
49360       SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
49361       if (!Mask && TLI.isTypeLegal(SrcOps[0].getValueType()))
49362         Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
49363       if (Mask) {
49364         assert(SrcPartials[0].getBitWidth() == NumElts &&
49365                "Unexpected partial reduction mask");
49366         SDValue ZeroBits = DAG.getConstant(0, dl, MaskVT);
49367         SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
49368         Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits);
49369         return DAG.getSetCC(dl, MVT::i1, Mask, ZeroBits, ISD::SETNE);
49370       }
49371     }
49372   }
49373 
49374   if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
49375     return R;
49376 
49377   if (SDValue R = combineBitOpWithShift(N, DAG))
49378     return R;
49379 
49380   if (SDValue R = combineBitOpWithPACK(N, DAG))
49381     return R;
49382 
49383   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
49384     return FPLogic;
49385 
49386   if (DCI.isBeforeLegalizeOps())
49387     return SDValue();
49388 
49389   if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
49390     return R;
49391 
49392   if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget))
49393     return R;
49394 
49395   if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget))
49396     return R;
49397 
49398   // (0 - SetCC) | C -> (zext (not SetCC)) * (C + 1) - 1 if we can get a LEA out of it.
49399   if ((VT == MVT::i32 || VT == MVT::i64) &&
49400       N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
49401       isNullConstant(N0.getOperand(0))) {
49402     SDValue Cond = N0.getOperand(1);
49403     if (Cond.getOpcode() == ISD::ZERO_EXTEND && Cond.hasOneUse())
49404       Cond = Cond.getOperand(0);
49405 
49406     if (Cond.getOpcode() == X86ISD::SETCC && Cond.hasOneUse()) {
49407       if (auto *CN = dyn_cast<ConstantSDNode>(N1)) {
49408         uint64_t Val = CN->getZExtValue();
49409         if (Val == 1 || Val == 2 || Val == 3 || Val == 4 || Val == 7 || Val == 8) {
49410           X86::CondCode CCode = (X86::CondCode)Cond.getConstantOperandVal(0);
49411           CCode = X86::GetOppositeBranchCondition(CCode);
49412           SDValue NotCond = getSETCC(CCode, Cond.getOperand(1), SDLoc(Cond), DAG);
49413 
49414           SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT);
49415           R = DAG.getNode(ISD::MUL, dl, VT, R, DAG.getConstant(Val + 1, dl, VT));
49416           R = DAG.getNode(ISD::SUB, dl, VT, R, DAG.getConstant(1, dl, VT));
49417           return R;
49418         }
49419       }
49420     }
49421   }
49422 
49423   // Combine OR(X,KSHIFTL(Y,Elts/2)) -> CONCAT_VECTORS(X,Y) == KUNPCK(X,Y).
49424   // Combine OR(KSHIFTL(X,Elts/2),Y) -> CONCAT_VECTORS(Y,X) == KUNPCK(Y,X).
49425   // iff the upper elements of the non-shifted arg are zero.
49426   // KUNPCK require 16+ bool vector elements.
49427   if (N0.getOpcode() == X86ISD::KSHIFTL || N1.getOpcode() == X86ISD::KSHIFTL) {
49428     unsigned NumElts = VT.getVectorNumElements();
49429     unsigned HalfElts = NumElts / 2;
49430     APInt UpperElts = APInt::getHighBitsSet(NumElts, HalfElts);
49431     if (NumElts >= 16 && N1.getOpcode() == X86ISD::KSHIFTL &&
49432         N1.getConstantOperandAPInt(1) == HalfElts &&
49433         DAG.MaskedVectorIsZero(N0, UpperElts)) {
49434       return DAG.getNode(
49435           ISD::CONCAT_VECTORS, dl, VT,
49436           extractSubVector(N0, 0, DAG, dl, HalfElts),
49437           extractSubVector(N1.getOperand(0), 0, DAG, dl, HalfElts));
49438     }
49439     if (NumElts >= 16 && N0.getOpcode() == X86ISD::KSHIFTL &&
49440         N0.getConstantOperandAPInt(1) == HalfElts &&
49441         DAG.MaskedVectorIsZero(N1, UpperElts)) {
49442       return DAG.getNode(
49443           ISD::CONCAT_VECTORS, dl, VT,
49444           extractSubVector(N1, 0, DAG, dl, HalfElts),
49445           extractSubVector(N0.getOperand(0), 0, DAG, dl, HalfElts));
49446     }
49447   }
49448 
49449   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
49450     // Attempt to recursively combine an OR of shuffles.
49451     SDValue Op(N, 0);
49452     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
49453       return Res;
49454 
49455     // If either operand is a constant mask, then only the elements that aren't
49456     // allones are actually demanded by the other operand.
49457     auto SimplifyUndemandedElts = [&](SDValue Op, SDValue OtherOp) {
49458       APInt UndefElts;
49459       SmallVector<APInt> EltBits;
49460       int NumElts = VT.getVectorNumElements();
49461       int EltSizeInBits = VT.getScalarSizeInBits();
49462       if (!getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts, EltBits))
49463         return false;
49464 
49465       APInt DemandedElts = APInt::getZero(NumElts);
49466       for (int I = 0; I != NumElts; ++I)
49467         if (!EltBits[I].isAllOnes())
49468           DemandedElts.setBit(I);
49469 
49470       return TLI.SimplifyDemandedVectorElts(OtherOp, DemandedElts, DCI);
49471     };
49472     if (SimplifyUndemandedElts(N0, N1) || SimplifyUndemandedElts(N1, N0)) {
49473       if (N->getOpcode() != ISD::DELETED_NODE)
49474         DCI.AddToWorklist(N);
49475       return SDValue(N, 0);
49476     }
49477   }
49478 
49479   // We should fold "masked merge" patterns when `andn` is not available.
49480   if (!Subtarget.hasBMI() && VT.isScalarInteger() && VT != MVT::i1)
49481     if (SDValue R = foldMaskedMerge(N, DAG))
49482       return R;
49483 
49484   if (SDValue R = combineOrXorWithSETCC(N, N0, N1, DAG))
49485     return R;
49486 
49487   return SDValue();
49488 }
49489 
49490 /// Try to turn tests against the signbit in the form of:
49491 ///   XOR(TRUNCATE(SRL(X, size(X)-1)), 1)
49492 /// into:
49493 ///   SETGT(X, -1)
49494 static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
49495   // This is only worth doing if the output type is i8 or i1.
49496   EVT ResultType = N->getValueType(0);
49497   if (ResultType != MVT::i8 && ResultType != MVT::i1)
49498     return SDValue();
49499 
49500   SDValue N0 = N->getOperand(0);
49501   SDValue N1 = N->getOperand(1);
49502 
49503   // We should be performing an xor against a truncated shift.
49504   if (N0.getOpcode() != ISD::TRUNCATE || !N0.hasOneUse())
49505     return SDValue();
49506 
49507   // Make sure we are performing an xor against one.
49508   if (!isOneConstant(N1))
49509     return SDValue();
49510 
49511   // SetCC on x86 zero extends so only act on this if it's a logical shift.
49512   SDValue Shift = N0.getOperand(0);
49513   if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse())
49514     return SDValue();
49515 
49516   // Make sure we are truncating from one of i16, i32 or i64.
49517   EVT ShiftTy = Shift.getValueType();
49518   if (ShiftTy != MVT::i16 && ShiftTy != MVT::i32 && ShiftTy != MVT::i64)
49519     return SDValue();
49520 
49521   // Make sure the shift amount extracts the sign bit.
49522   if (!isa<ConstantSDNode>(Shift.getOperand(1)) ||
49523       Shift.getConstantOperandAPInt(1) != (ShiftTy.getSizeInBits() - 1))
49524     return SDValue();
49525 
49526   // Create a greater-than comparison against -1.
49527   // N.B. Using SETGE against 0 works but we want a canonical looking
49528   // comparison, using SETGT matches up with what TranslateX86CC.
49529   SDLoc DL(N);
49530   SDValue ShiftOp = Shift.getOperand(0);
49531   EVT ShiftOpTy = ShiftOp.getValueType();
49532   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49533   EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(),
49534                                                *DAG.getContext(), ResultType);
49535   SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp,
49536                               DAG.getConstant(-1, DL, ShiftOpTy), ISD::SETGT);
49537   if (SetCCResultType != ResultType)
49538     Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond);
49539   return Cond;
49540 }
49541 
49542 /// Turn vector tests of the signbit in the form of:
49543 ///   xor (sra X, elt_size(X)-1), -1
49544 /// into:
49545 ///   pcmpgt X, -1
49546 ///
49547 /// This should be called before type legalization because the pattern may not
49548 /// persist after that.
49549 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
49550                                          const X86Subtarget &Subtarget) {
49551   EVT VT = N->getValueType(0);
49552   if (!VT.isSimple())
49553     return SDValue();
49554 
49555   switch (VT.getSimpleVT().SimpleTy) {
49556   default: return SDValue();
49557   case MVT::v16i8:
49558   case MVT::v8i16:
49559   case MVT::v4i32:
49560   case MVT::v2i64: if (!Subtarget.hasSSE2()) return SDValue(); break;
49561   case MVT::v32i8:
49562   case MVT::v16i16:
49563   case MVT::v8i32:
49564   case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break;
49565   }
49566 
49567   // There must be a shift right algebraic before the xor, and the xor must be a
49568   // 'not' operation.
49569   SDValue Shift = N->getOperand(0);
49570   SDValue Ones = N->getOperand(1);
49571   if (Shift.getOpcode() != ISD::SRA || !Shift.hasOneUse() ||
49572       !ISD::isBuildVectorAllOnes(Ones.getNode()))
49573     return SDValue();
49574 
49575   // The shift should be smearing the sign bit across each vector element.
49576   auto *ShiftAmt =
49577       isConstOrConstSplat(Shift.getOperand(1), /*AllowUndefs*/ true);
49578   if (!ShiftAmt ||
49579       ShiftAmt->getAPIntValue() != (Shift.getScalarValueSizeInBits() - 1))
49580     return SDValue();
49581 
49582   // Create a greater-than comparison against -1. We don't use the more obvious
49583   // greater-than-or-equal-to-zero because SSE/AVX don't have that instruction.
49584   return DAG.getSetCC(SDLoc(N), VT, Shift.getOperand(0), Ones, ISD::SETGT);
49585 }
49586 
49587 /// Detect patterns of truncation with unsigned saturation:
49588 ///
49589 /// 1. (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type).
49590 ///   Return the source value x to be truncated or SDValue() if the pattern was
49591 ///   not matched.
49592 ///
49593 /// 2. (truncate (smin (smax (x, C1), C2)) to dest_type),
49594 ///   where C1 >= 0 and C2 is unsigned max of destination type.
49595 ///
49596 ///    (truncate (smax (smin (x, C2), C1)) to dest_type)
49597 ///   where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.
49598 ///
49599 ///   These two patterns are equivalent to:
49600 ///   (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type)
49601 ///   So return the smax(x, C1) value to be truncated or SDValue() if the
49602 ///   pattern was not matched.
49603 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
49604                                  const SDLoc &DL) {
49605   EVT InVT = In.getValueType();
49606 
49607   // Saturation with truncation. We truncate from InVT to VT.
49608   assert(InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() &&
49609          "Unexpected types for truncate operation");
49610 
49611   // Match min/max and return limit value as a parameter.
49612   auto MatchMinMax = [](SDValue V, unsigned Opcode, APInt &Limit) -> SDValue {
49613     if (V.getOpcode() == Opcode &&
49614         ISD::isConstantSplatVector(V.getOperand(1).getNode(), Limit))
49615       return V.getOperand(0);
49616     return SDValue();
49617   };
49618 
49619   APInt C1, C2;
49620   if (SDValue UMin = MatchMinMax(In, ISD::UMIN, C2))
49621     // C2 should be equal to UINT32_MAX / UINT16_MAX / UINT8_MAX according
49622     // the element size of the destination type.
49623     if (C2.isMask(VT.getScalarSizeInBits()))
49624       return UMin;
49625 
49626   if (SDValue SMin = MatchMinMax(In, ISD::SMIN, C2))
49627     if (MatchMinMax(SMin, ISD::SMAX, C1))
49628       if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()))
49629         return SMin;
49630 
49631   if (SDValue SMax = MatchMinMax(In, ISD::SMAX, C1))
49632     if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, C2))
49633       if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()) &&
49634           C2.uge(C1)) {
49635         return DAG.getNode(ISD::SMAX, DL, InVT, SMin, In.getOperand(1));
49636       }
49637 
49638   return SDValue();
49639 }
49640 
49641 /// Detect patterns of truncation with signed saturation:
49642 /// (truncate (smin ((smax (x, signed_min_of_dest_type)),
49643 ///                  signed_max_of_dest_type)) to dest_type)
49644 /// or:
49645 /// (truncate (smax ((smin (x, signed_max_of_dest_type)),
49646 ///                  signed_min_of_dest_type)) to dest_type).
49647 /// With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type].
49648 /// Return the source value to be truncated or SDValue() if the pattern was not
49649 /// matched.
49650 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {
49651   unsigned NumDstBits = VT.getScalarSizeInBits();
49652   unsigned NumSrcBits = In.getScalarValueSizeInBits();
49653   assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
49654 
49655   auto MatchMinMax = [](SDValue V, unsigned Opcode,
49656                         const APInt &Limit) -> SDValue {
49657     APInt C;
49658     if (V.getOpcode() == Opcode &&
49659         ISD::isConstantSplatVector(V.getOperand(1).getNode(), C) && C == Limit)
49660       return V.getOperand(0);
49661     return SDValue();
49662   };
49663 
49664   APInt SignedMax, SignedMin;
49665   if (MatchPackUS) {
49666     SignedMax = APInt::getAllOnes(NumDstBits).zext(NumSrcBits);
49667     SignedMin = APInt(NumSrcBits, 0);
49668   } else {
49669     SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
49670     SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
49671   }
49672 
49673   if (SDValue SMin = MatchMinMax(In, ISD::SMIN, SignedMax))
49674     if (SDValue SMax = MatchMinMax(SMin, ISD::SMAX, SignedMin))
49675       return SMax;
49676 
49677   if (SDValue SMax = MatchMinMax(In, ISD::SMAX, SignedMin))
49678     if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, SignedMax))
49679       return SMin;
49680 
49681   return SDValue();
49682 }
49683 
49684 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
49685                                       SelectionDAG &DAG,
49686                                       const X86Subtarget &Subtarget) {
49687   if (!Subtarget.hasSSE2() || !VT.isVector())
49688     return SDValue();
49689 
49690   EVT SVT = VT.getVectorElementType();
49691   EVT InVT = In.getValueType();
49692   EVT InSVT = InVT.getVectorElementType();
49693 
49694   // If we're clamping a signed 32-bit vector to 0-255 and the 32-bit vector is
49695   // split across two registers. We can use a packusdw+perm to clamp to 0-65535
49696   // and concatenate at the same time. Then we can use a final vpmovuswb to
49697   // clip to 0-255.
49698   if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
49699       InVT == MVT::v16i32 && VT == MVT::v16i8) {
49700     if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
49701       // Emit a VPACKUSDW+VPERMQ followed by a VPMOVUSWB.
49702       SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
49703                                            DL, DAG, Subtarget);
49704       assert(Mid && "Failed to pack!");
49705       return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid);
49706     }
49707   }
49708 
49709   // vXi32 truncate instructions are available with AVX512F.
49710   // vXi16 truncate instructions are only available with AVX512BW.
49711   // For 256-bit or smaller vectors, we require VLX.
49712   // FIXME: We could widen truncates to 512 to remove the VLX restriction.
49713   // If the result type is 256-bits or larger and we have disable 512-bit
49714   // registers, we should go ahead and use the pack instructions if possible.
49715   bool PreferAVX512 = ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||
49716                        (Subtarget.hasBWI() && InSVT == MVT::i16)) &&
49717                       (InVT.getSizeInBits() > 128) &&
49718                       (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) &&
49719                       !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256);
49720 
49721   if (!PreferAVX512 && VT.getVectorNumElements() > 1 &&
49722       isPowerOf2_32(VT.getVectorNumElements()) &&
49723       (SVT == MVT::i8 || SVT == MVT::i16) &&
49724       (InSVT == MVT::i16 || InSVT == MVT::i32)) {
49725     if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
49726       // vXi32 -> vXi8 must be performed as PACKUSWB(PACKSSDW,PACKSSDW).
49727       if (SVT == MVT::i8 && InSVT == MVT::i32) {
49728         EVT MidVT = VT.changeVectorElementType(MVT::i16);
49729         SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
49730                                              DAG, Subtarget);
49731         assert(Mid && "Failed to pack!");
49732         SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
49733                                            Subtarget);
49734         assert(V && "Failed to pack!");
49735         return V;
49736       } else if (SVT == MVT::i8 || Subtarget.hasSSE41())
49737         return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG,
49738                                       Subtarget);
49739     }
49740     if (SDValue SSatVal = detectSSatPattern(In, VT))
49741       return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG,
49742                                     Subtarget);
49743   }
49744 
49745   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49746   if (TLI.isTypeLegal(InVT) && InVT.isVector() && SVT != MVT::i1 &&
49747       Subtarget.hasAVX512() && (InSVT != MVT::i16 || Subtarget.hasBWI()) &&
49748       (SVT == MVT::i32 || SVT == MVT::i16 || SVT == MVT::i8)) {
49749     unsigned TruncOpc = 0;
49750     SDValue SatVal;
49751     if (SDValue SSatVal = detectSSatPattern(In, VT)) {
49752       SatVal = SSatVal;
49753       TruncOpc = X86ISD::VTRUNCS;
49754     } else if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) {
49755       SatVal = USatVal;
49756       TruncOpc = X86ISD::VTRUNCUS;
49757     }
49758     if (SatVal) {
49759       unsigned ResElts = VT.getVectorNumElements();
49760       // If the input type is less than 512 bits and we don't have VLX, we need
49761       // to widen to 512 bits.
49762       if (!Subtarget.hasVLX() && !InVT.is512BitVector()) {
49763         unsigned NumConcats = 512 / InVT.getSizeInBits();
49764         ResElts *= NumConcats;
49765         SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT));
49766         ConcatOps[0] = SatVal;
49767         InVT = EVT::getVectorVT(*DAG.getContext(), InSVT,
49768                                 NumConcats * InVT.getVectorNumElements());
49769         SatVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, InVT, ConcatOps);
49770       }
49771       // Widen the result if its narrower than 128 bits.
49772       if (ResElts * SVT.getSizeInBits() < 128)
49773         ResElts = 128 / SVT.getSizeInBits();
49774       EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), SVT, ResElts);
49775       SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
49776       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
49777                          DAG.getIntPtrConstant(0, DL));
49778     }
49779   }
49780 
49781   return SDValue();
49782 }
49783 
49784 /// This function detects the AVG pattern between vectors of unsigned i8/i16,
49785 /// which is c = (a + b + 1) / 2, and replace this operation with the efficient
49786 /// ISD::AVGCEILU (AVG) instruction.
49787 static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG,
49788                                 const X86Subtarget &Subtarget,
49789                                 const SDLoc &DL) {
49790   if (!VT.isVector())
49791     return SDValue();
49792   EVT InVT = In.getValueType();
49793   unsigned NumElems = VT.getVectorNumElements();
49794 
49795   EVT ScalarVT = VT.getVectorElementType();
49796   if (!((ScalarVT == MVT::i8 || ScalarVT == MVT::i16) && NumElems >= 2))
49797     return SDValue();
49798 
49799   // InScalarVT is the intermediate type in AVG pattern and it should be greater
49800   // than the original input type (i8/i16).
49801   EVT InScalarVT = InVT.getVectorElementType();
49802   if (InScalarVT.getFixedSizeInBits() <= ScalarVT.getFixedSizeInBits())
49803     return SDValue();
49804 
49805   if (!Subtarget.hasSSE2())
49806     return SDValue();
49807 
49808   // Detect the following pattern:
49809   //
49810   //   %1 = zext <N x i8> %a to <N x i32>
49811   //   %2 = zext <N x i8> %b to <N x i32>
49812   //   %3 = add nuw nsw <N x i32> %1, <i32 1 x N>
49813   //   %4 = add nuw nsw <N x i32> %3, %2
49814   //   %5 = lshr <N x i32> %N, <i32 1 x N>
49815   //   %6 = trunc <N x i32> %5 to <N x i8>
49816   //
49817   // In AVX512, the last instruction can also be a trunc store.
49818   if (In.getOpcode() != ISD::SRL)
49819     return SDValue();
49820 
49821   // A lambda checking the given SDValue is a constant vector and each element
49822   // is in the range [Min, Max].
49823   auto IsConstVectorInRange = [](SDValue V, unsigned Min, unsigned Max) {
49824     return ISD::matchUnaryPredicate(V, [Min, Max](ConstantSDNode *C) {
49825       return !(C->getAPIntValue().ult(Min) || C->getAPIntValue().ugt(Max));
49826     });
49827   };
49828 
49829   auto IsZExtLike = [DAG = &DAG, ScalarVT](SDValue V) {
49830     unsigned MaxActiveBits = DAG->computeKnownBits(V).countMaxActiveBits();
49831     return MaxActiveBits <= ScalarVT.getSizeInBits();
49832   };
49833 
49834   // Check if each element of the vector is right-shifted by one.
49835   SDValue LHS = In.getOperand(0);
49836   SDValue RHS = In.getOperand(1);
49837   if (!IsConstVectorInRange(RHS, 1, 1))
49838     return SDValue();
49839   if (LHS.getOpcode() != ISD::ADD)
49840     return SDValue();
49841 
49842   // Detect a pattern of a + b + 1 where the order doesn't matter.
49843   SDValue Operands[3];
49844   Operands[0] = LHS.getOperand(0);
49845   Operands[1] = LHS.getOperand(1);
49846 
49847   auto AVGBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
49848                        ArrayRef<SDValue> Ops) {
49849     return DAG.getNode(ISD::AVGCEILU, DL, Ops[0].getValueType(), Ops);
49850   };
49851 
49852   auto AVGSplitter = [&](std::array<SDValue, 2> Ops) {
49853     for (SDValue &Op : Ops)
49854       if (Op.getValueType() != VT)
49855         Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
49856     // Pad to a power-of-2 vector, split+apply and extract the original vector.
49857     unsigned NumElemsPow2 = PowerOf2Ceil(NumElems);
49858     EVT Pow2VT = EVT::getVectorVT(*DAG.getContext(), ScalarVT, NumElemsPow2);
49859     if (NumElemsPow2 != NumElems) {
49860       for (SDValue &Op : Ops) {
49861         SmallVector<SDValue, 32> EltsOfOp(NumElemsPow2, DAG.getUNDEF(ScalarVT));
49862         for (unsigned i = 0; i != NumElems; ++i) {
49863           SDValue Idx = DAG.getIntPtrConstant(i, DL);
49864           EltsOfOp[i] =
49865               DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Op, Idx);
49866         }
49867         Op = DAG.getBuildVector(Pow2VT, DL, EltsOfOp);
49868       }
49869     }
49870     SDValue Res = SplitOpsAndApply(DAG, Subtarget, DL, Pow2VT, Ops, AVGBuilder);
49871     if (NumElemsPow2 == NumElems)
49872       return Res;
49873     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
49874                        DAG.getIntPtrConstant(0, DL));
49875   };
49876 
49877   // Take care of the case when one of the operands is a constant vector whose
49878   // element is in the range [1, 256].
49879   if (IsConstVectorInRange(Operands[1], 1, ScalarVT == MVT::i8 ? 256 : 65536) &&
49880       IsZExtLike(Operands[0])) {
49881     // The pattern is detected. Subtract one from the constant vector, then
49882     // demote it and emit X86ISD::AVG instruction.
49883     SDValue VecOnes = DAG.getConstant(1, DL, InVT);
49884     Operands[1] = DAG.getNode(ISD::SUB, DL, InVT, Operands[1], VecOnes);
49885     return AVGSplitter({Operands[0], Operands[1]});
49886   }
49887 
49888   // Matches 'add like' patterns: add(Op0,Op1) + zext(or(Op0,Op1)).
49889   // Match the or case only if its 'add-like' - can be replaced by an add.
49890   auto FindAddLike = [&](SDValue V, SDValue &Op0, SDValue &Op1) {
49891     if (ISD::ADD == V.getOpcode()) {
49892       Op0 = V.getOperand(0);
49893       Op1 = V.getOperand(1);
49894       return true;
49895     }
49896     if (ISD::ZERO_EXTEND != V.getOpcode())
49897       return false;
49898     V = V.getOperand(0);
49899     if (V.getValueType() != VT || ISD::OR != V.getOpcode() ||
49900         !DAG.haveNoCommonBitsSet(V.getOperand(0), V.getOperand(1)))
49901       return false;
49902     Op0 = V.getOperand(0);
49903     Op1 = V.getOperand(1);
49904     return true;
49905   };
49906 
49907   SDValue Op0, Op1;
49908   if (FindAddLike(Operands[0], Op0, Op1))
49909     std::swap(Operands[0], Operands[1]);
49910   else if (!FindAddLike(Operands[1], Op0, Op1))
49911     return SDValue();
49912   Operands[2] = Op0;
49913   Operands[1] = Op1;
49914 
49915   // Now we have three operands of two additions. Check that one of them is a
49916   // constant vector with ones, and the other two can be promoted from i8/i16.
49917   for (SDValue &Op : Operands) {
49918     if (!IsConstVectorInRange(Op, 1, 1))
49919       continue;
49920     std::swap(Op, Operands[2]);
49921 
49922     // Check if Operands[0] and Operands[1] are results of type promotion.
49923     for (int j = 0; j < 2; ++j)
49924       if (Operands[j].getValueType() != VT)
49925         if (!IsZExtLike(Operands[j]))
49926           return SDValue();
49927 
49928     // The pattern is detected, emit X86ISD::AVG instruction(s).
49929     return AVGSplitter({Operands[0], Operands[1]});
49930   }
49931 
49932   return SDValue();
49933 }
49934 
49935 static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
49936                            TargetLowering::DAGCombinerInfo &DCI,
49937                            const X86Subtarget &Subtarget) {
49938   LoadSDNode *Ld = cast<LoadSDNode>(N);
49939   EVT RegVT = Ld->getValueType(0);
49940   EVT MemVT = Ld->getMemoryVT();
49941   SDLoc dl(Ld);
49942   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49943 
49944   // For chips with slow 32-byte unaligned loads, break the 32-byte operation
49945   // into two 16-byte operations. Also split non-temporal aligned loads on
49946   // pre-AVX2 targets as 32-byte loads will lower to regular temporal loads.
49947   ISD::LoadExtType Ext = Ld->getExtensionType();
49948   unsigned Fast;
49949   if (RegVT.is256BitVector() && !DCI.isBeforeLegalizeOps() &&
49950       Ext == ISD::NON_EXTLOAD &&
49951       ((Ld->isNonTemporal() && !Subtarget.hasInt256() &&
49952         Ld->getAlign() >= Align(16)) ||
49953        (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), RegVT,
49954                                *Ld->getMemOperand(), &Fast) &&
49955         !Fast))) {
49956     unsigned NumElems = RegVT.getVectorNumElements();
49957     if (NumElems < 2)
49958       return SDValue();
49959 
49960     unsigned HalfOffset = 16;
49961     SDValue Ptr1 = Ld->getBasePtr();
49962     SDValue Ptr2 =
49963         DAG.getMemBasePlusOffset(Ptr1, TypeSize::getFixed(HalfOffset), dl);
49964     EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
49965                                   NumElems / 2);
49966     SDValue Load1 =
49967         DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr1, Ld->getPointerInfo(),
49968                     Ld->getOriginalAlign(),
49969                     Ld->getMemOperand()->getFlags());
49970     SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2,
49971                                 Ld->getPointerInfo().getWithOffset(HalfOffset),
49972                                 Ld->getOriginalAlign(),
49973                                 Ld->getMemOperand()->getFlags());
49974     SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
49975                              Load1.getValue(1), Load2.getValue(1));
49976 
49977     SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2);
49978     return DCI.CombineTo(N, NewVec, TF, true);
49979   }
49980 
49981   // Bool vector load - attempt to cast to an integer, as we have good
49982   // (vXiY *ext(vXi1 bitcast(iX))) handling.
49983   if (Ext == ISD::NON_EXTLOAD && !Subtarget.hasAVX512() && RegVT.isVector() &&
49984       RegVT.getScalarType() == MVT::i1 && DCI.isBeforeLegalize()) {
49985     unsigned NumElts = RegVT.getVectorNumElements();
49986     EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
49987     if (TLI.isTypeLegal(IntVT)) {
49988       SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(),
49989                                     Ld->getPointerInfo(),
49990                                     Ld->getOriginalAlign(),
49991                                     Ld->getMemOperand()->getFlags());
49992       SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad);
49993       return DCI.CombineTo(N, BoolVec, IntLoad.getValue(1), true);
49994     }
49995   }
49996 
49997   // If we also load/broadcast this to a wider type, then just extract the
49998   // lowest subvector.
49999   if (Ext == ISD::NON_EXTLOAD && Subtarget.hasAVX() && Ld->isSimple() &&
50000       (RegVT.is128BitVector() || RegVT.is256BitVector())) {
50001     SDValue Ptr = Ld->getBasePtr();
50002     SDValue Chain = Ld->getChain();
50003     for (SDNode *User : Chain->uses()) {
50004       auto *UserLd = dyn_cast<MemSDNode>(User);
50005       if (User != N && UserLd &&
50006           (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD ||
50007            User->getOpcode() == X86ISD::VBROADCAST_LOAD ||
50008            ISD::isNormalLoad(User)) &&
50009           UserLd->getChain() == Chain && !User->hasAnyUseOfValue(1) &&
50010           User->getValueSizeInBits(0).getFixedValue() >
50011               RegVT.getFixedSizeInBits()) {
50012         if (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
50013             UserLd->getBasePtr() == Ptr &&
50014             UserLd->getMemoryVT().getSizeInBits() == MemVT.getSizeInBits()) {
50015           SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N),
50016                                              RegVT.getSizeInBits());
50017           Extract = DAG.getBitcast(RegVT, Extract);
50018           return DCI.CombineTo(N, Extract, SDValue(User, 1));
50019         }
50020         auto MatchingBits = [](const APInt &Undefs, const APInt &UserUndefs,
50021                                ArrayRef<APInt> Bits, ArrayRef<APInt> UserBits) {
50022           for (unsigned I = 0, E = Undefs.getBitWidth(); I != E; ++I) {
50023             if (Undefs[I])
50024               continue;
50025             if (UserUndefs[I] || Bits[I] != UserBits[I])
50026               return false;
50027           }
50028           return true;
50029         };
50030         // See if we are loading a constant that matches in the lower
50031         // bits of a longer constant (but from a different constant pool ptr).
50032         EVT UserVT = User->getValueType(0);
50033         SDValue UserPtr = UserLd->getBasePtr();
50034         const Constant *LdC = getTargetConstantFromBasePtr(Ptr);
50035         const Constant *UserC = getTargetConstantFromBasePtr(UserPtr);
50036         if (LdC && UserC && UserPtr != Ptr) {
50037           unsigned LdSize = LdC->getType()->getPrimitiveSizeInBits();
50038           unsigned UserSize = UserC->getType()->getPrimitiveSizeInBits();
50039           if (LdSize < UserSize || !ISD::isNormalLoad(User)) {
50040             APInt Undefs, UserUndefs;
50041             SmallVector<APInt> Bits, UserBits;
50042             unsigned NumBits = std::min(RegVT.getScalarSizeInBits(),
50043                                         UserVT.getScalarSizeInBits());
50044             if (getTargetConstantBitsFromNode(SDValue(N, 0), NumBits, Undefs,
50045                                               Bits) &&
50046                 getTargetConstantBitsFromNode(SDValue(User, 0), NumBits,
50047                                               UserUndefs, UserBits)) {
50048               if (MatchingBits(Undefs, UserUndefs, Bits, UserBits)) {
50049                 SDValue Extract = extractSubVector(
50050                     SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits());
50051                 Extract = DAG.getBitcast(RegVT, Extract);
50052                 return DCI.CombineTo(N, Extract, SDValue(User, 1));
50053               }
50054             }
50055           }
50056         }
50057       }
50058     }
50059   }
50060 
50061   // Cast ptr32 and ptr64 pointers to the default address space before a load.
50062   unsigned AddrSpace = Ld->getAddressSpace();
50063   if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR ||
50064       AddrSpace == X86AS::PTR32_UPTR) {
50065     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
50066     if (PtrVT != Ld->getBasePtr().getSimpleValueType()) {
50067       SDValue Cast =
50068           DAG.getAddrSpaceCast(dl, PtrVT, Ld->getBasePtr(), AddrSpace, 0);
50069       return DAG.getExtLoad(Ext, dl, RegVT, Ld->getChain(), Cast,
50070                             Ld->getPointerInfo(), MemVT, Ld->getOriginalAlign(),
50071                             Ld->getMemOperand()->getFlags());
50072     }
50073   }
50074 
50075   return SDValue();
50076 }
50077 
50078 /// If V is a build vector of boolean constants and exactly one of those
50079 /// constants is true, return the operand index of that true element.
50080 /// Otherwise, return -1.
50081 static int getOneTrueElt(SDValue V) {
50082   // This needs to be a build vector of booleans.
50083   // TODO: Checking for the i1 type matches the IR definition for the mask,
50084   // but the mask check could be loosened to i8 or other types. That might
50085   // also require checking more than 'allOnesValue'; eg, the x86 HW
50086   // instructions only require that the MSB is set for each mask element.
50087   // The ISD::MSTORE comments/definition do not specify how the mask operand
50088   // is formatted.
50089   auto *BV = dyn_cast<BuildVectorSDNode>(V);
50090   if (!BV || BV->getValueType(0).getVectorElementType() != MVT::i1)
50091     return -1;
50092 
50093   int TrueIndex = -1;
50094   unsigned NumElts = BV->getValueType(0).getVectorNumElements();
50095   for (unsigned i = 0; i < NumElts; ++i) {
50096     const SDValue &Op = BV->getOperand(i);
50097     if (Op.isUndef())
50098       continue;
50099     auto *ConstNode = dyn_cast<ConstantSDNode>(Op);
50100     if (!ConstNode)
50101       return -1;
50102     if (ConstNode->getAPIntValue().countr_one() >= 1) {
50103       // If we already found a one, this is too many.
50104       if (TrueIndex >= 0)
50105         return -1;
50106       TrueIndex = i;
50107     }
50108   }
50109   return TrueIndex;
50110 }
50111 
50112 /// Given a masked memory load/store operation, return true if it has one mask
50113 /// bit set. If it has one mask bit set, then also return the memory address of
50114 /// the scalar element to load/store, the vector index to insert/extract that
50115 /// scalar element, and the alignment for the scalar memory access.
50116 static bool getParamsForOneTrueMaskedElt(MaskedLoadStoreSDNode *MaskedOp,
50117                                          SelectionDAG &DAG, SDValue &Addr,
50118                                          SDValue &Index, Align &Alignment,
50119                                          unsigned &Offset) {
50120   int TrueMaskElt = getOneTrueElt(MaskedOp->getMask());
50121   if (TrueMaskElt < 0)
50122     return false;
50123 
50124   // Get the address of the one scalar element that is specified by the mask
50125   // using the appropriate offset from the base pointer.
50126   EVT EltVT = MaskedOp->getMemoryVT().getVectorElementType();
50127   Offset = 0;
50128   Addr = MaskedOp->getBasePtr();
50129   if (TrueMaskElt != 0) {
50130     Offset = TrueMaskElt * EltVT.getStoreSize();
50131     Addr = DAG.getMemBasePlusOffset(Addr, TypeSize::getFixed(Offset),
50132                                     SDLoc(MaskedOp));
50133   }
50134 
50135   Index = DAG.getIntPtrConstant(TrueMaskElt, SDLoc(MaskedOp));
50136   Alignment = commonAlignment(MaskedOp->getOriginalAlign(),
50137                               EltVT.getStoreSize());
50138   return true;
50139 }
50140 
50141 /// If exactly one element of the mask is set for a non-extending masked load,
50142 /// it is a scalar load and vector insert.
50143 /// Note: It is expected that the degenerate cases of an all-zeros or all-ones
50144 /// mask have already been optimized in IR, so we don't bother with those here.
50145 static SDValue
50146 reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG,
50147                              TargetLowering::DAGCombinerInfo &DCI,
50148                              const X86Subtarget &Subtarget) {
50149   assert(ML->isUnindexed() && "Unexpected indexed masked load!");
50150   // TODO: This is not x86-specific, so it could be lifted to DAGCombiner.
50151   // However, some target hooks may need to be added to know when the transform
50152   // is profitable. Endianness would also have to be considered.
50153 
50154   SDValue Addr, VecIndex;
50155   Align Alignment;
50156   unsigned Offset;
50157   if (!getParamsForOneTrueMaskedElt(ML, DAG, Addr, VecIndex, Alignment, Offset))
50158     return SDValue();
50159 
50160   // Load the one scalar element that is specified by the mask using the
50161   // appropriate offset from the base pointer.
50162   SDLoc DL(ML);
50163   EVT VT = ML->getValueType(0);
50164   EVT EltVT = VT.getVectorElementType();
50165 
50166   EVT CastVT = VT;
50167   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
50168     EltVT = MVT::f64;
50169     CastVT = VT.changeVectorElementType(EltVT);
50170   }
50171 
50172   SDValue Load =
50173       DAG.getLoad(EltVT, DL, ML->getChain(), Addr,
50174                   ML->getPointerInfo().getWithOffset(Offset),
50175                   Alignment, ML->getMemOperand()->getFlags());
50176 
50177   SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru());
50178 
50179   // Insert the loaded element into the appropriate place in the vector.
50180   SDValue Insert =
50181       DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, CastVT, PassThru, Load, VecIndex);
50182   Insert = DAG.getBitcast(VT, Insert);
50183   return DCI.CombineTo(ML, Insert, Load.getValue(1), true);
50184 }
50185 
50186 static SDValue
50187 combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
50188                               TargetLowering::DAGCombinerInfo &DCI) {
50189   assert(ML->isUnindexed() && "Unexpected indexed masked load!");
50190   if (!ISD::isBuildVectorOfConstantSDNodes(ML->getMask().getNode()))
50191     return SDValue();
50192 
50193   SDLoc DL(ML);
50194   EVT VT = ML->getValueType(0);
50195 
50196   // If we are loading the first and last elements of a vector, it is safe and
50197   // always faster to load the whole vector. Replace the masked load with a
50198   // vector load and select.
50199   unsigned NumElts = VT.getVectorNumElements();
50200   BuildVectorSDNode *MaskBV = cast<BuildVectorSDNode>(ML->getMask());
50201   bool LoadFirstElt = !isNullConstant(MaskBV->getOperand(0));
50202   bool LoadLastElt = !isNullConstant(MaskBV->getOperand(NumElts - 1));
50203   if (LoadFirstElt && LoadLastElt) {
50204     SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
50205                                 ML->getMemOperand());
50206     SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
50207                                   ML->getPassThru());
50208     return DCI.CombineTo(ML, Blend, VecLd.getValue(1), true);
50209   }
50210 
50211   // Convert a masked load with a constant mask into a masked load and a select.
50212   // This allows the select operation to use a faster kind of select instruction
50213   // (for example, vblendvps -> vblendps).
50214 
50215   // Don't try this if the pass-through operand is already undefined. That would
50216   // cause an infinite loop because that's what we're about to create.
50217   if (ML->getPassThru().isUndef())
50218     return SDValue();
50219 
50220   if (ISD::isBuildVectorAllZeros(ML->getPassThru().getNode()))
50221     return SDValue();
50222 
50223   // The new masked load has an undef pass-through operand. The select uses the
50224   // original pass-through operand.
50225   SDValue NewML = DAG.getMaskedLoad(
50226       VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(),
50227       DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(),
50228       ML->getAddressingMode(), ML->getExtensionType());
50229   SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
50230                                 ML->getPassThru());
50231 
50232   return DCI.CombineTo(ML, Blend, NewML.getValue(1), true);
50233 }
50234 
50235 static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
50236                                  TargetLowering::DAGCombinerInfo &DCI,
50237                                  const X86Subtarget &Subtarget) {
50238   auto *Mld = cast<MaskedLoadSDNode>(N);
50239 
50240   // TODO: Expanding load with constant mask may be optimized as well.
50241   if (Mld->isExpandingLoad())
50242     return SDValue();
50243 
50244   if (Mld->getExtensionType() == ISD::NON_EXTLOAD) {
50245     if (SDValue ScalarLoad =
50246             reduceMaskedLoadToScalarLoad(Mld, DAG, DCI, Subtarget))
50247       return ScalarLoad;
50248 
50249     // TODO: Do some AVX512 subsets benefit from this transform?
50250     if (!Subtarget.hasAVX512())
50251       if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI))
50252         return Blend;
50253   }
50254 
50255   // If the mask value has been legalized to a non-boolean vector, try to
50256   // simplify ops leading up to it. We only demand the MSB of each lane.
50257   SDValue Mask = Mld->getMask();
50258   if (Mask.getScalarValueSizeInBits() != 1) {
50259     EVT VT = Mld->getValueType(0);
50260     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50261     APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits()));
50262     if (TLI.SimplifyDemandedBits(Mask, DemandedBits, DCI)) {
50263       if (N->getOpcode() != ISD::DELETED_NODE)
50264         DCI.AddToWorklist(N);
50265       return SDValue(N, 0);
50266     }
50267     if (SDValue NewMask =
50268             TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG))
50269       return DAG.getMaskedLoad(
50270           VT, SDLoc(N), Mld->getChain(), Mld->getBasePtr(), Mld->getOffset(),
50271           NewMask, Mld->getPassThru(), Mld->getMemoryVT(), Mld->getMemOperand(),
50272           Mld->getAddressingMode(), Mld->getExtensionType());
50273   }
50274 
50275   return SDValue();
50276 }
50277 
50278 /// If exactly one element of the mask is set for a non-truncating masked store,
50279 /// it is a vector extract and scalar store.
50280 /// Note: It is expected that the degenerate cases of an all-zeros or all-ones
50281 /// mask have already been optimized in IR, so we don't bother with those here.
50282 static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
50283                                               SelectionDAG &DAG,
50284                                               const X86Subtarget &Subtarget) {
50285   // TODO: This is not x86-specific, so it could be lifted to DAGCombiner.
50286   // However, some target hooks may need to be added to know when the transform
50287   // is profitable. Endianness would also have to be considered.
50288 
50289   SDValue Addr, VecIndex;
50290   Align Alignment;
50291   unsigned Offset;
50292   if (!getParamsForOneTrueMaskedElt(MS, DAG, Addr, VecIndex, Alignment, Offset))
50293     return SDValue();
50294 
50295   // Extract the one scalar element that is actually being stored.
50296   SDLoc DL(MS);
50297   SDValue Value = MS->getValue();
50298   EVT VT = Value.getValueType();
50299   EVT EltVT = VT.getVectorElementType();
50300   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
50301     EltVT = MVT::f64;
50302     EVT CastVT = VT.changeVectorElementType(EltVT);
50303     Value = DAG.getBitcast(CastVT, Value);
50304   }
50305   SDValue Extract =
50306       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Value, VecIndex);
50307 
50308   // Store that element at the appropriate offset from the base pointer.
50309   return DAG.getStore(MS->getChain(), DL, Extract, Addr,
50310                       MS->getPointerInfo().getWithOffset(Offset),
50311                       Alignment, MS->getMemOperand()->getFlags());
50312 }
50313 
50314 static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
50315                                   TargetLowering::DAGCombinerInfo &DCI,
50316                                   const X86Subtarget &Subtarget) {
50317   MaskedStoreSDNode *Mst = cast<MaskedStoreSDNode>(N);
50318   if (Mst->isCompressingStore())
50319     return SDValue();
50320 
50321   EVT VT = Mst->getValue().getValueType();
50322   SDLoc dl(Mst);
50323   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50324 
50325   if (Mst->isTruncatingStore())
50326     return SDValue();
50327 
50328   if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG, Subtarget))
50329     return ScalarStore;
50330 
50331   // If the mask value has been legalized to a non-boolean vector, try to
50332   // simplify ops leading up to it. We only demand the MSB of each lane.
50333   SDValue Mask = Mst->getMask();
50334   if (Mask.getScalarValueSizeInBits() != 1) {
50335     APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits()));
50336     if (TLI.SimplifyDemandedBits(Mask, DemandedBits, DCI)) {
50337       if (N->getOpcode() != ISD::DELETED_NODE)
50338         DCI.AddToWorklist(N);
50339       return SDValue(N, 0);
50340     }
50341     if (SDValue NewMask =
50342             TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG))
50343       return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Mst->getValue(),
50344                                 Mst->getBasePtr(), Mst->getOffset(), NewMask,
50345                                 Mst->getMemoryVT(), Mst->getMemOperand(),
50346                                 Mst->getAddressingMode());
50347   }
50348 
50349   SDValue Value = Mst->getValue();
50350   if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() &&
50351       TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
50352                             Mst->getMemoryVT())) {
50353     return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0),
50354                               Mst->getBasePtr(), Mst->getOffset(), Mask,
50355                               Mst->getMemoryVT(), Mst->getMemOperand(),
50356                               Mst->getAddressingMode(), true);
50357   }
50358 
50359   return SDValue();
50360 }
50361 
50362 static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
50363                             TargetLowering::DAGCombinerInfo &DCI,
50364                             const X86Subtarget &Subtarget) {
50365   StoreSDNode *St = cast<StoreSDNode>(N);
50366   EVT StVT = St->getMemoryVT();
50367   SDLoc dl(St);
50368   SDValue StoredVal = St->getValue();
50369   EVT VT = StoredVal.getValueType();
50370   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50371 
50372   // Convert a store of vXi1 into a store of iX and a bitcast.
50373   if (!Subtarget.hasAVX512() && VT == StVT && VT.isVector() &&
50374       VT.getVectorElementType() == MVT::i1) {
50375 
50376     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
50377     StoredVal = DAG.getBitcast(NewVT, StoredVal);
50378 
50379     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
50380                         St->getPointerInfo(), St->getOriginalAlign(),
50381                         St->getMemOperand()->getFlags());
50382   }
50383 
50384   // If this is a store of a scalar_to_vector to v1i1, just use a scalar store.
50385   // This will avoid a copy to k-register.
50386   if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() &&
50387       StoredVal.getOpcode() == ISD::SCALAR_TO_VECTOR &&
50388       StoredVal.getOperand(0).getValueType() == MVT::i8) {
50389     SDValue Val = StoredVal.getOperand(0);
50390     // We must store zeros to the unused bits.
50391     Val = DAG.getZeroExtendInReg(Val, dl, MVT::i1);
50392     return DAG.getStore(St->getChain(), dl, Val,
50393                         St->getBasePtr(), St->getPointerInfo(),
50394                         St->getOriginalAlign(),
50395                         St->getMemOperand()->getFlags());
50396   }
50397 
50398   // Widen v2i1/v4i1 stores to v8i1.
50399   if ((VT == MVT::v1i1 || VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT &&
50400       Subtarget.hasAVX512()) {
50401     unsigned NumConcats = 8 / VT.getVectorNumElements();
50402     // We must store zeros to the unused bits.
50403     SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT));
50404     Ops[0] = StoredVal;
50405     StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
50406     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
50407                         St->getPointerInfo(), St->getOriginalAlign(),
50408                         St->getMemOperand()->getFlags());
50409   }
50410 
50411   // Turn vXi1 stores of constants into a scalar store.
50412   if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 ||
50413        VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) &&
50414       ISD::isBuildVectorOfConstantSDNodes(StoredVal.getNode())) {
50415     // If its a v64i1 store without 64-bit support, we need two stores.
50416     if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) {
50417       SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
50418                                       StoredVal->ops().slice(0, 32));
50419       Lo = combinevXi1ConstantToInteger(Lo, DAG);
50420       SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
50421                                       StoredVal->ops().slice(32, 32));
50422       Hi = combinevXi1ConstantToInteger(Hi, DAG);
50423 
50424       SDValue Ptr0 = St->getBasePtr();
50425       SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(4), dl);
50426 
50427       SDValue Ch0 =
50428           DAG.getStore(St->getChain(), dl, Lo, Ptr0, St->getPointerInfo(),
50429                        St->getOriginalAlign(),
50430                        St->getMemOperand()->getFlags());
50431       SDValue Ch1 =
50432           DAG.getStore(St->getChain(), dl, Hi, Ptr1,
50433                        St->getPointerInfo().getWithOffset(4),
50434                        St->getOriginalAlign(),
50435                        St->getMemOperand()->getFlags());
50436       return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
50437     }
50438 
50439     StoredVal = combinevXi1ConstantToInteger(StoredVal, DAG);
50440     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
50441                         St->getPointerInfo(), St->getOriginalAlign(),
50442                         St->getMemOperand()->getFlags());
50443   }
50444 
50445   // If we are saving a 32-byte vector and 32-byte stores are slow, such as on
50446   // Sandy Bridge, perform two 16-byte stores.
50447   unsigned Fast;
50448   if (VT.is256BitVector() && StVT == VT &&
50449       TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
50450                              *St->getMemOperand(), &Fast) &&
50451       !Fast) {
50452     unsigned NumElems = VT.getVectorNumElements();
50453     if (NumElems < 2)
50454       return SDValue();
50455 
50456     return splitVectorStore(St, DAG);
50457   }
50458 
50459   // Split under-aligned vector non-temporal stores.
50460   if (St->isNonTemporal() && StVT == VT &&
50461       St->getAlign().value() < VT.getStoreSize()) {
50462     // ZMM/YMM nt-stores - either it can be stored as a series of shorter
50463     // vectors or the legalizer can scalarize it to use MOVNTI.
50464     if (VT.is256BitVector() || VT.is512BitVector()) {
50465       unsigned NumElems = VT.getVectorNumElements();
50466       if (NumElems < 2)
50467         return SDValue();
50468       return splitVectorStore(St, DAG);
50469     }
50470 
50471     // XMM nt-stores - scalarize this to f64 nt-stores on SSE4A, else i32/i64
50472     // to use MOVNTI.
50473     if (VT.is128BitVector() && Subtarget.hasSSE2()) {
50474       MVT NTVT = Subtarget.hasSSE4A()
50475                      ? MVT::v2f64
50476                      : (TLI.isTypeLegal(MVT::i64) ? MVT::v2i64 : MVT::v4i32);
50477       return scalarizeVectorStore(St, NTVT, DAG);
50478     }
50479   }
50480 
50481   // Try to optimize v16i16->v16i8 truncating stores when BWI is not
50482   // supported, but avx512f is by extending to v16i32 and truncating.
50483   if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() &&
50484       St->getValue().getOpcode() == ISD::TRUNCATE &&
50485       St->getValue().getOperand(0).getValueType() == MVT::v16i16 &&
50486       TLI.isTruncStoreLegal(MVT::v16i32, MVT::v16i8) &&
50487       St->getValue().hasOneUse() && !DCI.isBeforeLegalizeOps()) {
50488     SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32,
50489                               St->getValue().getOperand(0));
50490     return DAG.getTruncStore(St->getChain(), dl, Ext, St->getBasePtr(),
50491                              MVT::v16i8, St->getMemOperand());
50492   }
50493 
50494   // Try to fold a VTRUNCUS or VTRUNCS into a truncating store.
50495   if (!St->isTruncatingStore() &&
50496       (StoredVal.getOpcode() == X86ISD::VTRUNCUS ||
50497        StoredVal.getOpcode() == X86ISD::VTRUNCS) &&
50498       StoredVal.hasOneUse() &&
50499       TLI.isTruncStoreLegal(StoredVal.getOperand(0).getValueType(), VT)) {
50500     bool IsSigned = StoredVal.getOpcode() == X86ISD::VTRUNCS;
50501     return EmitTruncSStore(IsSigned, St->getChain(),
50502                            dl, StoredVal.getOperand(0), St->getBasePtr(),
50503                            VT, St->getMemOperand(), DAG);
50504   }
50505 
50506   // Try to fold a extract_element(VTRUNC) pattern into a truncating store.
50507   if (!St->isTruncatingStore()) {
50508     auto IsExtractedElement = [](SDValue V) {
50509       if (V.getOpcode() == ISD::TRUNCATE && V.hasOneUse())
50510         V = V.getOperand(0);
50511       unsigned Opc = V.getOpcode();
50512       if ((Opc == ISD::EXTRACT_VECTOR_ELT || Opc == X86ISD::PEXTRW) &&
50513           isNullConstant(V.getOperand(1)) && V.hasOneUse() &&
50514           V.getOperand(0).hasOneUse())
50515         return V.getOperand(0);
50516       return SDValue();
50517     };
50518     if (SDValue Extract = IsExtractedElement(StoredVal)) {
50519       SDValue Trunc = peekThroughOneUseBitcasts(Extract);
50520       if (Trunc.getOpcode() == X86ISD::VTRUNC) {
50521         SDValue Src = Trunc.getOperand(0);
50522         MVT DstVT = Trunc.getSimpleValueType();
50523         MVT SrcVT = Src.getSimpleValueType();
50524         unsigned NumSrcElts = SrcVT.getVectorNumElements();
50525         unsigned NumTruncBits = DstVT.getScalarSizeInBits() * NumSrcElts;
50526         MVT TruncVT = MVT::getVectorVT(DstVT.getScalarType(), NumSrcElts);
50527         if (NumTruncBits == VT.getSizeInBits() &&
50528             TLI.isTruncStoreLegal(SrcVT, TruncVT)) {
50529           return DAG.getTruncStore(St->getChain(), dl, Src, St->getBasePtr(),
50530                                    TruncVT, St->getMemOperand());
50531         }
50532       }
50533     }
50534   }
50535 
50536   // Optimize trunc store (of multiple scalars) to shuffle and store.
50537   // First, pack all of the elements in one place. Next, store to memory
50538   // in fewer chunks.
50539   if (St->isTruncatingStore() && VT.isVector()) {
50540     // Check if we can detect an AVG pattern from the truncation. If yes,
50541     // replace the trunc store by a normal store with the result of X86ISD::AVG
50542     // instruction.
50543     if (DCI.isBeforeLegalize() || TLI.isTypeLegal(St->getMemoryVT()))
50544       if (SDValue Avg = detectAVGPattern(St->getValue(), St->getMemoryVT(), DAG,
50545                                          Subtarget, dl))
50546         return DAG.getStore(St->getChain(), dl, Avg, St->getBasePtr(),
50547                             St->getPointerInfo(), St->getOriginalAlign(),
50548                             St->getMemOperand()->getFlags());
50549 
50550     if (TLI.isTruncStoreLegal(VT, StVT)) {
50551       if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
50552         return EmitTruncSStore(true /* Signed saturation */, St->getChain(),
50553                                dl, Val, St->getBasePtr(),
50554                                St->getMemoryVT(), St->getMemOperand(), DAG);
50555       if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
50556                                           DAG, dl))
50557         return EmitTruncSStore(false /* Unsigned saturation */, St->getChain(),
50558                                dl, Val, St->getBasePtr(),
50559                                St->getMemoryVT(), St->getMemOperand(), DAG);
50560     }
50561 
50562     return SDValue();
50563   }
50564 
50565   // Cast ptr32 and ptr64 pointers to the default address space before a store.
50566   unsigned AddrSpace = St->getAddressSpace();
50567   if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR ||
50568       AddrSpace == X86AS::PTR32_UPTR) {
50569     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
50570     if (PtrVT != St->getBasePtr().getSimpleValueType()) {
50571       SDValue Cast =
50572           DAG.getAddrSpaceCast(dl, PtrVT, St->getBasePtr(), AddrSpace, 0);
50573       return DAG.getTruncStore(
50574           St->getChain(), dl, StoredVal, Cast, St->getPointerInfo(), StVT,
50575           St->getOriginalAlign(), St->getMemOperand()->getFlags(),
50576           St->getAAInfo());
50577     }
50578   }
50579 
50580   // Turn load->store of MMX types into GPR load/stores.  This avoids clobbering
50581   // the FP state in cases where an emms may be missing.
50582   // A preferable solution to the general problem is to figure out the right
50583   // places to insert EMMS.  This qualifies as a quick hack.
50584 
50585   // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
50586   if (VT.getSizeInBits() != 64)
50587     return SDValue();
50588 
50589   const Function &F = DAG.getMachineFunction().getFunction();
50590   bool NoImplicitFloatOps = F.hasFnAttribute(Attribute::NoImplicitFloat);
50591   bool F64IsLegal =
50592       !Subtarget.useSoftFloat() && !NoImplicitFloatOps && Subtarget.hasSSE2();
50593 
50594   if (!F64IsLegal || Subtarget.is64Bit())
50595     return SDValue();
50596 
50597   if (VT == MVT::i64 && isa<LoadSDNode>(St->getValue()) &&
50598       cast<LoadSDNode>(St->getValue())->isSimple() &&
50599       St->getChain().hasOneUse() && St->isSimple()) {
50600     auto *Ld = cast<LoadSDNode>(St->getValue());
50601 
50602     if (!ISD::isNormalLoad(Ld))
50603       return SDValue();
50604 
50605     // Avoid the transformation if there are multiple uses of the loaded value.
50606     if (!Ld->hasNUsesOfValue(1, 0))
50607       return SDValue();
50608 
50609     SDLoc LdDL(Ld);
50610     SDLoc StDL(N);
50611     // Lower to a single movq load/store pair.
50612     SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(),
50613                                 Ld->getBasePtr(), Ld->getMemOperand());
50614 
50615     // Make sure new load is placed in same chain order.
50616     DAG.makeEquivalentMemoryOrdering(Ld, NewLd);
50617     return DAG.getStore(St->getChain(), StDL, NewLd, St->getBasePtr(),
50618                         St->getMemOperand());
50619   }
50620 
50621   // This is similar to the above case, but here we handle a scalar 64-bit
50622   // integer store that is extracted from a vector on a 32-bit target.
50623   // If we have SSE2, then we can treat it like a floating-point double
50624   // to get past legalization. The execution dependencies fixup pass will
50625   // choose the optimal machine instruction for the store if this really is
50626   // an integer or v2f32 rather than an f64.
50627   if (VT == MVT::i64 &&
50628       St->getOperand(1).getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
50629     SDValue OldExtract = St->getOperand(1);
50630     SDValue ExtOp0 = OldExtract.getOperand(0);
50631     unsigned VecSize = ExtOp0.getValueSizeInBits();
50632     EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64);
50633     SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
50634     SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
50635                                      BitCast, OldExtract.getOperand(1));
50636     return DAG.getStore(St->getChain(), dl, NewExtract, St->getBasePtr(),
50637                         St->getPointerInfo(), St->getOriginalAlign(),
50638                         St->getMemOperand()->getFlags());
50639   }
50640 
50641   return SDValue();
50642 }
50643 
50644 static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG,
50645                                      TargetLowering::DAGCombinerInfo &DCI,
50646                                      const X86Subtarget &Subtarget) {
50647   auto *St = cast<MemIntrinsicSDNode>(N);
50648 
50649   SDValue StoredVal = N->getOperand(1);
50650   MVT VT = StoredVal.getSimpleValueType();
50651   EVT MemVT = St->getMemoryVT();
50652 
50653   // Figure out which elements we demand.
50654   unsigned StElts = MemVT.getSizeInBits() / VT.getScalarSizeInBits();
50655   APInt DemandedElts = APInt::getLowBitsSet(VT.getVectorNumElements(), StElts);
50656 
50657   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50658   if (TLI.SimplifyDemandedVectorElts(StoredVal, DemandedElts, DCI)) {
50659     if (N->getOpcode() != ISD::DELETED_NODE)
50660       DCI.AddToWorklist(N);
50661     return SDValue(N, 0);
50662   }
50663 
50664   return SDValue();
50665 }
50666 
50667 /// Return 'true' if this vector operation is "horizontal"
50668 /// and return the operands for the horizontal operation in LHS and RHS.  A
50669 /// horizontal operation performs the binary operation on successive elements
50670 /// of its first operand, then on successive elements of its second operand,
50671 /// returning the resulting values in a vector.  For example, if
50672 ///   A = < float a0, float a1, float a2, float a3 >
50673 /// and
50674 ///   B = < float b0, float b1, float b2, float b3 >
50675 /// then the result of doing a horizontal operation on A and B is
50676 ///   A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
50677 /// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
50678 /// A horizontal-op B, for some already available A and B, and if so then LHS is
50679 /// set to A, RHS to B, and the routine returns 'true'.
50680 static bool isHorizontalBinOp(unsigned HOpcode, SDValue &LHS, SDValue &RHS,
50681                               SelectionDAG &DAG, const X86Subtarget &Subtarget,
50682                               bool IsCommutative,
50683                               SmallVectorImpl<int> &PostShuffleMask) {
50684   // If either operand is undef, bail out. The binop should be simplified.
50685   if (LHS.isUndef() || RHS.isUndef())
50686     return false;
50687 
50688   // Look for the following pattern:
50689   //   A = < float a0, float a1, float a2, float a3 >
50690   //   B = < float b0, float b1, float b2, float b3 >
50691   // and
50692   //   LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6>
50693   //   RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7>
50694   // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
50695   // which is A horizontal-op B.
50696 
50697   MVT VT = LHS.getSimpleValueType();
50698   assert((VT.is128BitVector() || VT.is256BitVector()) &&
50699          "Unsupported vector type for horizontal add/sub");
50700   unsigned NumElts = VT.getVectorNumElements();
50701 
50702   auto GetShuffle = [&](SDValue Op, SDValue &N0, SDValue &N1,
50703                         SmallVectorImpl<int> &ShuffleMask) {
50704     bool UseSubVector = false;
50705     if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
50706         Op.getOperand(0).getValueType().is256BitVector() &&
50707         llvm::isNullConstant(Op.getOperand(1))) {
50708       Op = Op.getOperand(0);
50709       UseSubVector = true;
50710     }
50711     SmallVector<SDValue, 2> SrcOps;
50712     SmallVector<int, 16> SrcMask, ScaledMask;
50713     SDValue BC = peekThroughBitcasts(Op);
50714     if (getTargetShuffleInputs(BC, SrcOps, SrcMask, DAG) &&
50715         !isAnyZero(SrcMask) && all_of(SrcOps, [BC](SDValue Op) {
50716           return Op.getValueSizeInBits() == BC.getValueSizeInBits();
50717         })) {
50718       resolveTargetShuffleInputsAndMask(SrcOps, SrcMask);
50719       if (!UseSubVector && SrcOps.size() <= 2 &&
50720           scaleShuffleElements(SrcMask, NumElts, ScaledMask)) {
50721         N0 = !SrcOps.empty() ? SrcOps[0] : SDValue();
50722         N1 = SrcOps.size() > 1 ? SrcOps[1] : SDValue();
50723         ShuffleMask.assign(ScaledMask.begin(), ScaledMask.end());
50724       }
50725       if (UseSubVector && SrcOps.size() == 1 &&
50726           scaleShuffleElements(SrcMask, 2 * NumElts, ScaledMask)) {
50727         std::tie(N0, N1) = DAG.SplitVector(SrcOps[0], SDLoc(Op));
50728         ArrayRef<int> Mask = ArrayRef<int>(ScaledMask).slice(0, NumElts);
50729         ShuffleMask.assign(Mask.begin(), Mask.end());
50730       }
50731     }
50732   };
50733 
50734   // View LHS in the form
50735   //   LHS = VECTOR_SHUFFLE A, B, LMask
50736   // If LHS is not a shuffle, then pretend it is the identity shuffle:
50737   //   LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1>
50738   // NOTE: A default initialized SDValue represents an UNDEF of type VT.
50739   SDValue A, B;
50740   SmallVector<int, 16> LMask;
50741   GetShuffle(LHS, A, B, LMask);
50742 
50743   // Likewise, view RHS in the form
50744   //   RHS = VECTOR_SHUFFLE C, D, RMask
50745   SDValue C, D;
50746   SmallVector<int, 16> RMask;
50747   GetShuffle(RHS, C, D, RMask);
50748 
50749   // At least one of the operands should be a vector shuffle.
50750   unsigned NumShuffles = (LMask.empty() ? 0 : 1) + (RMask.empty() ? 0 : 1);
50751   if (NumShuffles == 0)
50752     return false;
50753 
50754   if (LMask.empty()) {
50755     A = LHS;
50756     for (unsigned i = 0; i != NumElts; ++i)
50757       LMask.push_back(i);
50758   }
50759 
50760   if (RMask.empty()) {
50761     C = RHS;
50762     for (unsigned i = 0; i != NumElts; ++i)
50763       RMask.push_back(i);
50764   }
50765 
50766   // If we have an unary mask, ensure the other op is set to null.
50767   if (isUndefOrInRange(LMask, 0, NumElts))
50768     B = SDValue();
50769   else if (isUndefOrInRange(LMask, NumElts, NumElts * 2))
50770     A = SDValue();
50771 
50772   if (isUndefOrInRange(RMask, 0, NumElts))
50773     D = SDValue();
50774   else if (isUndefOrInRange(RMask, NumElts, NumElts * 2))
50775     C = SDValue();
50776 
50777   // If A and B occur in reverse order in RHS, then canonicalize by commuting
50778   // RHS operands and shuffle mask.
50779   if (A != C) {
50780     std::swap(C, D);
50781     ShuffleVectorSDNode::commuteMask(RMask);
50782   }
50783   // Check that the shuffles are both shuffling the same vectors.
50784   if (!(A == C && B == D))
50785     return false;
50786 
50787   PostShuffleMask.clear();
50788   PostShuffleMask.append(NumElts, SM_SentinelUndef);
50789 
50790   // LHS and RHS are now:
50791   //   LHS = shuffle A, B, LMask
50792   //   RHS = shuffle A, B, RMask
50793   // Check that the masks correspond to performing a horizontal operation.
50794   // AVX defines horizontal add/sub to operate independently on 128-bit lanes,
50795   // so we just repeat the inner loop if this is a 256-bit op.
50796   unsigned Num128BitChunks = VT.getSizeInBits() / 128;
50797   unsigned NumEltsPer128BitChunk = NumElts / Num128BitChunks;
50798   unsigned NumEltsPer64BitChunk = NumEltsPer128BitChunk / 2;
50799   assert((NumEltsPer128BitChunk % 2 == 0) &&
50800          "Vector type should have an even number of elements in each lane");
50801   for (unsigned j = 0; j != NumElts; j += NumEltsPer128BitChunk) {
50802     for (unsigned i = 0; i != NumEltsPer128BitChunk; ++i) {
50803       // Ignore undefined components.
50804       int LIdx = LMask[i + j], RIdx = RMask[i + j];
50805       if (LIdx < 0 || RIdx < 0 ||
50806           (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) ||
50807           (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts)))
50808         continue;
50809 
50810       // Check that successive odd/even elements are being operated on. If not,
50811       // this is not a horizontal operation.
50812       if (!((RIdx & 1) == 1 && (LIdx + 1) == RIdx) &&
50813           !((LIdx & 1) == 1 && (RIdx + 1) == LIdx && IsCommutative))
50814         return false;
50815 
50816       // Compute the post-shuffle mask index based on where the element
50817       // is stored in the HOP result, and where it needs to be moved to.
50818       int Base = LIdx & ~1u;
50819       int Index = ((Base % NumEltsPer128BitChunk) / 2) +
50820                   ((Base % NumElts) & ~(NumEltsPer128BitChunk - 1));
50821 
50822       // The  low half of the 128-bit result must choose from A.
50823       // The high half of the 128-bit result must choose from B,
50824       // unless B is undef. In that case, we are always choosing from A.
50825       if ((B && Base >= (int)NumElts) || (!B && i >= NumEltsPer64BitChunk))
50826         Index += NumEltsPer64BitChunk;
50827       PostShuffleMask[i + j] = Index;
50828     }
50829   }
50830 
50831   SDValue NewLHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
50832   SDValue NewRHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
50833 
50834   bool IsIdentityPostShuffle =
50835       isSequentialOrUndefInRange(PostShuffleMask, 0, NumElts, 0);
50836   if (IsIdentityPostShuffle)
50837     PostShuffleMask.clear();
50838 
50839   // Avoid 128-bit multi lane shuffles if pre-AVX2 and FP (integer will split).
50840   if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() &&
50841       isMultiLaneShuffleMask(128, VT.getScalarSizeInBits(), PostShuffleMask))
50842     return false;
50843 
50844   // If the source nodes are already used in HorizOps then always accept this.
50845   // Shuffle folding should merge these back together.
50846   bool FoundHorizLHS = llvm::any_of(NewLHS->uses(), [&](SDNode *User) {
50847     return User->getOpcode() == HOpcode && User->getValueType(0) == VT;
50848   });
50849   bool FoundHorizRHS = llvm::any_of(NewRHS->uses(), [&](SDNode *User) {
50850     return User->getOpcode() == HOpcode && User->getValueType(0) == VT;
50851   });
50852   bool ForceHorizOp = FoundHorizLHS && FoundHorizRHS;
50853 
50854   // Assume a SingleSource HOP if we only shuffle one input and don't need to
50855   // shuffle the result.
50856   if (!ForceHorizOp &&
50857       !shouldUseHorizontalOp(NewLHS == NewRHS &&
50858                                  (NumShuffles < 2 || !IsIdentityPostShuffle),
50859                              DAG, Subtarget))
50860     return false;
50861 
50862   LHS = DAG.getBitcast(VT, NewLHS);
50863   RHS = DAG.getBitcast(VT, NewRHS);
50864   return true;
50865 }
50866 
50867 // Try to synthesize horizontal (f)hadd/hsub from (f)adds/subs of shuffles.
50868 static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG,
50869                                          const X86Subtarget &Subtarget) {
50870   EVT VT = N->getValueType(0);
50871   unsigned Opcode = N->getOpcode();
50872   bool IsAdd = (Opcode == ISD::FADD) || (Opcode == ISD::ADD);
50873   SmallVector<int, 8> PostShuffleMask;
50874 
50875   switch (Opcode) {
50876   case ISD::FADD:
50877   case ISD::FSUB:
50878     if ((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
50879         (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) {
50880       SDValue LHS = N->getOperand(0);
50881       SDValue RHS = N->getOperand(1);
50882       auto HorizOpcode = IsAdd ? X86ISD::FHADD : X86ISD::FHSUB;
50883       if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
50884                             PostShuffleMask)) {
50885         SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
50886         if (!PostShuffleMask.empty())
50887           HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
50888                                             DAG.getUNDEF(VT), PostShuffleMask);
50889         return HorizBinOp;
50890       }
50891     }
50892     break;
50893   case ISD::ADD:
50894   case ISD::SUB:
50895     if (Subtarget.hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
50896                                  VT == MVT::v16i16 || VT == MVT::v8i32)) {
50897       SDValue LHS = N->getOperand(0);
50898       SDValue RHS = N->getOperand(1);
50899       auto HorizOpcode = IsAdd ? X86ISD::HADD : X86ISD::HSUB;
50900       if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
50901                             PostShuffleMask)) {
50902         auto HOpBuilder = [HorizOpcode](SelectionDAG &DAG, const SDLoc &DL,
50903                                         ArrayRef<SDValue> Ops) {
50904           return DAG.getNode(HorizOpcode, DL, Ops[0].getValueType(), Ops);
50905         };
50906         SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT,
50907                                               {LHS, RHS}, HOpBuilder);
50908         if (!PostShuffleMask.empty())
50909           HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
50910                                             DAG.getUNDEF(VT), PostShuffleMask);
50911         return HorizBinOp;
50912       }
50913     }
50914     break;
50915   }
50916 
50917   return SDValue();
50918 }
50919 
50920 //  Try to combine the following nodes
50921 //  t29: i64 = X86ISD::Wrapper TargetConstantPool:i64
50922 //    <i32 -2147483648[float -0.000000e+00]> 0
50923 //  t27: v16i32[v16f32],ch = X86ISD::VBROADCAST_LOAD
50924 //    <(load 4 from constant-pool)> t0, t29
50925 //  [t30: v16i32 = bitcast t27]
50926 //  t6: v16i32 = xor t7, t27[t30]
50927 //  t11: v16f32 = bitcast t6
50928 //  t21: v16f32 = X86ISD::VFMULC[X86ISD::VCFMULC] t11, t8
50929 //  into X86ISD::VFCMULC[X86ISD::VFMULC] if possible:
50930 //  t22: v16f32 = bitcast t7
50931 //  t23: v16f32 = X86ISD::VFCMULC[X86ISD::VFMULC] t8, t22
50932 //  t24: v32f16 = bitcast t23
50933 static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
50934                                   const X86Subtarget &Subtarget) {
50935   EVT VT = N->getValueType(0);
50936   SDValue LHS = N->getOperand(0);
50937   SDValue RHS = N->getOperand(1);
50938   int CombineOpcode =
50939       N->getOpcode() == X86ISD::VFCMULC ? X86ISD::VFMULC : X86ISD::VFCMULC;
50940   auto combineConjugation = [&](SDValue &r) {
50941     if (LHS->getOpcode() == ISD::BITCAST && RHS.hasOneUse()) {
50942       SDValue XOR = LHS.getOperand(0);
50943       if (XOR->getOpcode() == ISD::XOR && XOR.hasOneUse()) {
50944         KnownBits XORRHS = DAG.computeKnownBits(XOR.getOperand(1));
50945         if (XORRHS.isConstant()) {
50946           APInt ConjugationInt32 = APInt(32, 0x80000000, true);
50947           APInt ConjugationInt64 = APInt(64, 0x8000000080000000ULL, true);
50948           if ((XORRHS.getBitWidth() == 32 &&
50949                XORRHS.getConstant() == ConjugationInt32) ||
50950               (XORRHS.getBitWidth() == 64 &&
50951                XORRHS.getConstant() == ConjugationInt64)) {
50952             SelectionDAG::FlagInserter FlagsInserter(DAG, N);
50953             SDValue I2F = DAG.getBitcast(VT, LHS.getOperand(0).getOperand(0));
50954             SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F);
50955             r = DAG.getBitcast(VT, FCMulC);
50956             return true;
50957           }
50958         }
50959       }
50960     }
50961     return false;
50962   };
50963   SDValue Res;
50964   if (combineConjugation(Res))
50965     return Res;
50966   std::swap(LHS, RHS);
50967   if (combineConjugation(Res))
50968     return Res;
50969   return Res;
50970 }
50971 
50972 //  Try to combine the following nodes:
50973 //  FADD(A, FMA(B, C, 0)) and FADD(A, FMUL(B, C)) to FMA(B, C, A)
50974 static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
50975                                 const X86Subtarget &Subtarget) {
50976   auto AllowContract = [&DAG](const SDNodeFlags &Flags) {
50977     return DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast ||
50978            Flags.hasAllowContract();
50979   };
50980 
50981   auto HasNoSignedZero = [&DAG](const SDNodeFlags &Flags) {
50982     return DAG.getTarget().Options.NoSignedZerosFPMath ||
50983            Flags.hasNoSignedZeros();
50984   };
50985   auto IsVectorAllNegativeZero = [&DAG](SDValue Op) {
50986     APInt AI = APInt(32, 0x80008000, true);
50987     KnownBits Bits = DAG.computeKnownBits(Op);
50988     return Bits.getBitWidth() == 32 && Bits.isConstant() &&
50989            Bits.getConstant() == AI;
50990   };
50991 
50992   if (N->getOpcode() != ISD::FADD || !Subtarget.hasFP16() ||
50993       !AllowContract(N->getFlags()))
50994     return SDValue();
50995 
50996   EVT VT = N->getValueType(0);
50997   if (VT != MVT::v8f16 && VT != MVT::v16f16 && VT != MVT::v32f16)
50998     return SDValue();
50999 
51000   SDValue LHS = N->getOperand(0);
51001   SDValue RHS = N->getOperand(1);
51002   bool IsConj;
51003   SDValue FAddOp1, MulOp0, MulOp1;
51004   auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj, &AllowContract,
51005                        &IsVectorAllNegativeZero,
51006                        &HasNoSignedZero](SDValue N) -> bool {
51007     if (!N.hasOneUse() || N.getOpcode() != ISD::BITCAST)
51008       return false;
51009     SDValue Op0 = N.getOperand(0);
51010     unsigned Opcode = Op0.getOpcode();
51011     if (Op0.hasOneUse() && AllowContract(Op0->getFlags())) {
51012       if ((Opcode == X86ISD::VFMULC || Opcode == X86ISD::VFCMULC)) {
51013         MulOp0 = Op0.getOperand(0);
51014         MulOp1 = Op0.getOperand(1);
51015         IsConj = Opcode == X86ISD::VFCMULC;
51016         return true;
51017       }
51018       if ((Opcode == X86ISD::VFMADDC || Opcode == X86ISD::VFCMADDC) &&
51019           ((ISD::isBuildVectorAllZeros(Op0->getOperand(2).getNode()) &&
51020             HasNoSignedZero(Op0->getFlags())) ||
51021            IsVectorAllNegativeZero(Op0->getOperand(2)))) {
51022         MulOp0 = Op0.getOperand(0);
51023         MulOp1 = Op0.getOperand(1);
51024         IsConj = Opcode == X86ISD::VFCMADDC;
51025         return true;
51026       }
51027     }
51028     return false;
51029   };
51030 
51031   if (GetCFmulFrom(LHS))
51032     FAddOp1 = RHS;
51033   else if (GetCFmulFrom(RHS))
51034     FAddOp1 = LHS;
51035   else
51036     return SDValue();
51037 
51038   MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
51039   FAddOp1 = DAG.getBitcast(CVT, FAddOp1);
51040   unsigned NewOp = IsConj ? X86ISD::VFCMADDC : X86ISD::VFMADDC;
51041   // FIXME: How do we handle when fast math flags of FADD are different from
51042   // CFMUL's?
51043   SDValue CFmul =
51044       DAG.getNode(NewOp, SDLoc(N), CVT, MulOp0, MulOp1, FAddOp1, N->getFlags());
51045   return DAG.getBitcast(VT, CFmul);
51046 }
51047 
51048 /// Do target-specific dag combines on floating-point adds/subs.
51049 static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
51050                                const X86Subtarget &Subtarget) {
51051   if (SDValue HOp = combineToHorizontalAddSub(N, DAG, Subtarget))
51052     return HOp;
51053 
51054   if (SDValue COp = combineFaddCFmul(N, DAG, Subtarget))
51055     return COp;
51056 
51057   return SDValue();
51058 }
51059 
51060 /// Attempt to pre-truncate inputs to arithmetic ops if it will simplify
51061 /// the codegen.
51062 /// e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) )
51063 /// TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove
51064 ///       anything that is guaranteed to be transformed by DAGCombiner.
51065 static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG,
51066                                           const X86Subtarget &Subtarget,
51067                                           const SDLoc &DL) {
51068   assert(N->getOpcode() == ISD::TRUNCATE && "Wrong opcode");
51069   SDValue Src = N->getOperand(0);
51070   unsigned SrcOpcode = Src.getOpcode();
51071   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51072 
51073   EVT VT = N->getValueType(0);
51074   EVT SrcVT = Src.getValueType();
51075 
51076   auto IsFreeTruncation = [VT](SDValue Op) {
51077     unsigned TruncSizeInBits = VT.getScalarSizeInBits();
51078 
51079     // See if this has been extended from a smaller/equal size to
51080     // the truncation size, allowing a truncation to combine with the extend.
51081     unsigned Opcode = Op.getOpcode();
51082     if ((Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND ||
51083          Opcode == ISD::ZERO_EXTEND) &&
51084         Op.getOperand(0).getScalarValueSizeInBits() <= TruncSizeInBits)
51085       return true;
51086 
51087     // See if this is a single use constant which can be constant folded.
51088     // NOTE: We don't peek throught bitcasts here because there is currently
51089     // no support for constant folding truncate+bitcast+vector_of_constants. So
51090     // we'll just send up with a truncate on both operands which will
51091     // get turned back into (truncate (binop)) causing an infinite loop.
51092     return ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
51093   };
51094 
51095   auto TruncateArithmetic = [&](SDValue N0, SDValue N1) {
51096     SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
51097     SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
51098     return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1);
51099   };
51100 
51101   // Don't combine if the operation has other uses.
51102   if (!Src.hasOneUse())
51103     return SDValue();
51104 
51105   // Only support vector truncation for now.
51106   // TODO: i64 scalar math would benefit as well.
51107   if (!VT.isVector())
51108     return SDValue();
51109 
51110   // In most cases its only worth pre-truncating if we're only facing the cost
51111   // of one truncation.
51112   // i.e. if one of the inputs will constant fold or the input is repeated.
51113   switch (SrcOpcode) {
51114   case ISD::MUL:
51115     // X86 is rubbish at scalar and vector i64 multiplies (until AVX512DQ) - its
51116     // better to truncate if we have the chance.
51117     if (SrcVT.getScalarType() == MVT::i64 &&
51118         TLI.isOperationLegal(SrcOpcode, VT) &&
51119         !TLI.isOperationLegal(SrcOpcode, SrcVT))
51120       return TruncateArithmetic(Src.getOperand(0), Src.getOperand(1));
51121     [[fallthrough]];
51122   case ISD::AND:
51123   case ISD::XOR:
51124   case ISD::OR:
51125   case ISD::ADD:
51126   case ISD::SUB: {
51127     SDValue Op0 = Src.getOperand(0);
51128     SDValue Op1 = Src.getOperand(1);
51129     if (TLI.isOperationLegal(SrcOpcode, VT) &&
51130         (Op0 == Op1 || IsFreeTruncation(Op0) || IsFreeTruncation(Op1)))
51131       return TruncateArithmetic(Op0, Op1);
51132     break;
51133   }
51134   }
51135 
51136   return SDValue();
51137 }
51138 
51139 // Try to form a MULHU or MULHS node by looking for
51140 // (trunc (srl (mul ext, ext), 16))
51141 // TODO: This is X86 specific because we want to be able to handle wide types
51142 // before type legalization. But we can only do it if the vector will be
51143 // legalized via widening/splitting. Type legalization can't handle promotion
51144 // of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
51145 // combiner.
51146 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
51147                             SelectionDAG &DAG, const X86Subtarget &Subtarget) {
51148   // First instruction should be a right shift of a multiply.
51149   if (Src.getOpcode() != ISD::SRL ||
51150       Src.getOperand(0).getOpcode() != ISD::MUL)
51151     return SDValue();
51152 
51153   if (!Subtarget.hasSSE2())
51154     return SDValue();
51155 
51156   // Only handle vXi16 types that are at least 128-bits unless they will be
51157   // widened.
51158   if (!VT.isVector() || VT.getVectorElementType() != MVT::i16)
51159     return SDValue();
51160 
51161   // Input type should be at least vXi32.
51162   EVT InVT = Src.getValueType();
51163   if (InVT.getVectorElementType().getSizeInBits() < 32)
51164     return SDValue();
51165 
51166   // Need a shift by 16.
51167   APInt ShiftAmt;
51168   if (!ISD::isConstantSplatVector(Src.getOperand(1).getNode(), ShiftAmt) ||
51169       ShiftAmt != 16)
51170     return SDValue();
51171 
51172   SDValue LHS = Src.getOperand(0).getOperand(0);
51173   SDValue RHS = Src.getOperand(0).getOperand(1);
51174 
51175   // Count leading sign/zero bits on both inputs - if there are enough then
51176   // truncation back to vXi16 will be cheap - either as a pack/shuffle
51177   // sequence or using AVX512 truncations. If the inputs are sext/zext then the
51178   // truncations may actually be free by peeking through to the ext source.
51179   auto IsSext = [&DAG](SDValue V) {
51180     return DAG.ComputeMaxSignificantBits(V) <= 16;
51181   };
51182   auto IsZext = [&DAG](SDValue V) {
51183     return DAG.computeKnownBits(V).countMaxActiveBits() <= 16;
51184   };
51185 
51186   bool IsSigned = IsSext(LHS) && IsSext(RHS);
51187   bool IsUnsigned = IsZext(LHS) && IsZext(RHS);
51188   if (!IsSigned && !IsUnsigned)
51189     return SDValue();
51190 
51191   // Check if both inputs are extensions, which will be removed by truncation.
51192   bool IsTruncateFree = (LHS.getOpcode() == ISD::SIGN_EXTEND ||
51193                          LHS.getOpcode() == ISD::ZERO_EXTEND) &&
51194                         (RHS.getOpcode() == ISD::SIGN_EXTEND ||
51195                          RHS.getOpcode() == ISD::ZERO_EXTEND) &&
51196                         LHS.getOperand(0).getScalarValueSizeInBits() <= 16 &&
51197                         RHS.getOperand(0).getScalarValueSizeInBits() <= 16;
51198 
51199   // For AVX2+ targets, with the upper bits known zero, we can perform MULHU on
51200   // the (bitcasted) inputs directly, and then cheaply pack/truncate the result
51201   // (upper elts will be zero). Don't attempt this with just AVX512F as MULHU
51202   // will have to split anyway.
51203   unsigned InSizeInBits = InVT.getSizeInBits();
51204   if (IsUnsigned && !IsTruncateFree && Subtarget.hasInt256() &&
51205       !(Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.is256BitVector()) &&
51206       (InSizeInBits % 16) == 0) {
51207     EVT BCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
51208                                 InVT.getSizeInBits() / 16);
51209     SDValue Res = DAG.getNode(ISD::MULHU, DL, BCVT, DAG.getBitcast(BCVT, LHS),
51210                               DAG.getBitcast(BCVT, RHS));
51211     return DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getBitcast(InVT, Res));
51212   }
51213 
51214   // Truncate back to source type.
51215   LHS = DAG.getNode(ISD::TRUNCATE, DL, VT, LHS);
51216   RHS = DAG.getNode(ISD::TRUNCATE, DL, VT, RHS);
51217 
51218   unsigned Opc = IsSigned ? ISD::MULHS : ISD::MULHU;
51219   return DAG.getNode(Opc, DL, VT, LHS, RHS);
51220 }
51221 
51222 // Attempt to match PMADDUBSW, which multiplies corresponding unsigned bytes
51223 // from one vector with signed bytes from another vector, adds together
51224 // adjacent pairs of 16-bit products, and saturates the result before
51225 // truncating to 16-bits.
51226 //
51227 // Which looks something like this:
51228 // (i16 (ssat (add (mul (zext (even elts (i8 A))), (sext (even elts (i8 B)))),
51229 //                 (mul (zext (odd elts (i8 A)), (sext (odd elts (i8 B))))))))
51230 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
51231                                const X86Subtarget &Subtarget,
51232                                const SDLoc &DL) {
51233   if (!VT.isVector() || !Subtarget.hasSSSE3())
51234     return SDValue();
51235 
51236   unsigned NumElems = VT.getVectorNumElements();
51237   EVT ScalarVT = VT.getVectorElementType();
51238   if (ScalarVT != MVT::i16 || NumElems < 8 || !isPowerOf2_32(NumElems))
51239     return SDValue();
51240 
51241   SDValue SSatVal = detectSSatPattern(In, VT);
51242   if (!SSatVal || SSatVal.getOpcode() != ISD::ADD)
51243     return SDValue();
51244 
51245   // Ok this is a signed saturation of an ADD. See if this ADD is adding pairs
51246   // of multiplies from even/odd elements.
51247   SDValue N0 = SSatVal.getOperand(0);
51248   SDValue N1 = SSatVal.getOperand(1);
51249 
51250   if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
51251     return SDValue();
51252 
51253   SDValue N00 = N0.getOperand(0);
51254   SDValue N01 = N0.getOperand(1);
51255   SDValue N10 = N1.getOperand(0);
51256   SDValue N11 = N1.getOperand(1);
51257 
51258   // TODO: Handle constant vectors and use knownbits/computenumsignbits?
51259   // Canonicalize zero_extend to LHS.
51260   if (N01.getOpcode() == ISD::ZERO_EXTEND)
51261     std::swap(N00, N01);
51262   if (N11.getOpcode() == ISD::ZERO_EXTEND)
51263     std::swap(N10, N11);
51264 
51265   // Ensure we have a zero_extend and a sign_extend.
51266   if (N00.getOpcode() != ISD::ZERO_EXTEND ||
51267       N01.getOpcode() != ISD::SIGN_EXTEND ||
51268       N10.getOpcode() != ISD::ZERO_EXTEND ||
51269       N11.getOpcode() != ISD::SIGN_EXTEND)
51270     return SDValue();
51271 
51272   // Peek through the extends.
51273   N00 = N00.getOperand(0);
51274   N01 = N01.getOperand(0);
51275   N10 = N10.getOperand(0);
51276   N11 = N11.getOperand(0);
51277 
51278   // Ensure the extend is from vXi8.
51279   if (N00.getValueType().getVectorElementType() != MVT::i8 ||
51280       N01.getValueType().getVectorElementType() != MVT::i8 ||
51281       N10.getValueType().getVectorElementType() != MVT::i8 ||
51282       N11.getValueType().getVectorElementType() != MVT::i8)
51283     return SDValue();
51284 
51285   // All inputs should be build_vectors.
51286   if (N00.getOpcode() != ISD::BUILD_VECTOR ||
51287       N01.getOpcode() != ISD::BUILD_VECTOR ||
51288       N10.getOpcode() != ISD::BUILD_VECTOR ||
51289       N11.getOpcode() != ISD::BUILD_VECTOR)
51290     return SDValue();
51291 
51292   // N00/N10 are zero extended. N01/N11 are sign extended.
51293 
51294   // For each element, we need to ensure we have an odd element from one vector
51295   // multiplied by the odd element of another vector and the even element from
51296   // one of the same vectors being multiplied by the even element from the
51297   // other vector. So we need to make sure for each element i, this operator
51298   // is being performed:
51299   //  A[2 * i] * B[2 * i] + A[2 * i + 1] * B[2 * i + 1]
51300   SDValue ZExtIn, SExtIn;
51301   for (unsigned i = 0; i != NumElems; ++i) {
51302     SDValue N00Elt = N00.getOperand(i);
51303     SDValue N01Elt = N01.getOperand(i);
51304     SDValue N10Elt = N10.getOperand(i);
51305     SDValue N11Elt = N11.getOperand(i);
51306     // TODO: Be more tolerant to undefs.
51307     if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
51308         N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
51309         N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
51310         N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
51311       return SDValue();
51312     auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
51313     auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
51314     auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
51315     auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
51316     if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
51317       return SDValue();
51318     unsigned IdxN00 = ConstN00Elt->getZExtValue();
51319     unsigned IdxN01 = ConstN01Elt->getZExtValue();
51320     unsigned IdxN10 = ConstN10Elt->getZExtValue();
51321     unsigned IdxN11 = ConstN11Elt->getZExtValue();
51322     // Add is commutative so indices can be reordered.
51323     if (IdxN00 > IdxN10) {
51324       std::swap(IdxN00, IdxN10);
51325       std::swap(IdxN01, IdxN11);
51326     }
51327     // N0 indices be the even element. N1 indices must be the next odd element.
51328     if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
51329         IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
51330       return SDValue();
51331     SDValue N00In = N00Elt.getOperand(0);
51332     SDValue N01In = N01Elt.getOperand(0);
51333     SDValue N10In = N10Elt.getOperand(0);
51334     SDValue N11In = N11Elt.getOperand(0);
51335     // First time we find an input capture it.
51336     if (!ZExtIn) {
51337       ZExtIn = N00In;
51338       SExtIn = N01In;
51339     }
51340     if (ZExtIn != N00In || SExtIn != N01In ||
51341         ZExtIn != N10In || SExtIn != N11In)
51342       return SDValue();
51343   }
51344 
51345   auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
51346                          ArrayRef<SDValue> Ops) {
51347     // Shrink by adding truncate nodes and let DAGCombine fold with the
51348     // sources.
51349     EVT InVT = Ops[0].getValueType();
51350     assert(InVT.getScalarType() == MVT::i8 &&
51351            "Unexpected scalar element type");
51352     assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
51353     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
51354                                  InVT.getVectorNumElements() / 2);
51355     return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
51356   };
51357   return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },
51358                           PMADDBuilder);
51359 }
51360 
51361 static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
51362                                const X86Subtarget &Subtarget) {
51363   EVT VT = N->getValueType(0);
51364   SDValue Src = N->getOperand(0);
51365   SDLoc DL(N);
51366 
51367   // Attempt to pre-truncate inputs to arithmetic ops instead.
51368   if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
51369     return V;
51370 
51371   // Try to detect AVG pattern first.
51372   if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL))
51373     return Avg;
51374 
51375   // Try to detect PMADD
51376   if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
51377     return PMAdd;
51378 
51379   // Try to combine truncation with signed/unsigned saturation.
51380   if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
51381     return Val;
51382 
51383   // Try to combine PMULHUW/PMULHW for vXi16.
51384   if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
51385     return V;
51386 
51387   // The bitcast source is a direct mmx result.
51388   // Detect bitcasts between i32 to x86mmx
51389   if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) {
51390     SDValue BCSrc = Src.getOperand(0);
51391     if (BCSrc.getValueType() == MVT::x86mmx)
51392       return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
51393   }
51394 
51395   return SDValue();
51396 }
51397 
51398 static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG,
51399                              TargetLowering::DAGCombinerInfo &DCI) {
51400   EVT VT = N->getValueType(0);
51401   SDValue In = N->getOperand(0);
51402   SDLoc DL(N);
51403 
51404   if (SDValue SSatVal = detectSSatPattern(In, VT))
51405     return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal);
51406   if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL))
51407     return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);
51408 
51409   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51410   APInt DemandedMask(APInt::getAllOnes(VT.getScalarSizeInBits()));
51411   if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
51412     return SDValue(N, 0);
51413 
51414   return SDValue();
51415 }
51416 
51417 /// Returns the negated value if the node \p N flips sign of FP value.
51418 ///
51419 /// FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000)
51420 /// or FSUB(0, x)
51421 /// AVX512F does not have FXOR, so FNEG is lowered as
51422 /// (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))).
51423 /// In this case we go though all bitcasts.
51424 /// This also recognizes splat of a negated value and returns the splat of that
51425 /// value.
51426 static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) {
51427   if (N->getOpcode() == ISD::FNEG)
51428     return N->getOperand(0);
51429 
51430   // Don't recurse exponentially.
51431   if (Depth > SelectionDAG::MaxRecursionDepth)
51432     return SDValue();
51433 
51434   unsigned ScalarSize = N->getValueType(0).getScalarSizeInBits();
51435 
51436   SDValue Op = peekThroughBitcasts(SDValue(N, 0));
51437   EVT VT = Op->getValueType(0);
51438 
51439   // Make sure the element size doesn't change.
51440   if (VT.getScalarSizeInBits() != ScalarSize)
51441     return SDValue();
51442 
51443   unsigned Opc = Op.getOpcode();
51444   switch (Opc) {
51445   case ISD::VECTOR_SHUFFLE: {
51446     // For a VECTOR_SHUFFLE(VEC1, VEC2), if the VEC2 is undef, then the negate
51447     // of this is VECTOR_SHUFFLE(-VEC1, UNDEF).  The mask can be anything here.
51448     if (!Op.getOperand(1).isUndef())
51449       return SDValue();
51450     if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1))
51451       if (NegOp0.getValueType() == VT) // FIXME: Can we do better?
51452         return DAG.getVectorShuffle(VT, SDLoc(Op), NegOp0, DAG.getUNDEF(VT),
51453                                     cast<ShuffleVectorSDNode>(Op)->getMask());
51454     break;
51455   }
51456   case ISD::INSERT_VECTOR_ELT: {
51457     // Negate of INSERT_VECTOR_ELT(UNDEF, V, INDEX) is INSERT_VECTOR_ELT(UNDEF,
51458     // -V, INDEX).
51459     SDValue InsVector = Op.getOperand(0);
51460     SDValue InsVal = Op.getOperand(1);
51461     if (!InsVector.isUndef())
51462       return SDValue();
51463     if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1))
51464       if (NegInsVal.getValueType() == VT.getVectorElementType()) // FIXME
51465         return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector,
51466                            NegInsVal, Op.getOperand(2));
51467     break;
51468   }
51469   case ISD::FSUB:
51470   case ISD::XOR:
51471   case X86ISD::FXOR: {
51472     SDValue Op1 = Op.getOperand(1);
51473     SDValue Op0 = Op.getOperand(0);
51474 
51475     // For XOR and FXOR, we want to check if constant
51476     // bits of Op1 are sign bit masks. For FSUB, we
51477     // have to check if constant bits of Op0 are sign
51478     // bit masks and hence we swap the operands.
51479     if (Opc == ISD::FSUB)
51480       std::swap(Op0, Op1);
51481 
51482     APInt UndefElts;
51483     SmallVector<APInt, 16> EltBits;
51484     // Extract constant bits and see if they are all
51485     // sign bit masks. Ignore the undef elements.
51486     if (getTargetConstantBitsFromNode(Op1, ScalarSize, UndefElts, EltBits,
51487                                       /* AllowWholeUndefs */ true,
51488                                       /* AllowPartialUndefs */ false)) {
51489       for (unsigned I = 0, E = EltBits.size(); I < E; I++)
51490         if (!UndefElts[I] && !EltBits[I].isSignMask())
51491           return SDValue();
51492 
51493       // Only allow bitcast from correctly-sized constant.
51494       Op0 = peekThroughBitcasts(Op0);
51495       if (Op0.getScalarValueSizeInBits() == ScalarSize)
51496         return Op0;
51497     }
51498     break;
51499   } // case
51500   } // switch
51501 
51502   return SDValue();
51503 }
51504 
51505 static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
51506                                 bool NegRes) {
51507   if (NegMul) {
51508     switch (Opcode) {
51509     default: llvm_unreachable("Unexpected opcode");
51510     case ISD::FMA:              Opcode = X86ISD::FNMADD;        break;
51511     case ISD::STRICT_FMA:       Opcode = X86ISD::STRICT_FNMADD; break;
51512     case X86ISD::FMADD_RND:     Opcode = X86ISD::FNMADD_RND;    break;
51513     case X86ISD::FMSUB:         Opcode = X86ISD::FNMSUB;        break;
51514     case X86ISD::STRICT_FMSUB:  Opcode = X86ISD::STRICT_FNMSUB; break;
51515     case X86ISD::FMSUB_RND:     Opcode = X86ISD::FNMSUB_RND;    break;
51516     case X86ISD::FNMADD:        Opcode = ISD::FMA;              break;
51517     case X86ISD::STRICT_FNMADD: Opcode = ISD::STRICT_FMA;       break;
51518     case X86ISD::FNMADD_RND:    Opcode = X86ISD::FMADD_RND;     break;
51519     case X86ISD::FNMSUB:        Opcode = X86ISD::FMSUB;         break;
51520     case X86ISD::STRICT_FNMSUB: Opcode = X86ISD::STRICT_FMSUB;  break;
51521     case X86ISD::FNMSUB_RND:    Opcode = X86ISD::FMSUB_RND;     break;
51522     }
51523   }
51524 
51525   if (NegAcc) {
51526     switch (Opcode) {
51527     default: llvm_unreachable("Unexpected opcode");
51528     case ISD::FMA:              Opcode = X86ISD::FMSUB;         break;
51529     case ISD::STRICT_FMA:       Opcode = X86ISD::STRICT_FMSUB;  break;
51530     case X86ISD::FMADD_RND:     Opcode = X86ISD::FMSUB_RND;     break;
51531     case X86ISD::FMSUB:         Opcode = ISD::FMA;              break;
51532     case X86ISD::STRICT_FMSUB:  Opcode = ISD::STRICT_FMA;       break;
51533     case X86ISD::FMSUB_RND:     Opcode = X86ISD::FMADD_RND;     break;
51534     case X86ISD::FNMADD:        Opcode = X86ISD::FNMSUB;        break;
51535     case X86ISD::STRICT_FNMADD: Opcode = X86ISD::STRICT_FNMSUB; break;
51536     case X86ISD::FNMADD_RND:    Opcode = X86ISD::FNMSUB_RND;    break;
51537     case X86ISD::FNMSUB:        Opcode = X86ISD::FNMADD;        break;
51538     case X86ISD::STRICT_FNMSUB: Opcode = X86ISD::STRICT_FNMADD; break;
51539     case X86ISD::FNMSUB_RND:    Opcode = X86ISD::FNMADD_RND;    break;
51540     case X86ISD::FMADDSUB:      Opcode = X86ISD::FMSUBADD;      break;
51541     case X86ISD::FMADDSUB_RND:  Opcode = X86ISD::FMSUBADD_RND;  break;
51542     case X86ISD::FMSUBADD:      Opcode = X86ISD::FMADDSUB;      break;
51543     case X86ISD::FMSUBADD_RND:  Opcode = X86ISD::FMADDSUB_RND;  break;
51544     }
51545   }
51546 
51547   if (NegRes) {
51548     switch (Opcode) {
51549     // For accuracy reason, we never combine fneg and fma under strict FP.
51550     default: llvm_unreachable("Unexpected opcode");
51551     case ISD::FMA:             Opcode = X86ISD::FNMSUB;       break;
51552     case X86ISD::FMADD_RND:    Opcode = X86ISD::FNMSUB_RND;   break;
51553     case X86ISD::FMSUB:        Opcode = X86ISD::FNMADD;       break;
51554     case X86ISD::FMSUB_RND:    Opcode = X86ISD::FNMADD_RND;   break;
51555     case X86ISD::FNMADD:       Opcode = X86ISD::FMSUB;        break;
51556     case X86ISD::FNMADD_RND:   Opcode = X86ISD::FMSUB_RND;    break;
51557     case X86ISD::FNMSUB:       Opcode = ISD::FMA;             break;
51558     case X86ISD::FNMSUB_RND:   Opcode = X86ISD::FMADD_RND;    break;
51559     }
51560   }
51561 
51562   return Opcode;
51563 }
51564 
51565 /// Do target-specific dag combines on floating point negations.
51566 static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
51567                            TargetLowering::DAGCombinerInfo &DCI,
51568                            const X86Subtarget &Subtarget) {
51569   EVT OrigVT = N->getValueType(0);
51570   SDValue Arg = isFNEG(DAG, N);
51571   if (!Arg)
51572     return SDValue();
51573 
51574   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51575   EVT VT = Arg.getValueType();
51576   EVT SVT = VT.getScalarType();
51577   SDLoc DL(N);
51578 
51579   // Let legalize expand this if it isn't a legal type yet.
51580   if (!TLI.isTypeLegal(VT))
51581     return SDValue();
51582 
51583   // If we're negating a FMUL node on a target with FMA, then we can avoid the
51584   // use of a constant by performing (-0 - A*B) instead.
51585   // FIXME: Check rounding control flags as well once it becomes available.
51586   if (Arg.getOpcode() == ISD::FMUL && (SVT == MVT::f32 || SVT == MVT::f64) &&
51587       Arg->getFlags().hasNoSignedZeros() && Subtarget.hasAnyFMA()) {
51588     SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
51589     SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
51590                                   Arg.getOperand(1), Zero);
51591     return DAG.getBitcast(OrigVT, NewNode);
51592   }
51593 
51594   bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
51595   bool LegalOperations = !DCI.isBeforeLegalizeOps();
51596   if (SDValue NegArg =
51597           TLI.getNegatedExpression(Arg, DAG, LegalOperations, CodeSize))
51598     return DAG.getBitcast(OrigVT, NegArg);
51599 
51600   return SDValue();
51601 }
51602 
51603 SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
51604                                                 bool LegalOperations,
51605                                                 bool ForCodeSize,
51606                                                 NegatibleCost &Cost,
51607                                                 unsigned Depth) const {
51608   // fneg patterns are removable even if they have multiple uses.
51609   if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) {
51610     Cost = NegatibleCost::Cheaper;
51611     return DAG.getBitcast(Op.getValueType(), Arg);
51612   }
51613 
51614   EVT VT = Op.getValueType();
51615   EVT SVT = VT.getScalarType();
51616   unsigned Opc = Op.getOpcode();
51617   SDNodeFlags Flags = Op.getNode()->getFlags();
51618   switch (Opc) {
51619   case ISD::FMA:
51620   case X86ISD::FMSUB:
51621   case X86ISD::FNMADD:
51622   case X86ISD::FNMSUB:
51623   case X86ISD::FMADD_RND:
51624   case X86ISD::FMSUB_RND:
51625   case X86ISD::FNMADD_RND:
51626   case X86ISD::FNMSUB_RND: {
51627     if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) ||
51628         !(SVT == MVT::f32 || SVT == MVT::f64) ||
51629         !isOperationLegal(ISD::FMA, VT))
51630       break;
51631 
51632     // Don't fold (fneg (fma (fneg x), y, (fneg z))) to (fma x, y, z)
51633     // if it may have signed zeros.
51634     if (!Flags.hasNoSignedZeros())
51635       break;
51636 
51637     // This is always negatible for free but we might be able to remove some
51638     // extra operand negations as well.
51639     SmallVector<SDValue, 4> NewOps(Op.getNumOperands(), SDValue());
51640     for (int i = 0; i != 3; ++i)
51641       NewOps[i] = getCheaperNegatedExpression(
51642           Op.getOperand(i), DAG, LegalOperations, ForCodeSize, Depth + 1);
51643 
51644     bool NegA = !!NewOps[0];
51645     bool NegB = !!NewOps[1];
51646     bool NegC = !!NewOps[2];
51647     unsigned NewOpc = negateFMAOpcode(Opc, NegA != NegB, NegC, true);
51648 
51649     Cost = (NegA || NegB || NegC) ? NegatibleCost::Cheaper
51650                                   : NegatibleCost::Neutral;
51651 
51652     // Fill in the non-negated ops with the original values.
51653     for (int i = 0, e = Op.getNumOperands(); i != e; ++i)
51654       if (!NewOps[i])
51655         NewOps[i] = Op.getOperand(i);
51656     return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps);
51657   }
51658   case X86ISD::FRCP:
51659     if (SDValue NegOp0 =
51660             getNegatedExpression(Op.getOperand(0), DAG, LegalOperations,
51661                                  ForCodeSize, Cost, Depth + 1))
51662       return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0);
51663     break;
51664   }
51665 
51666   return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
51667                                               ForCodeSize, Cost, Depth);
51668 }
51669 
51670 static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
51671                                  const X86Subtarget &Subtarget) {
51672   MVT VT = N->getSimpleValueType(0);
51673   // If we have integer vector types available, use the integer opcodes.
51674   if (!VT.isVector() || !Subtarget.hasSSE2())
51675     return SDValue();
51676 
51677   SDLoc dl(N);
51678 
51679   unsigned IntBits = VT.getScalarSizeInBits();
51680   MVT IntSVT = MVT::getIntegerVT(IntBits);
51681   MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits);
51682 
51683   SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
51684   SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
51685   unsigned IntOpcode;
51686   switch (N->getOpcode()) {
51687   default: llvm_unreachable("Unexpected FP logic op");
51688   case X86ISD::FOR:   IntOpcode = ISD::OR; break;
51689   case X86ISD::FXOR:  IntOpcode = ISD::XOR; break;
51690   case X86ISD::FAND:  IntOpcode = ISD::AND; break;
51691   case X86ISD::FANDN: IntOpcode = X86ISD::ANDNP; break;
51692   }
51693   SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
51694   return DAG.getBitcast(VT, IntOp);
51695 }
51696 
51697 
51698 /// Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
51699 static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
51700   if (N->getOpcode() != ISD::XOR)
51701     return SDValue();
51702 
51703   SDValue LHS = N->getOperand(0);
51704   if (!isOneConstant(N->getOperand(1)) || LHS->getOpcode() != X86ISD::SETCC)
51705     return SDValue();
51706 
51707   X86::CondCode NewCC = X86::GetOppositeBranchCondition(
51708       X86::CondCode(LHS->getConstantOperandVal(0)));
51709   SDLoc DL(N);
51710   return getSETCC(NewCC, LHS->getOperand(1), DL, DAG);
51711 }
51712 
51713 static SDValue combineXorSubCTLZ(SDNode *N, SelectionDAG &DAG,
51714                                  const X86Subtarget &Subtarget) {
51715   assert((N->getOpcode() == ISD::XOR || N->getOpcode() == ISD::SUB) &&
51716          "Invalid opcode for combing with CTLZ");
51717   if (Subtarget.hasFastLZCNT())
51718     return SDValue();
51719 
51720   EVT VT = N->getValueType(0);
51721   if (VT != MVT::i8 && VT != MVT::i16 && VT != MVT::i32 &&
51722       (VT != MVT::i64 || !Subtarget.is64Bit()))
51723     return SDValue();
51724 
51725   SDValue N0 = N->getOperand(0);
51726   SDValue N1 = N->getOperand(1);
51727 
51728   if (N0.getOpcode() != ISD::CTLZ_ZERO_UNDEF &&
51729       N1.getOpcode() != ISD::CTLZ_ZERO_UNDEF)
51730     return SDValue();
51731 
51732   SDValue OpCTLZ;
51733   SDValue OpSizeTM1;
51734 
51735   if (N1.getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
51736     OpCTLZ = N1;
51737     OpSizeTM1 = N0;
51738   } else if (N->getOpcode() == ISD::SUB) {
51739     return SDValue();
51740   } else {
51741     OpCTLZ = N0;
51742     OpSizeTM1 = N1;
51743   }
51744 
51745   if (!OpCTLZ.hasOneUse())
51746     return SDValue();
51747   auto *C = dyn_cast<ConstantSDNode>(OpSizeTM1);
51748   if (!C)
51749     return SDValue();
51750 
51751   if (C->getZExtValue() != uint64_t(OpCTLZ.getValueSizeInBits() - 1))
51752     return SDValue();
51753   SDLoc DL(N);
51754   EVT OpVT = VT;
51755   SDValue Op = OpCTLZ.getOperand(0);
51756   if (VT == MVT::i8) {
51757     // Zero extend to i32 since there is not an i8 bsr.
51758     OpVT = MVT::i32;
51759     Op = DAG.getNode(ISD::ZERO_EXTEND, DL, OpVT, Op);
51760   }
51761 
51762   SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
51763   Op = DAG.getNode(X86ISD::BSR, DL, VTs, Op);
51764   if (VT == MVT::i8)
51765     Op = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Op);
51766 
51767   return Op;
51768 }
51769 
51770 static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
51771                           TargetLowering::DAGCombinerInfo &DCI,
51772                           const X86Subtarget &Subtarget) {
51773   SDValue N0 = N->getOperand(0);
51774   SDValue N1 = N->getOperand(1);
51775   EVT VT = N->getValueType(0);
51776 
51777   // If this is SSE1 only convert to FXOR to avoid scalarization.
51778   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
51779     return DAG.getBitcast(MVT::v4i32,
51780                           DAG.getNode(X86ISD::FXOR, SDLoc(N), MVT::v4f32,
51781                                       DAG.getBitcast(MVT::v4f32, N0),
51782                                       DAG.getBitcast(MVT::v4f32, N1)));
51783   }
51784 
51785   if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
51786     return Cmp;
51787 
51788   if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
51789     return R;
51790 
51791   if (SDValue R = combineBitOpWithShift(N, DAG))
51792     return R;
51793 
51794   if (SDValue R = combineBitOpWithPACK(N, DAG))
51795     return R;
51796 
51797   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
51798     return FPLogic;
51799 
51800   if (SDValue R = combineXorSubCTLZ(N, DAG, Subtarget))
51801     return R;
51802 
51803   if (DCI.isBeforeLegalizeOps())
51804     return SDValue();
51805 
51806   if (SDValue SetCC = foldXor1SetCC(N, DAG))
51807     return SetCC;
51808 
51809   if (SDValue R = combineOrXorWithSETCC(N, N0, N1, DAG))
51810     return R;
51811 
51812   if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
51813     return RV;
51814 
51815   // Fold not(iX bitcast(vXi1)) -> (iX bitcast(not(vec))) for legal boolvecs.
51816   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51817   if (llvm::isAllOnesConstant(N1) && N0.getOpcode() == ISD::BITCAST &&
51818       N0.getOperand(0).getValueType().isVector() &&
51819       N0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
51820       TLI.isTypeLegal(N0.getOperand(0).getValueType()) && N0.hasOneUse()) {
51821     return DAG.getBitcast(VT, DAG.getNOT(SDLoc(N), N0.getOperand(0),
51822                                          N0.getOperand(0).getValueType()));
51823   }
51824 
51825   // Handle AVX512 mask widening.
51826   // Fold not(insert_subvector(undef,sub)) -> insert_subvector(undef,not(sub))
51827   if (ISD::isBuildVectorAllOnes(N1.getNode()) && VT.isVector() &&
51828       VT.getVectorElementType() == MVT::i1 &&
51829       N0.getOpcode() == ISD::INSERT_SUBVECTOR && N0.getOperand(0).isUndef() &&
51830       TLI.isTypeLegal(N0.getOperand(1).getValueType())) {
51831     return DAG.getNode(
51832         ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0),
51833         DAG.getNOT(SDLoc(N), N0.getOperand(1), N0.getOperand(1).getValueType()),
51834         N0.getOperand(2));
51835   }
51836 
51837   // Fold xor(zext(xor(x,c1)),c2) -> xor(zext(x),xor(zext(c1),c2))
51838   // Fold xor(truncate(xor(x,c1)),c2) -> xor(truncate(x),xor(truncate(c1),c2))
51839   // TODO: Under what circumstances could this be performed in DAGCombine?
51840   if ((N0.getOpcode() == ISD::TRUNCATE || N0.getOpcode() == ISD::ZERO_EXTEND) &&
51841       N0.getOperand(0).getOpcode() == N->getOpcode()) {
51842     SDValue TruncExtSrc = N0.getOperand(0);
51843     auto *N1C = dyn_cast<ConstantSDNode>(N1);
51844     auto *N001C = dyn_cast<ConstantSDNode>(TruncExtSrc.getOperand(1));
51845     if (N1C && !N1C->isOpaque() && N001C && !N001C->isOpaque()) {
51846       SDLoc DL(N);
51847       SDValue LHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(0), DL, VT);
51848       SDValue RHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(1), DL, VT);
51849       return DAG.getNode(ISD::XOR, DL, VT, LHS,
51850                          DAG.getNode(ISD::XOR, DL, VT, RHS, N1));
51851     }
51852   }
51853 
51854   if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
51855     return R;
51856 
51857   return combineFneg(N, DAG, DCI, Subtarget);
51858 }
51859 
51860 static SDValue combineBITREVERSE(SDNode *N, SelectionDAG &DAG,
51861                                  TargetLowering::DAGCombinerInfo &DCI,
51862                                  const X86Subtarget &Subtarget) {
51863   SDValue N0 = N->getOperand(0);
51864   EVT VT = N->getValueType(0);
51865 
51866   // Convert a (iX bitreverse(bitcast(vXi1 X))) -> (iX bitcast(shuffle(X)))
51867   if (VT.isInteger() && N0.getOpcode() == ISD::BITCAST && N0.hasOneUse()) {
51868     SDValue Src = N0.getOperand(0);
51869     EVT SrcVT = Src.getValueType();
51870     if (SrcVT.isVector() && SrcVT.getScalarType() == MVT::i1 &&
51871         (DCI.isBeforeLegalize() ||
51872          DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) &&
51873         Subtarget.hasSSSE3()) {
51874       unsigned NumElts = SrcVT.getVectorNumElements();
51875       SmallVector<int, 32> ReverseMask(NumElts);
51876       for (unsigned I = 0; I != NumElts; ++I)
51877         ReverseMask[I] = (NumElts - 1) - I;
51878       SDValue Rev =
51879           DAG.getVectorShuffle(SrcVT, SDLoc(N), Src, Src, ReverseMask);
51880       return DAG.getBitcast(VT, Rev);
51881     }
51882   }
51883 
51884   return SDValue();
51885 }
51886 
51887 static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG,
51888                             TargetLowering::DAGCombinerInfo &DCI,
51889                             const X86Subtarget &Subtarget) {
51890   EVT VT = N->getValueType(0);
51891   unsigned NumBits = VT.getSizeInBits();
51892 
51893   // TODO - Constant Folding.
51894 
51895   // Simplify the inputs.
51896   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51897   APInt DemandedMask(APInt::getAllOnes(NumBits));
51898   if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
51899     return SDValue(N, 0);
51900 
51901   return SDValue();
51902 }
51903 
51904 static bool isNullFPScalarOrVectorConst(SDValue V) {
51905   return isNullFPConstant(V) || ISD::isBuildVectorAllZeros(V.getNode());
51906 }
51907 
51908 /// If a value is a scalar FP zero or a vector FP zero (potentially including
51909 /// undefined elements), return a zero constant that may be used to fold away
51910 /// that value. In the case of a vector, the returned constant will not contain
51911 /// undefined elements even if the input parameter does. This makes it suitable
51912 /// to be used as a replacement operand with operations (eg, bitwise-and) where
51913 /// an undef should not propagate.
51914 static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
51915                                         const X86Subtarget &Subtarget) {
51916   if (!isNullFPScalarOrVectorConst(V))
51917     return SDValue();
51918 
51919   if (V.getValueType().isVector())
51920     return getZeroVector(V.getSimpleValueType(), Subtarget, DAG, SDLoc(V));
51921 
51922   return V;
51923 }
51924 
51925 static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG,
51926                                       const X86Subtarget &Subtarget) {
51927   SDValue N0 = N->getOperand(0);
51928   SDValue N1 = N->getOperand(1);
51929   EVT VT = N->getValueType(0);
51930   SDLoc DL(N);
51931 
51932   // Vector types are handled in combineANDXORWithAllOnesIntoANDNP().
51933   if (!((VT == MVT::f32 && Subtarget.hasSSE1()) ||
51934         (VT == MVT::f64 && Subtarget.hasSSE2()) ||
51935         (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2())))
51936     return SDValue();
51937 
51938   auto isAllOnesConstantFP = [](SDValue V) {
51939     if (V.getSimpleValueType().isVector())
51940       return ISD::isBuildVectorAllOnes(V.getNode());
51941     auto *C = dyn_cast<ConstantFPSDNode>(V);
51942     return C && C->getConstantFPValue()->isAllOnesValue();
51943   };
51944 
51945   // fand (fxor X, -1), Y --> fandn X, Y
51946   if (N0.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N0.getOperand(1)))
51947     return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1);
51948 
51949   // fand X, (fxor Y, -1) --> fandn Y, X
51950   if (N1.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N1.getOperand(1)))
51951     return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0);
51952 
51953   return SDValue();
51954 }
51955 
51956 /// Do target-specific dag combines on X86ISD::FAND nodes.
51957 static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG,
51958                            const X86Subtarget &Subtarget) {
51959   // FAND(0.0, x) -> 0.0
51960   if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget))
51961     return V;
51962 
51963   // FAND(x, 0.0) -> 0.0
51964   if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
51965     return V;
51966 
51967   if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget))
51968     return V;
51969 
51970   return lowerX86FPLogicOp(N, DAG, Subtarget);
51971 }
51972 
51973 /// Do target-specific dag combines on X86ISD::FANDN nodes.
51974 static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG,
51975                             const X86Subtarget &Subtarget) {
51976   // FANDN(0.0, x) -> x
51977   if (isNullFPScalarOrVectorConst(N->getOperand(0)))
51978     return N->getOperand(1);
51979 
51980   // FANDN(x, 0.0) -> 0.0
51981   if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
51982     return V;
51983 
51984   return lowerX86FPLogicOp(N, DAG, Subtarget);
51985 }
51986 
51987 /// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
51988 static SDValue combineFOr(SDNode *N, SelectionDAG &DAG,
51989                           TargetLowering::DAGCombinerInfo &DCI,
51990                           const X86Subtarget &Subtarget) {
51991   assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
51992 
51993   // F[X]OR(0.0, x) -> x
51994   if (isNullFPScalarOrVectorConst(N->getOperand(0)))
51995     return N->getOperand(1);
51996 
51997   // F[X]OR(x, 0.0) -> x
51998   if (isNullFPScalarOrVectorConst(N->getOperand(1)))
51999     return N->getOperand(0);
52000 
52001   if (SDValue NewVal = combineFneg(N, DAG, DCI, Subtarget))
52002     return NewVal;
52003 
52004   return lowerX86FPLogicOp(N, DAG, Subtarget);
52005 }
52006 
52007 /// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
52008 static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
52009   assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
52010 
52011   // FMIN/FMAX are commutative if no NaNs and no negative zeros are allowed.
52012   if (!DAG.getTarget().Options.NoNaNsFPMath ||
52013       !DAG.getTarget().Options.NoSignedZerosFPMath)
52014     return SDValue();
52015 
52016   // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes
52017   // into FMINC and FMAXC, which are Commutative operations.
52018   unsigned NewOp = 0;
52019   switch (N->getOpcode()) {
52020     default: llvm_unreachable("unknown opcode");
52021     case X86ISD::FMIN:  NewOp = X86ISD::FMINC; break;
52022     case X86ISD::FMAX:  NewOp = X86ISD::FMAXC; break;
52023   }
52024 
52025   return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
52026                      N->getOperand(0), N->getOperand(1));
52027 }
52028 
52029 static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
52030                                      const X86Subtarget &Subtarget) {
52031   EVT VT = N->getValueType(0);
52032   if (Subtarget.useSoftFloat() || isSoftF16(VT, Subtarget))
52033     return SDValue();
52034 
52035   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52036 
52037   if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
52038         (Subtarget.hasSSE2() && VT == MVT::f64) ||
52039         (Subtarget.hasFP16() && VT == MVT::f16) ||
52040         (VT.isVector() && TLI.isTypeLegal(VT))))
52041     return SDValue();
52042 
52043   SDValue Op0 = N->getOperand(0);
52044   SDValue Op1 = N->getOperand(1);
52045   SDLoc DL(N);
52046   auto MinMaxOp = N->getOpcode() == ISD::FMAXNUM ? X86ISD::FMAX : X86ISD::FMIN;
52047 
52048   // If we don't have to respect NaN inputs, this is a direct translation to x86
52049   // min/max instructions.
52050   if (DAG.getTarget().Options.NoNaNsFPMath || N->getFlags().hasNoNaNs())
52051     return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
52052 
52053   // If one of the operands is known non-NaN use the native min/max instructions
52054   // with the non-NaN input as second operand.
52055   if (DAG.isKnownNeverNaN(Op1))
52056     return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
52057   if (DAG.isKnownNeverNaN(Op0))
52058     return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags());
52059 
52060   // If we have to respect NaN inputs, this takes at least 3 instructions.
52061   // Favor a library call when operating on a scalar and minimizing code size.
52062   if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize())
52063     return SDValue();
52064 
52065   EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
52066                                          VT);
52067 
52068   // There are 4 possibilities involving NaN inputs, and these are the required
52069   // outputs:
52070   //                   Op1
52071   //               Num     NaN
52072   //            ----------------
52073   //       Num  |  Max  |  Op0 |
52074   // Op0        ----------------
52075   //       NaN  |  Op1  |  NaN |
52076   //            ----------------
52077   //
52078   // The SSE FP max/min instructions were not designed for this case, but rather
52079   // to implement:
52080   //   Min = Op1 < Op0 ? Op1 : Op0
52081   //   Max = Op1 > Op0 ? Op1 : Op0
52082   //
52083   // So they always return Op0 if either input is a NaN. However, we can still
52084   // use those instructions for fmaxnum by selecting away a NaN input.
52085 
52086   // If either operand is NaN, the 2nd source operand (Op0) is passed through.
52087   SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
52088   SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
52089 
52090   // If Op0 is a NaN, select Op1. Otherwise, select the max. If both operands
52091   // are NaN, the NaN value of Op1 is the result.
52092   return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax);
52093 }
52094 
52095 static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
52096                                    TargetLowering::DAGCombinerInfo &DCI) {
52097   EVT VT = N->getValueType(0);
52098   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52099 
52100   APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
52101   if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
52102     return SDValue(N, 0);
52103 
52104   // Convert a full vector load into vzload when not all bits are needed.
52105   SDValue In = N->getOperand(0);
52106   MVT InVT = In.getSimpleValueType();
52107   if (VT.getVectorNumElements() < InVT.getVectorNumElements() &&
52108       ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) {
52109     assert(InVT.is128BitVector() && "Expected 128-bit input vector");
52110     LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(0));
52111     unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements();
52112     MVT MemVT = MVT::getIntegerVT(NumBits);
52113     MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
52114     if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
52115       SDLoc dl(N);
52116       SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
52117                                     DAG.getBitcast(InVT, VZLoad));
52118       DCI.CombineTo(N, Convert);
52119       DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
52120       DCI.recursivelyDeleteUnusedNodes(LN);
52121       return SDValue(N, 0);
52122     }
52123   }
52124 
52125   return SDValue();
52126 }
52127 
52128 static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
52129                                      TargetLowering::DAGCombinerInfo &DCI) {
52130   bool IsStrict = N->isTargetStrictFPOpcode();
52131   EVT VT = N->getValueType(0);
52132 
52133   // Convert a full vector load into vzload when not all bits are needed.
52134   SDValue In = N->getOperand(IsStrict ? 1 : 0);
52135   MVT InVT = In.getSimpleValueType();
52136   if (VT.getVectorNumElements() < InVT.getVectorNumElements() &&
52137       ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) {
52138     assert(InVT.is128BitVector() && "Expected 128-bit input vector");
52139     LoadSDNode *LN = cast<LoadSDNode>(In);
52140     unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements();
52141     MVT MemVT = MVT::getFloatingPointVT(NumBits);
52142     MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
52143     if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
52144       SDLoc dl(N);
52145       if (IsStrict) {
52146         SDValue Convert =
52147             DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},
52148                         {N->getOperand(0), DAG.getBitcast(InVT, VZLoad)});
52149         DCI.CombineTo(N, Convert, Convert.getValue(1));
52150       } else {
52151         SDValue Convert =
52152             DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad));
52153         DCI.CombineTo(N, Convert);
52154       }
52155       DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
52156       DCI.recursivelyDeleteUnusedNodes(LN);
52157       return SDValue(N, 0);
52158     }
52159   }
52160 
52161   return SDValue();
52162 }
52163 
52164 /// Do target-specific dag combines on X86ISD::ANDNP nodes.
52165 static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
52166                             TargetLowering::DAGCombinerInfo &DCI,
52167                             const X86Subtarget &Subtarget) {
52168   SDValue N0 = N->getOperand(0);
52169   SDValue N1 = N->getOperand(1);
52170   MVT VT = N->getSimpleValueType(0);
52171   int NumElts = VT.getVectorNumElements();
52172   unsigned EltSizeInBits = VT.getScalarSizeInBits();
52173   SDLoc DL(N);
52174 
52175   // ANDNP(undef, x) -> 0
52176   // ANDNP(x, undef) -> 0
52177   if (N0.isUndef() || N1.isUndef())
52178     return DAG.getConstant(0, DL, VT);
52179 
52180   // ANDNP(0, x) -> x
52181   if (ISD::isBuildVectorAllZeros(N0.getNode()))
52182     return N1;
52183 
52184   // ANDNP(x, 0) -> 0
52185   if (ISD::isBuildVectorAllZeros(N1.getNode()))
52186     return DAG.getConstant(0, DL, VT);
52187 
52188   // ANDNP(x, -1) -> NOT(x) -> XOR(x, -1)
52189   if (ISD::isBuildVectorAllOnes(N1.getNode()))
52190     return DAG.getNOT(DL, N0, VT);
52191 
52192   // Turn ANDNP back to AND if input is inverted.
52193   if (SDValue Not = IsNOT(N0, DAG))
52194     return DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, Not), N1);
52195 
52196   // Fold for better commutatvity:
52197   // ANDNP(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)).
52198   if (N1->hasOneUse())
52199     if (SDValue Not = IsNOT(N1, DAG))
52200       return DAG.getNOT(
52201           DL, DAG.getNode(ISD::OR, DL, VT, N0, DAG.getBitcast(VT, Not)), VT);
52202 
52203   // Constant Folding
52204   APInt Undefs0, Undefs1;
52205   SmallVector<APInt> EltBits0, EltBits1;
52206   if (getTargetConstantBitsFromNode(N0, EltSizeInBits, Undefs0, EltBits0)) {
52207     if (getTargetConstantBitsFromNode(N1, EltSizeInBits, Undefs1, EltBits1)) {
52208       SmallVector<APInt> ResultBits;
52209       for (int I = 0; I != NumElts; ++I)
52210         ResultBits.push_back(~EltBits0[I] & EltBits1[I]);
52211       return getConstVector(ResultBits, VT, DAG, DL);
52212     }
52213 
52214     // Constant fold NOT(N0) to allow us to use AND.
52215     // Ensure this is only performed if we can confirm that the bitcasted source
52216     // has oneuse to prevent an infinite loop with canonicalizeBitSelect.
52217     if (N0->hasOneUse()) {
52218       SDValue BC0 = peekThroughOneUseBitcasts(N0);
52219       if (BC0.getOpcode() != ISD::BITCAST) {
52220         for (APInt &Elt : EltBits0)
52221           Elt = ~Elt;
52222         SDValue Not = getConstVector(EltBits0, VT, DAG, DL);
52223         return DAG.getNode(ISD::AND, DL, VT, Not, N1);
52224       }
52225     }
52226   }
52227 
52228   // Attempt to recursively combine a bitmask ANDNP with shuffles.
52229   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
52230     SDValue Op(N, 0);
52231     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
52232       return Res;
52233 
52234     // If either operand is a constant mask, then only the elements that aren't
52235     // zero are actually demanded by the other operand.
52236     auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) {
52237       APInt UndefElts;
52238       SmallVector<APInt> EltBits;
52239       APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
52240       APInt DemandedElts = APInt::getAllOnes(NumElts);
52241       if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
52242                                         EltBits)) {
52243         DemandedBits.clearAllBits();
52244         DemandedElts.clearAllBits();
52245         for (int I = 0; I != NumElts; ++I) {
52246           if (UndefElts[I]) {
52247             // We can't assume an undef src element gives an undef dst - the
52248             // other src might be zero.
52249             DemandedBits.setAllBits();
52250             DemandedElts.setBit(I);
52251           } else if ((Invert && !EltBits[I].isAllOnes()) ||
52252                      (!Invert && !EltBits[I].isZero())) {
52253             DemandedBits |= Invert ? ~EltBits[I] : EltBits[I];
52254             DemandedElts.setBit(I);
52255           }
52256         }
52257       }
52258       return std::make_pair(DemandedBits, DemandedElts);
52259     };
52260     APInt Bits0, Elts0;
52261     APInt Bits1, Elts1;
52262     std::tie(Bits0, Elts0) = GetDemandedMasks(N1);
52263     std::tie(Bits1, Elts1) = GetDemandedMasks(N0, true);
52264 
52265     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52266     if (TLI.SimplifyDemandedVectorElts(N0, Elts0, DCI) ||
52267         TLI.SimplifyDemandedVectorElts(N1, Elts1, DCI) ||
52268         TLI.SimplifyDemandedBits(N0, Bits0, Elts0, DCI) ||
52269         TLI.SimplifyDemandedBits(N1, Bits1, Elts1, DCI)) {
52270       if (N->getOpcode() != ISD::DELETED_NODE)
52271         DCI.AddToWorklist(N);
52272       return SDValue(N, 0);
52273     }
52274   }
52275 
52276   return SDValue();
52277 }
52278 
52279 static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
52280                          TargetLowering::DAGCombinerInfo &DCI) {
52281   SDValue N1 = N->getOperand(1);
52282 
52283   // BT ignores high bits in the bit index operand.
52284   unsigned BitWidth = N1.getValueSizeInBits();
52285   APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
52286   if (DAG.getTargetLoweringInfo().SimplifyDemandedBits(N1, DemandedMask, DCI)) {
52287     if (N->getOpcode() != ISD::DELETED_NODE)
52288       DCI.AddToWorklist(N);
52289     return SDValue(N, 0);
52290   }
52291 
52292   return SDValue();
52293 }
52294 
52295 static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG,
52296                                TargetLowering::DAGCombinerInfo &DCI) {
52297   bool IsStrict = N->getOpcode() == X86ISD::STRICT_CVTPH2PS;
52298   SDValue Src = N->getOperand(IsStrict ? 1 : 0);
52299 
52300   if (N->getValueType(0) == MVT::v4f32 && Src.getValueType() == MVT::v8i16) {
52301     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52302     APInt DemandedElts = APInt::getLowBitsSet(8, 4);
52303     if (TLI.SimplifyDemandedVectorElts(Src, DemandedElts, DCI)) {
52304       if (N->getOpcode() != ISD::DELETED_NODE)
52305         DCI.AddToWorklist(N);
52306       return SDValue(N, 0);
52307     }
52308 
52309     // Convert a full vector load into vzload when not all bits are needed.
52310     if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) {
52311       LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(IsStrict ? 1 : 0));
52312       if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::i64, MVT::v2i64, DAG)) {
52313         SDLoc dl(N);
52314         if (IsStrict) {
52315           SDValue Convert = DAG.getNode(
52316               N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
52317               {N->getOperand(0), DAG.getBitcast(MVT::v8i16, VZLoad)});
52318           DCI.CombineTo(N, Convert, Convert.getValue(1));
52319         } else {
52320           SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32,
52321                                         DAG.getBitcast(MVT::v8i16, VZLoad));
52322           DCI.CombineTo(N, Convert);
52323         }
52324 
52325         DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
52326         DCI.recursivelyDeleteUnusedNodes(LN);
52327         return SDValue(N, 0);
52328       }
52329     }
52330   }
52331 
52332   return SDValue();
52333 }
52334 
52335 // Try to combine sext_in_reg of a cmov of constants by extending the constants.
52336 static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) {
52337   assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
52338 
52339   EVT DstVT = N->getValueType(0);
52340 
52341   SDValue N0 = N->getOperand(0);
52342   SDValue N1 = N->getOperand(1);
52343   EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
52344 
52345   if (ExtraVT != MVT::i8 && ExtraVT != MVT::i16)
52346     return SDValue();
52347 
52348   // Look through single use any_extends / truncs.
52349   SDValue IntermediateBitwidthOp;
52350   if ((N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::TRUNCATE) &&
52351       N0.hasOneUse()) {
52352     IntermediateBitwidthOp = N0;
52353     N0 = N0.getOperand(0);
52354   }
52355 
52356   // See if we have a single use cmov.
52357   if (N0.getOpcode() != X86ISD::CMOV || !N0.hasOneUse())
52358     return SDValue();
52359 
52360   SDValue CMovOp0 = N0.getOperand(0);
52361   SDValue CMovOp1 = N0.getOperand(1);
52362 
52363   // Make sure both operands are constants.
52364   if (!isa<ConstantSDNode>(CMovOp0.getNode()) ||
52365       !isa<ConstantSDNode>(CMovOp1.getNode()))
52366     return SDValue();
52367 
52368   SDLoc DL(N);
52369 
52370   // If we looked through an any_extend/trunc above, add one to the constants.
52371   if (IntermediateBitwidthOp) {
52372     unsigned IntermediateOpc = IntermediateBitwidthOp.getOpcode();
52373     CMovOp0 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp0);
52374     CMovOp1 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp1);
52375   }
52376 
52377   CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp0, N1);
52378   CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp1, N1);
52379 
52380   EVT CMovVT = DstVT;
52381   // We do not want i16 CMOV's. Promote to i32 and truncate afterwards.
52382   if (DstVT == MVT::i16) {
52383     CMovVT = MVT::i32;
52384     CMovOp0 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp0);
52385     CMovOp1 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp1);
52386   }
52387 
52388   SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
52389                              N0.getOperand(2), N0.getOperand(3));
52390 
52391   if (CMovVT != DstVT)
52392     CMov = DAG.getNode(ISD::TRUNCATE, DL, DstVT, CMov);
52393 
52394   return CMov;
52395 }
52396 
52397 static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG,
52398                                       const X86Subtarget &Subtarget) {
52399   assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
52400 
52401   if (SDValue V = combineSextInRegCmov(N, DAG))
52402     return V;
52403 
52404   EVT VT = N->getValueType(0);
52405   SDValue N0 = N->getOperand(0);
52406   SDValue N1 = N->getOperand(1);
52407   EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
52408   SDLoc dl(N);
52409 
52410   // The SIGN_EXTEND_INREG to v4i64 is expensive operation on the
52411   // both SSE and AVX2 since there is no sign-extended shift right
52412   // operation on a vector with 64-bit elements.
52413   //(sext_in_reg (v4i64 anyext (v4i32 x )), ExtraVT) ->
52414   // (v4i64 sext (v4i32 sext_in_reg (v4i32 x , ExtraVT)))
52415   if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||
52416                            N0.getOpcode() == ISD::SIGN_EXTEND)) {
52417     SDValue N00 = N0.getOperand(0);
52418 
52419     // EXTLOAD has a better solution on AVX2,
52420     // it may be replaced with X86ISD::VSEXT node.
52421     if (N00.getOpcode() == ISD::LOAD && Subtarget.hasInt256())
52422       if (!ISD::isNormalLoad(N00.getNode()))
52423         return SDValue();
52424 
52425     // Attempt to promote any comparison mask ops before moving the
52426     // SIGN_EXTEND_INREG in the way.
52427     if (SDValue Promote = PromoteMaskArithmetic(N0.getNode(), DAG, Subtarget))
52428       return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Promote, N1);
52429 
52430     if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) {
52431       SDValue Tmp =
52432           DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, N00, N1);
52433       return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);
52434     }
52435   }
52436   return SDValue();
52437 }
52438 
52439 /// sext(add_nsw(x, C)) --> add(sext(x), C_sext)
52440 /// zext(add_nuw(x, C)) --> add(zext(x), C_zext)
52441 /// Promoting a sign/zero extension ahead of a no overflow 'add' exposes
52442 /// opportunities to combine math ops, use an LEA, or use a complex addressing
52443 /// mode. This can eliminate extend, add, and shift instructions.
52444 static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
52445                                    const X86Subtarget &Subtarget) {
52446   if (Ext->getOpcode() != ISD::SIGN_EXTEND &&
52447       Ext->getOpcode() != ISD::ZERO_EXTEND)
52448     return SDValue();
52449 
52450   // TODO: This should be valid for other integer types.
52451   EVT VT = Ext->getValueType(0);
52452   if (VT != MVT::i64)
52453     return SDValue();
52454 
52455   SDValue Add = Ext->getOperand(0);
52456   if (Add.getOpcode() != ISD::ADD)
52457     return SDValue();
52458 
52459   SDValue AddOp0 = Add.getOperand(0);
52460   SDValue AddOp1 = Add.getOperand(1);
52461   bool Sext = Ext->getOpcode() == ISD::SIGN_EXTEND;
52462   bool NSW = Add->getFlags().hasNoSignedWrap();
52463   bool NUW = Add->getFlags().hasNoUnsignedWrap();
52464   NSW = NSW || (Sext && DAG.willNotOverflowAdd(true, AddOp0, AddOp1));
52465   NUW = NUW || (!Sext && DAG.willNotOverflowAdd(false, AddOp0, AddOp1));
52466 
52467   // We need an 'add nsw' feeding into the 'sext' or 'add nuw' feeding
52468   // into the 'zext'
52469   if ((Sext && !NSW) || (!Sext && !NUW))
52470     return SDValue();
52471 
52472   // Having a constant operand to the 'add' ensures that we are not increasing
52473   // the instruction count because the constant is extended for free below.
52474   // A constant operand can also become the displacement field of an LEA.
52475   auto *AddOp1C = dyn_cast<ConstantSDNode>(AddOp1);
52476   if (!AddOp1C)
52477     return SDValue();
52478 
52479   // Don't make the 'add' bigger if there's no hope of combining it with some
52480   // other 'add' or 'shl' instruction.
52481   // TODO: It may be profitable to generate simpler LEA instructions in place
52482   // of single 'add' instructions, but the cost model for selecting an LEA
52483   // currently has a high threshold.
52484   bool HasLEAPotential = false;
52485   for (auto *User : Ext->uses()) {
52486     if (User->getOpcode() == ISD::ADD || User->getOpcode() == ISD::SHL) {
52487       HasLEAPotential = true;
52488       break;
52489     }
52490   }
52491   if (!HasLEAPotential)
52492     return SDValue();
52493 
52494   // Everything looks good, so pull the '{s|z}ext' ahead of the 'add'.
52495   int64_t AddC = Sext ? AddOp1C->getSExtValue() : AddOp1C->getZExtValue();
52496   SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);
52497   SDValue NewConstant = DAG.getConstant(AddC, SDLoc(Add), VT);
52498 
52499   // The wider add is guaranteed to not wrap because both operands are
52500   // sign-extended.
52501   SDNodeFlags Flags;
52502   Flags.setNoSignedWrap(NSW);
52503   Flags.setNoUnsignedWrap(NUW);
52504   return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags);
52505 }
52506 
52507 // If we face {ANY,SIGN,ZERO}_EXTEND that is applied to a CMOV with constant
52508 // operands and the result of CMOV is not used anywhere else - promote CMOV
52509 // itself instead of promoting its result. This could be beneficial, because:
52510 //     1) X86TargetLowering::EmitLoweredSelect later can do merging of two
52511 //        (or more) pseudo-CMOVs only when they go one-after-another and
52512 //        getting rid of result extension code after CMOV will help that.
52513 //     2) Promotion of constant CMOV arguments is free, hence the
52514 //        {ANY,SIGN,ZERO}_EXTEND will just be deleted.
52515 //     3) 16-bit CMOV encoding is 4 bytes, 32-bit CMOV is 3-byte, so this
52516 //        promotion is also good in terms of code-size.
52517 //        (64-bit CMOV is 4-bytes, that's why we don't do 32-bit => 64-bit
52518 //         promotion).
52519 static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
52520   SDValue CMovN = Extend->getOperand(0);
52521   if (CMovN.getOpcode() != X86ISD::CMOV || !CMovN.hasOneUse())
52522     return SDValue();
52523 
52524   EVT TargetVT = Extend->getValueType(0);
52525   unsigned ExtendOpcode = Extend->getOpcode();
52526   SDLoc DL(Extend);
52527 
52528   EVT VT = CMovN.getValueType();
52529   SDValue CMovOp0 = CMovN.getOperand(0);
52530   SDValue CMovOp1 = CMovN.getOperand(1);
52531 
52532   if (!isa<ConstantSDNode>(CMovOp0.getNode()) ||
52533       !isa<ConstantSDNode>(CMovOp1.getNode()))
52534     return SDValue();
52535 
52536   // Only extend to i32 or i64.
52537   if (TargetVT != MVT::i32 && TargetVT != MVT::i64)
52538     return SDValue();
52539 
52540   // Only extend from i16 unless its a sign_extend from i32. Zext/aext from i32
52541   // are free.
52542   if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32))
52543     return SDValue();
52544 
52545   // If this a zero extend to i64, we should only extend to i32 and use a free
52546   // zero extend to finish.
52547   EVT ExtendVT = TargetVT;
52548   if (TargetVT == MVT::i64 && ExtendOpcode != ISD::SIGN_EXTEND)
52549     ExtendVT = MVT::i32;
52550 
52551   CMovOp0 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp0);
52552   CMovOp1 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp1);
52553 
52554   SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
52555                             CMovN.getOperand(2), CMovN.getOperand(3));
52556 
52557   // Finish extending if needed.
52558   if (ExtendVT != TargetVT)
52559     Res = DAG.getNode(ExtendOpcode, DL, TargetVT, Res);
52560 
52561   return Res;
52562 }
52563 
52564 // Attempt to combine a (sext/zext (setcc)) to a setcc with a xmm/ymm/zmm
52565 // result type.
52566 static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
52567                                const X86Subtarget &Subtarget) {
52568   SDValue N0 = N->getOperand(0);
52569   EVT VT = N->getValueType(0);
52570   SDLoc dl(N);
52571 
52572   // Only do this combine with AVX512 for vector extends.
52573   if (!Subtarget.hasAVX512() || !VT.isVector() || N0.getOpcode() != ISD::SETCC)
52574     return SDValue();
52575 
52576   // Only combine legal element types.
52577   EVT SVT = VT.getVectorElementType();
52578   if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 &&
52579       SVT != MVT::i64 && SVT != MVT::f32 && SVT != MVT::f64)
52580     return SDValue();
52581 
52582   // We don't have CMPP Instruction for vxf16
52583   if (N0.getOperand(0).getValueType().getVectorElementType() == MVT::f16)
52584     return SDValue();
52585   // We can only do this if the vector size in 256 bits or less.
52586   unsigned Size = VT.getSizeInBits();
52587   if (Size > 256 && Subtarget.useAVX512Regs())
52588     return SDValue();
52589 
52590   // Don't fold if the condition code can't be handled by PCMPEQ/PCMPGT since
52591   // that's the only integer compares with we have.
52592   ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
52593   if (ISD::isUnsignedIntSetCC(CC))
52594     return SDValue();
52595 
52596   // Only do this combine if the extension will be fully consumed by the setcc.
52597   EVT N00VT = N0.getOperand(0).getValueType();
52598   EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
52599   if (Size != MatchingVecType.getSizeInBits())
52600     return SDValue();
52601 
52602   SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
52603 
52604   if (N->getOpcode() == ISD::ZERO_EXTEND)
52605     Res = DAG.getZeroExtendInReg(Res, dl, N0.getValueType());
52606 
52607   return Res;
52608 }
52609 
52610 static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
52611                            TargetLowering::DAGCombinerInfo &DCI,
52612                            const X86Subtarget &Subtarget) {
52613   SDValue N0 = N->getOperand(0);
52614   EVT VT = N->getValueType(0);
52615   SDLoc DL(N);
52616 
52617   // (i32 (sext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
52618   if (!DCI.isBeforeLegalizeOps() &&
52619       N0.getOpcode() == X86ISD::SETCC_CARRY) {
52620     SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
52621                                  N0->getOperand(1));
52622     bool ReplaceOtherUses = !N0.hasOneUse();
52623     DCI.CombineTo(N, Setcc);
52624     // Replace other uses with a truncate of the widened setcc_carry.
52625     if (ReplaceOtherUses) {
52626       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
52627                                   N0.getValueType(), Setcc);
52628       DCI.CombineTo(N0.getNode(), Trunc);
52629     }
52630 
52631     return SDValue(N, 0);
52632   }
52633 
52634   if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
52635     return NewCMov;
52636 
52637   if (!DCI.isBeforeLegalizeOps())
52638     return SDValue();
52639 
52640   if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
52641     return V;
52642 
52643   if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), DL, VT, N0,
52644                                                  DAG, DCI, Subtarget))
52645     return V;
52646 
52647   if (VT.isVector()) {
52648     if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
52649       return R;
52650 
52651     if (N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG)
52652       return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0));
52653   }
52654 
52655   if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
52656     return NewAdd;
52657 
52658   return SDValue();
52659 }
52660 
52661 // Inverting a constant vector is profitable if it can be eliminated and the
52662 // inverted vector is already present in DAG. Otherwise, it will be loaded
52663 // anyway.
52664 //
52665 // We determine which of the values can be completely eliminated and invert it.
52666 // If both are eliminable, select a vector with the first negative element.
52667 static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG) {
52668   assert(ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()) &&
52669          "ConstantFP build vector expected");
52670   // Check if we can eliminate V. We assume if a value is only used in FMAs, we
52671   // can eliminate it. Since this function is invoked for each FMA with this
52672   // vector.
52673   auto IsNotFMA = [](SDNode *Use) {
52674     return Use->getOpcode() != ISD::FMA && Use->getOpcode() != ISD::STRICT_FMA;
52675   };
52676   if (llvm::any_of(V->uses(), IsNotFMA))
52677     return SDValue();
52678 
52679   SmallVector<SDValue, 8> Ops;
52680   EVT VT = V.getValueType();
52681   EVT EltVT = VT.getVectorElementType();
52682   for (auto Op : V->op_values()) {
52683     if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
52684       Ops.push_back(DAG.getConstantFP(-Cst->getValueAPF(), SDLoc(Op), EltVT));
52685     } else {
52686       assert(Op.isUndef());
52687       Ops.push_back(DAG.getUNDEF(EltVT));
52688     }
52689   }
52690 
52691   SDNode *NV = DAG.getNodeIfExists(ISD::BUILD_VECTOR, DAG.getVTList(VT), Ops);
52692   if (!NV)
52693     return SDValue();
52694 
52695   // If an inverted version cannot be eliminated, choose it instead of the
52696   // original version.
52697   if (llvm::any_of(NV->uses(), IsNotFMA))
52698     return SDValue(NV, 0);
52699 
52700   // If the inverted version also can be eliminated, we have to consistently
52701   // prefer one of the values. We prefer a constant with a negative value on
52702   // the first place.
52703   // N.B. We need to skip undefs that may precede a value.
52704   for (auto op : V->op_values()) {
52705     if (auto *Cst = dyn_cast<ConstantFPSDNode>(op)) {
52706       if (Cst->isNegative())
52707         return SDValue();
52708       break;
52709     }
52710   }
52711   return SDValue(NV, 0);
52712 }
52713 
52714 static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
52715                           TargetLowering::DAGCombinerInfo &DCI,
52716                           const X86Subtarget &Subtarget) {
52717   SDLoc dl(N);
52718   EVT VT = N->getValueType(0);
52719   bool IsStrict = N->isStrictFPOpcode() || N->isTargetStrictFPOpcode();
52720 
52721   // Let legalize expand this if it isn't a legal type yet.
52722   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52723   if (!TLI.isTypeLegal(VT))
52724     return SDValue();
52725 
52726   SDValue A = N->getOperand(IsStrict ? 1 : 0);
52727   SDValue B = N->getOperand(IsStrict ? 2 : 1);
52728   SDValue C = N->getOperand(IsStrict ? 3 : 2);
52729 
52730   // If the operation allows fast-math and the target does not support FMA,
52731   // split this into mul+add to avoid libcall(s).
52732   SDNodeFlags Flags = N->getFlags();
52733   if (!IsStrict && Flags.hasAllowReassociation() &&
52734       TLI.isOperationExpand(ISD::FMA, VT)) {
52735     SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags);
52736     return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags);
52737   }
52738 
52739   EVT ScalarVT = VT.getScalarType();
52740   if (((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||
52741        !Subtarget.hasAnyFMA()) &&
52742       !(ScalarVT == MVT::f16 && Subtarget.hasFP16()))
52743     return SDValue();
52744 
52745   auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) {
52746     bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
52747     bool LegalOperations = !DCI.isBeforeLegalizeOps();
52748     if (SDValue NegV = TLI.getCheaperNegatedExpression(V, DAG, LegalOperations,
52749                                                        CodeSize)) {
52750       V = NegV;
52751       return true;
52752     }
52753     // Look through extract_vector_elts. If it comes from an FNEG, create a
52754     // new extract from the FNEG input.
52755     if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
52756         isNullConstant(V.getOperand(1))) {
52757       SDValue Vec = V.getOperand(0);
52758       if (SDValue NegV = TLI.getCheaperNegatedExpression(
52759               Vec, DAG, LegalOperations, CodeSize)) {
52760         V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(),
52761                         NegV, V.getOperand(1));
52762         return true;
52763       }
52764     }
52765     // Lookup if there is an inverted version of constant vector V in DAG.
52766     if (ISD::isBuildVectorOfConstantFPSDNodes(V.getNode())) {
52767       if (SDValue NegV = getInvertedVectorForFMA(V, DAG)) {
52768         V = NegV;
52769         return true;
52770       }
52771     }
52772     return false;
52773   };
52774 
52775   // Do not convert the passthru input of scalar intrinsics.
52776   // FIXME: We could allow negations of the lower element only.
52777   bool NegA = invertIfNegative(A);
52778   bool NegB = invertIfNegative(B);
52779   bool NegC = invertIfNegative(C);
52780 
52781   if (!NegA && !NegB && !NegC)
52782     return SDValue();
52783 
52784   unsigned NewOpcode =
52785       negateFMAOpcode(N->getOpcode(), NegA != NegB, NegC, false);
52786 
52787   // Propagate fast-math-flags to new FMA node.
52788   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
52789   if (IsStrict) {
52790     assert(N->getNumOperands() == 4 && "Shouldn't be greater than 4");
52791     return DAG.getNode(NewOpcode, dl, {VT, MVT::Other},
52792                        {N->getOperand(0), A, B, C});
52793   } else {
52794     if (N->getNumOperands() == 4)
52795       return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3));
52796     return DAG.getNode(NewOpcode, dl, VT, A, B, C);
52797   }
52798 }
52799 
52800 // Combine FMADDSUB(A, B, FNEG(C)) -> FMSUBADD(A, B, C)
52801 // Combine FMSUBADD(A, B, FNEG(C)) -> FMADDSUB(A, B, C)
52802 static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
52803                                TargetLowering::DAGCombinerInfo &DCI) {
52804   SDLoc dl(N);
52805   EVT VT = N->getValueType(0);
52806   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52807   bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
52808   bool LegalOperations = !DCI.isBeforeLegalizeOps();
52809 
52810   SDValue N2 = N->getOperand(2);
52811 
52812   SDValue NegN2 =
52813       TLI.getCheaperNegatedExpression(N2, DAG, LegalOperations, CodeSize);
52814   if (!NegN2)
52815     return SDValue();
52816   unsigned NewOpcode = negateFMAOpcode(N->getOpcode(), false, true, false);
52817 
52818   if (N->getNumOperands() == 4)
52819     return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
52820                        NegN2, N->getOperand(3));
52821   return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
52822                      NegN2);
52823 }
52824 
52825 static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
52826                            TargetLowering::DAGCombinerInfo &DCI,
52827                            const X86Subtarget &Subtarget) {
52828   SDLoc dl(N);
52829   SDValue N0 = N->getOperand(0);
52830   EVT VT = N->getValueType(0);
52831 
52832   // (i32 (aext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
52833   // FIXME: Is this needed? We don't seem to have any tests for it.
52834   if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ANY_EXTEND &&
52835       N0.getOpcode() == X86ISD::SETCC_CARRY) {
52836     SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0),
52837                                  N0->getOperand(1));
52838     bool ReplaceOtherUses = !N0.hasOneUse();
52839     DCI.CombineTo(N, Setcc);
52840     // Replace other uses with a truncate of the widened setcc_carry.
52841     if (ReplaceOtherUses) {
52842       SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
52843                                   N0.getValueType(), Setcc);
52844       DCI.CombineTo(N0.getNode(), Trunc);
52845     }
52846 
52847     return SDValue(N, 0);
52848   }
52849 
52850   if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
52851     return NewCMov;
52852 
52853   if (DCI.isBeforeLegalizeOps())
52854     if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
52855       return V;
52856 
52857   if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), dl, VT, N0,
52858                                                  DAG, DCI, Subtarget))
52859     return V;
52860 
52861   if (VT.isVector())
52862     if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
52863       return R;
52864 
52865   if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
52866     return NewAdd;
52867 
52868   if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget))
52869     return R;
52870 
52871   // TODO: Combine with any target/faux shuffle.
52872   if (N0.getOpcode() == X86ISD::PACKUS && N0.getValueSizeInBits() == 128 &&
52873       VT.getScalarSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits()) {
52874     SDValue N00 = N0.getOperand(0);
52875     SDValue N01 = N0.getOperand(1);
52876     unsigned NumSrcEltBits = N00.getScalarValueSizeInBits();
52877     APInt ZeroMask = APInt::getHighBitsSet(NumSrcEltBits, NumSrcEltBits / 2);
52878     if ((N00.isUndef() || DAG.MaskedValueIsZero(N00, ZeroMask)) &&
52879         (N01.isUndef() || DAG.MaskedValueIsZero(N01, ZeroMask))) {
52880       return concatSubVectors(N00, N01, DAG, dl);
52881     }
52882   }
52883 
52884   return SDValue();
52885 }
52886 
52887 /// If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just
52888 /// pre-promote its result type since vXi1 vectors don't get promoted
52889 /// during type legalization.
52890 static SDValue truncateAVX512SetCCNoBWI(EVT VT, EVT OpVT, SDValue LHS,
52891                                         SDValue RHS, ISD::CondCode CC,
52892                                         const SDLoc &DL, SelectionDAG &DAG,
52893                                         const X86Subtarget &Subtarget) {
52894   if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() &&
52895       VT.getVectorElementType() == MVT::i1 &&
52896       (OpVT.getVectorElementType() == MVT::i8 ||
52897        OpVT.getVectorElementType() == MVT::i16)) {
52898     SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC);
52899     return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc);
52900   }
52901   return SDValue();
52902 }
52903 
52904 static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
52905                             TargetLowering::DAGCombinerInfo &DCI,
52906                             const X86Subtarget &Subtarget) {
52907   const ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
52908   const SDValue LHS = N->getOperand(0);
52909   const SDValue RHS = N->getOperand(1);
52910   EVT VT = N->getValueType(0);
52911   EVT OpVT = LHS.getValueType();
52912   SDLoc DL(N);
52913 
52914   if (CC == ISD::SETNE || CC == ISD::SETEQ) {
52915     if (SDValue V = combineVectorSizedSetCCEquality(VT, LHS, RHS, CC, DL, DAG,
52916                                                     Subtarget))
52917       return V;
52918 
52919     if (VT == MVT::i1) {
52920       X86::CondCode X86CC;
52921       if (SDValue V =
52922               MatchVectorAllEqualTest(LHS, RHS, CC, DL, Subtarget, DAG, X86CC))
52923         return DAG.getNode(ISD::TRUNCATE, DL, VT, getSETCC(X86CC, V, DL, DAG));
52924     }
52925 
52926     if (OpVT.isScalarInteger()) {
52927       // cmpeq(or(X,Y),X) --> cmpeq(and(~X,Y),0)
52928       // cmpne(or(X,Y),X) --> cmpne(and(~X,Y),0)
52929       auto MatchOrCmpEq = [&](SDValue N0, SDValue N1) {
52930         if (N0.getOpcode() == ISD::OR && N0->hasOneUse()) {
52931           if (N0.getOperand(0) == N1)
52932             return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT),
52933                                N0.getOperand(1));
52934           if (N0.getOperand(1) == N1)
52935             return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT),
52936                                N0.getOperand(0));
52937         }
52938         return SDValue();
52939       };
52940       if (SDValue AndN = MatchOrCmpEq(LHS, RHS))
52941         return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
52942       if (SDValue AndN = MatchOrCmpEq(RHS, LHS))
52943         return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
52944 
52945       // cmpeq(and(X,Y),Y) --> cmpeq(and(~X,Y),0)
52946       // cmpne(and(X,Y),Y) --> cmpne(and(~X,Y),0)
52947       auto MatchAndCmpEq = [&](SDValue N0, SDValue N1) {
52948         if (N0.getOpcode() == ISD::AND && N0->hasOneUse()) {
52949           if (N0.getOperand(0) == N1)
52950             return DAG.getNode(ISD::AND, DL, OpVT, N1,
52951                                DAG.getNOT(DL, N0.getOperand(1), OpVT));
52952           if (N0.getOperand(1) == N1)
52953             return DAG.getNode(ISD::AND, DL, OpVT, N1,
52954                                DAG.getNOT(DL, N0.getOperand(0), OpVT));
52955         }
52956         return SDValue();
52957       };
52958       if (SDValue AndN = MatchAndCmpEq(LHS, RHS))
52959         return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
52960       if (SDValue AndN = MatchAndCmpEq(RHS, LHS))
52961         return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
52962 
52963       // cmpeq(trunc(x),C) --> cmpeq(x,C)
52964       // cmpne(trunc(x),C) --> cmpne(x,C)
52965       // iff x upper bits are zero.
52966       if (LHS.getOpcode() == ISD::TRUNCATE &&
52967           LHS.getOperand(0).getScalarValueSizeInBits() >= 32 &&
52968           isa<ConstantSDNode>(RHS) && !DCI.isBeforeLegalize()) {
52969         EVT SrcVT = LHS.getOperand(0).getValueType();
52970         APInt UpperBits = APInt::getBitsSetFrom(SrcVT.getScalarSizeInBits(),
52971                                                 OpVT.getScalarSizeInBits());
52972         const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52973         auto *C = cast<ConstantSDNode>(RHS);
52974         if (DAG.MaskedValueIsZero(LHS.getOperand(0), UpperBits) &&
52975             TLI.isTypeLegal(LHS.getOperand(0).getValueType()))
52976           return DAG.getSetCC(DL, VT, LHS.getOperand(0),
52977                               DAG.getConstant(C->getAPIntValue().zextOrTrunc(
52978                                                   SrcVT.getScalarSizeInBits()),
52979                                               DL, SrcVT),
52980                               CC);
52981       }
52982 
52983       // With C as a power of 2 and C != 0 and C != INT_MIN:
52984       //    icmp eq Abs(X) C ->
52985       //        (icmp eq A, C) | (icmp eq A, -C)
52986       //    icmp ne Abs(X) C ->
52987       //        (icmp ne A, C) & (icmp ne A, -C)
52988       // Both of these patterns can be better optimized in
52989       // DAGCombiner::foldAndOrOfSETCC. Note this only applies for scalar
52990       // integers which is checked above.
52991       if (LHS.getOpcode() == ISD::ABS && LHS.hasOneUse()) {
52992         if (auto *C = dyn_cast<ConstantSDNode>(RHS)) {
52993           const APInt &CInt = C->getAPIntValue();
52994           // We can better optimize this case in DAGCombiner::foldAndOrOfSETCC.
52995           if (CInt.isPowerOf2() && !CInt.isMinSignedValue()) {
52996             SDValue BaseOp = LHS.getOperand(0);
52997             SDValue SETCC0 = DAG.getSetCC(DL, VT, BaseOp, RHS, CC);
52998             SDValue SETCC1 = DAG.getSetCC(
52999                 DL, VT, BaseOp, DAG.getConstant(-CInt, DL, OpVT), CC);
53000             return DAG.getNode(CC == ISD::SETEQ ? ISD::OR : ISD::AND, DL, VT,
53001                                SETCC0, SETCC1);
53002           }
53003         }
53004       }
53005     }
53006   }
53007 
53008   if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
53009       (CC == ISD::SETNE || CC == ISD::SETEQ || ISD::isSignedIntSetCC(CC))) {
53010     // Using temporaries to avoid messing up operand ordering for later
53011     // transformations if this doesn't work.
53012     SDValue Op0 = LHS;
53013     SDValue Op1 = RHS;
53014     ISD::CondCode TmpCC = CC;
53015     // Put build_vector on the right.
53016     if (Op0.getOpcode() == ISD::BUILD_VECTOR) {
53017       std::swap(Op0, Op1);
53018       TmpCC = ISD::getSetCCSwappedOperands(TmpCC);
53019     }
53020 
53021     bool IsSEXT0 =
53022         (Op0.getOpcode() == ISD::SIGN_EXTEND) &&
53023         (Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1);
53024     bool IsVZero1 = ISD::isBuildVectorAllZeros(Op1.getNode());
53025 
53026     if (IsSEXT0 && IsVZero1) {
53027       assert(VT == Op0.getOperand(0).getValueType() &&
53028              "Unexpected operand type");
53029       if (TmpCC == ISD::SETGT)
53030         return DAG.getConstant(0, DL, VT);
53031       if (TmpCC == ISD::SETLE)
53032         return DAG.getConstant(1, DL, VT);
53033       if (TmpCC == ISD::SETEQ || TmpCC == ISD::SETGE)
53034         return DAG.getNOT(DL, Op0.getOperand(0), VT);
53035 
53036       assert((TmpCC == ISD::SETNE || TmpCC == ISD::SETLT) &&
53037              "Unexpected condition code!");
53038       return Op0.getOperand(0);
53039     }
53040   }
53041 
53042   // Try and make unsigned vector comparison signed. On pre AVX512 targets there
53043   // only are unsigned comparisons (`PCMPGT`) and on AVX512 its often better to
53044   // use `PCMPGT` if the result is mean to stay in a vector (and if its going to
53045   // a mask, there are signed AVX512 comparisons).
53046   if (VT.isVector() && OpVT.isVector() && OpVT.isInteger()) {
53047     bool CanMakeSigned = false;
53048     if (ISD::isUnsignedIntSetCC(CC)) {
53049       KnownBits CmpKnown =
53050           DAG.computeKnownBits(LHS).intersectWith(DAG.computeKnownBits(RHS));
53051       // If we know LHS/RHS share the same sign bit at each element we can
53052       // make this signed.
53053       // NOTE: `computeKnownBits` on a vector type aggregates common bits
53054       // across all lanes. So a pattern where the sign varies from lane to
53055       // lane, but at each lane Sign(LHS) is known to equal Sign(RHS), will be
53056       // missed. We could get around this by demanding each lane
53057       // independently, but this isn't the most important optimization and
53058       // that may eat into compile time.
53059       CanMakeSigned =
53060           CmpKnown.Zero.isSignBitSet() || CmpKnown.One.isSignBitSet();
53061     }
53062     if (CanMakeSigned || ISD::isSignedIntSetCC(CC)) {
53063       SDValue LHSOut = LHS;
53064       SDValue RHSOut = RHS;
53065       ISD::CondCode NewCC = CC;
53066       switch (CC) {
53067       case ISD::SETGE:
53068       case ISD::SETUGE:
53069         if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ true,
53070                                                   /*NSW*/ true))
53071           LHSOut = NewLHS;
53072         else if (SDValue NewRHS = incDecVectorConstant(
53073                      RHS, DAG, /*IsInc*/ false, /*NSW*/ true))
53074           RHSOut = NewRHS;
53075         else
53076           break;
53077 
53078         [[fallthrough]];
53079       case ISD::SETUGT:
53080         NewCC = ISD::SETGT;
53081         break;
53082 
53083       case ISD::SETLE:
53084       case ISD::SETULE:
53085         if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ false,
53086                                                   /*NSW*/ true))
53087           LHSOut = NewLHS;
53088         else if (SDValue NewRHS = incDecVectorConstant(RHS, DAG, /*IsInc*/ true,
53089                                                        /*NSW*/ true))
53090           RHSOut = NewRHS;
53091         else
53092           break;
53093 
53094         [[fallthrough]];
53095       case ISD::SETULT:
53096         // Will be swapped to SETGT in LowerVSETCC*.
53097         NewCC = ISD::SETLT;
53098         break;
53099       default:
53100         break;
53101       }
53102       if (NewCC != CC) {
53103         if (SDValue R = truncateAVX512SetCCNoBWI(VT, OpVT, LHSOut, RHSOut,
53104                                                  NewCC, DL, DAG, Subtarget))
53105           return R;
53106         return DAG.getSetCC(DL, VT, LHSOut, RHSOut, NewCC);
53107       }
53108     }
53109   }
53110 
53111   if (SDValue R =
53112           truncateAVX512SetCCNoBWI(VT, OpVT, LHS, RHS, CC, DL, DAG, Subtarget))
53113     return R;
53114 
53115   // For an SSE1-only target, lower a comparison of v4f32 to X86ISD::CMPP early
53116   // to avoid scalarization via legalization because v4i32 is not a legal type.
53117   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 &&
53118       LHS.getValueType() == MVT::v4f32)
53119     return LowerVSETCC(SDValue(N, 0), Subtarget, DAG);
53120 
53121   // X pred 0.0 --> X pred -X
53122   // If the negation of X already exists, use it in the comparison. This removes
53123   // the need to materialize 0.0 and allows matching to SSE's MIN/MAX
53124   // instructions in patterns with a 'select' node.
53125   if (isNullFPScalarOrVectorConst(RHS)) {
53126     SDVTList FNegVT = DAG.getVTList(OpVT);
53127     if (SDNode *FNeg = DAG.getNodeIfExists(ISD::FNEG, FNegVT, {LHS}))
53128       return DAG.getSetCC(DL, VT, LHS, SDValue(FNeg, 0), CC);
53129   }
53130 
53131   return SDValue();
53132 }
53133 
53134 static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
53135                              TargetLowering::DAGCombinerInfo &DCI,
53136                              const X86Subtarget &Subtarget) {
53137   SDValue Src = N->getOperand(0);
53138   MVT SrcVT = Src.getSimpleValueType();
53139   MVT VT = N->getSimpleValueType(0);
53140   unsigned NumBits = VT.getScalarSizeInBits();
53141   unsigned NumElts = SrcVT.getVectorNumElements();
53142   unsigned NumBitsPerElt = SrcVT.getScalarSizeInBits();
53143   assert(VT == MVT::i32 && NumElts <= NumBits && "Unexpected MOVMSK types");
53144 
53145   // Perform constant folding.
53146   APInt UndefElts;
53147   SmallVector<APInt, 32> EltBits;
53148   if (getTargetConstantBitsFromNode(Src, NumBitsPerElt, UndefElts, EltBits)) {
53149     APInt Imm(32, 0);
53150     for (unsigned Idx = 0; Idx != NumElts; ++Idx)
53151       if (!UndefElts[Idx] && EltBits[Idx].isNegative())
53152         Imm.setBit(Idx);
53153 
53154     return DAG.getConstant(Imm, SDLoc(N), VT);
53155   }
53156 
53157   // Look through int->fp bitcasts that don't change the element width.
53158   unsigned EltWidth = SrcVT.getScalarSizeInBits();
53159   if (Subtarget.hasSSE2() && Src.getOpcode() == ISD::BITCAST &&
53160       Src.getOperand(0).getScalarValueSizeInBits() == EltWidth)
53161     return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0));
53162 
53163   // Fold movmsk(not(x)) -> not(movmsk(x)) to improve folding of movmsk results
53164   // with scalar comparisons.
53165   if (SDValue NotSrc = IsNOT(Src, DAG)) {
53166     SDLoc DL(N);
53167     APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts);
53168     NotSrc = DAG.getBitcast(SrcVT, NotSrc);
53169     return DAG.getNode(ISD::XOR, DL, VT,
53170                        DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc),
53171                        DAG.getConstant(NotMask, DL, VT));
53172   }
53173 
53174   // Fold movmsk(icmp_sgt(x,-1)) -> not(movmsk(x)) to improve folding of movmsk
53175   // results with scalar comparisons.
53176   if (Src.getOpcode() == X86ISD::PCMPGT &&
53177       ISD::isBuildVectorAllOnes(Src.getOperand(1).getNode())) {
53178     SDLoc DL(N);
53179     APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts);
53180     return DAG.getNode(ISD::XOR, DL, VT,
53181                        DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)),
53182                        DAG.getConstant(NotMask, DL, VT));
53183   }
53184 
53185   // Fold movmsk(icmp_eq(and(x,c1),c1)) -> movmsk(shl(x,c2))
53186   // Fold movmsk(icmp_eq(and(x,c1),0)) -> movmsk(not(shl(x,c2)))
53187   // iff pow2splat(c1).
53188   // Use KnownBits to determine if only a single bit is non-zero
53189   // in each element (pow2 or zero), and shift that bit to the msb.
53190   if (Src.getOpcode() == X86ISD::PCMPEQ) {
53191     KnownBits KnownLHS = DAG.computeKnownBits(Src.getOperand(0));
53192     KnownBits KnownRHS = DAG.computeKnownBits(Src.getOperand(1));
53193     unsigned ShiftAmt = KnownLHS.countMinLeadingZeros();
53194     if (KnownLHS.countMaxPopulation() == 1 &&
53195         (KnownRHS.isZero() || (KnownRHS.countMaxPopulation() == 1 &&
53196                                ShiftAmt == KnownRHS.countMinLeadingZeros()))) {
53197       SDLoc DL(N);
53198       MVT ShiftVT = SrcVT;
53199       SDValue ShiftLHS = Src.getOperand(0);
53200       SDValue ShiftRHS = Src.getOperand(1);
53201       if (ShiftVT.getScalarType() == MVT::i8) {
53202         // vXi8 shifts - we only care about the signbit so can use PSLLW.
53203         ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
53204         ShiftLHS = DAG.getBitcast(ShiftVT, ShiftLHS);
53205         ShiftRHS = DAG.getBitcast(ShiftVT, ShiftRHS);
53206       }
53207       ShiftLHS = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, ShiftVT,
53208                                             ShiftLHS, ShiftAmt, DAG);
53209       ShiftRHS = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, ShiftVT,
53210                                             ShiftRHS, ShiftAmt, DAG);
53211       ShiftLHS = DAG.getBitcast(SrcVT, ShiftLHS);
53212       ShiftRHS = DAG.getBitcast(SrcVT, ShiftRHS);
53213       SDValue Res = DAG.getNode(ISD::XOR, DL, SrcVT, ShiftLHS, ShiftRHS);
53214       return DAG.getNode(X86ISD::MOVMSK, DL, VT, DAG.getNOT(DL, Res, SrcVT));
53215     }
53216   }
53217 
53218   // Fold movmsk(logic(X,C)) -> logic(movmsk(X),C)
53219   if (N->isOnlyUserOf(Src.getNode())) {
53220     SDValue SrcBC = peekThroughOneUseBitcasts(Src);
53221     if (ISD::isBitwiseLogicOp(SrcBC.getOpcode())) {
53222       APInt UndefElts;
53223       SmallVector<APInt, 32> EltBits;
53224       if (getTargetConstantBitsFromNode(SrcBC.getOperand(1), NumBitsPerElt,
53225                                         UndefElts, EltBits)) {
53226         APInt Mask = APInt::getZero(NumBits);
53227         for (unsigned Idx = 0; Idx != NumElts; ++Idx) {
53228           if (!UndefElts[Idx] && EltBits[Idx].isNegative())
53229             Mask.setBit(Idx);
53230         }
53231         SDLoc DL(N);
53232         SDValue NewSrc = DAG.getBitcast(SrcVT, SrcBC.getOperand(0));
53233         SDValue NewMovMsk = DAG.getNode(X86ISD::MOVMSK, DL, VT, NewSrc);
53234         return DAG.getNode(SrcBC.getOpcode(), DL, VT, NewMovMsk,
53235                            DAG.getConstant(Mask, DL, VT));
53236       }
53237     }
53238   }
53239 
53240   // Simplify the inputs.
53241   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53242   APInt DemandedMask(APInt::getAllOnes(NumBits));
53243   if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
53244     return SDValue(N, 0);
53245 
53246   return SDValue();
53247 }
53248 
53249 static SDValue combineTESTP(SDNode *N, SelectionDAG &DAG,
53250                             TargetLowering::DAGCombinerInfo &DCI,
53251                             const X86Subtarget &Subtarget) {
53252   MVT VT = N->getSimpleValueType(0);
53253   unsigned NumBits = VT.getScalarSizeInBits();
53254 
53255   // Simplify the inputs.
53256   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53257   APInt DemandedMask(APInt::getAllOnes(NumBits));
53258   if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
53259     return SDValue(N, 0);
53260 
53261   return SDValue();
53262 }
53263 
53264 static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
53265                                        TargetLowering::DAGCombinerInfo &DCI) {
53266   auto *MemOp = cast<X86MaskedGatherScatterSDNode>(N);
53267   SDValue Mask = MemOp->getMask();
53268 
53269   // With vector masks we only demand the upper bit of the mask.
53270   if (Mask.getScalarValueSizeInBits() != 1) {
53271     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53272     APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
53273     if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI)) {
53274       if (N->getOpcode() != ISD::DELETED_NODE)
53275         DCI.AddToWorklist(N);
53276       return SDValue(N, 0);
53277     }
53278   }
53279 
53280   return SDValue();
53281 }
53282 
53283 static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS,
53284                                     SDValue Index, SDValue Base, SDValue Scale,
53285                                     SelectionDAG &DAG) {
53286   SDLoc DL(GorS);
53287 
53288   if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
53289     SDValue Ops[] = { Gather->getChain(), Gather->getPassThru(),
53290                       Gather->getMask(), Base, Index, Scale } ;
53291     return DAG.getMaskedGather(Gather->getVTList(),
53292                                Gather->getMemoryVT(), DL, Ops,
53293                                Gather->getMemOperand(),
53294                                Gather->getIndexType(),
53295                                Gather->getExtensionType());
53296   }
53297   auto *Scatter = cast<MaskedScatterSDNode>(GorS);
53298   SDValue Ops[] = { Scatter->getChain(), Scatter->getValue(),
53299                     Scatter->getMask(), Base, Index, Scale };
53300   return DAG.getMaskedScatter(Scatter->getVTList(),
53301                               Scatter->getMemoryVT(), DL,
53302                               Ops, Scatter->getMemOperand(),
53303                               Scatter->getIndexType(),
53304                               Scatter->isTruncatingStore());
53305 }
53306 
53307 static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
53308                                     TargetLowering::DAGCombinerInfo &DCI) {
53309   SDLoc DL(N);
53310   auto *GorS = cast<MaskedGatherScatterSDNode>(N);
53311   SDValue Index = GorS->getIndex();
53312   SDValue Base = GorS->getBasePtr();
53313   SDValue Scale = GorS->getScale();
53314 
53315   if (DCI.isBeforeLegalize()) {
53316     unsigned IndexWidth = Index.getScalarValueSizeInBits();
53317 
53318     // Shrink constant indices if they are larger than 32-bits.
53319     // Only do this before legalize types since v2i64 could become v2i32.
53320     // FIXME: We could check that the type is legal if we're after legalize
53321     // types, but then we would need to construct test cases where that happens.
53322     // FIXME: We could support more than just constant vectors, but we need to
53323     // careful with costing. A truncate that can be optimized out would be fine.
53324     // Otherwise we might only want to create a truncate if it avoids a split.
53325     if (auto *BV = dyn_cast<BuildVectorSDNode>(Index)) {
53326       if (BV->isConstant() && IndexWidth > 32 &&
53327           DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
53328         EVT NewVT = Index.getValueType().changeVectorElementType(MVT::i32);
53329         Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
53330         return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53331       }
53332     }
53333 
53334     // Shrink any sign/zero extends from 32 or smaller to larger than 32 if
53335     // there are sufficient sign bits. Only do this before legalize types to
53336     // avoid creating illegal types in truncate.
53337     if ((Index.getOpcode() == ISD::SIGN_EXTEND ||
53338          Index.getOpcode() == ISD::ZERO_EXTEND) &&
53339         IndexWidth > 32 &&
53340         Index.getOperand(0).getScalarValueSizeInBits() <= 32 &&
53341         DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
53342       EVT NewVT = Index.getValueType().changeVectorElementType(MVT::i32);
53343       Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
53344       return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53345     }
53346   }
53347 
53348   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53349   EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
53350   // Try to move splat constant adders from the index operand to the base
53351   // pointer operand. Taking care to multiply by the scale. We can only do
53352   // this when index element type is the same as the pointer type.
53353   // Otherwise we need to be sure the math doesn't wrap before the scale.
53354   if (Index.getOpcode() == ISD::ADD &&
53355       Index.getValueType().getVectorElementType() == PtrVT &&
53356       isa<ConstantSDNode>(Scale)) {
53357     uint64_t ScaleAmt = Scale->getAsZExtVal();
53358     if (auto *BV = dyn_cast<BuildVectorSDNode>(Index.getOperand(1))) {
53359       BitVector UndefElts;
53360       if (ConstantSDNode *C = BV->getConstantSplatNode(&UndefElts)) {
53361         // FIXME: Allow non-constant?
53362         if (UndefElts.none()) {
53363           // Apply the scale.
53364           APInt Adder = C->getAPIntValue() * ScaleAmt;
53365           // Add it to the existing base.
53366           Base = DAG.getNode(ISD::ADD, DL, PtrVT, Base,
53367                              DAG.getConstant(Adder, DL, PtrVT));
53368           Index = Index.getOperand(0);
53369           return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53370         }
53371       }
53372 
53373       // It's also possible base is just a constant. In that case, just
53374       // replace it with 0 and move the displacement into the index.
53375       if (BV->isConstant() && isa<ConstantSDNode>(Base) &&
53376           isOneConstant(Scale)) {
53377         SDValue Splat = DAG.getSplatBuildVector(Index.getValueType(), DL, Base);
53378         // Combine the constant build_vector and the constant base.
53379         Splat = DAG.getNode(ISD::ADD, DL, Index.getValueType(),
53380                             Index.getOperand(1), Splat);
53381         // Add to the LHS of the original Index add.
53382         Index = DAG.getNode(ISD::ADD, DL, Index.getValueType(),
53383                             Index.getOperand(0), Splat);
53384         Base = DAG.getConstant(0, DL, Base.getValueType());
53385         return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53386       }
53387     }
53388   }
53389 
53390   if (DCI.isBeforeLegalizeOps()) {
53391     unsigned IndexWidth = Index.getScalarValueSizeInBits();
53392 
53393     // Make sure the index is either i32 or i64
53394     if (IndexWidth != 32 && IndexWidth != 64) {
53395       MVT EltVT = IndexWidth > 32 ? MVT::i64 : MVT::i32;
53396       EVT IndexVT = Index.getValueType().changeVectorElementType(EltVT);
53397       Index = DAG.getSExtOrTrunc(Index, DL, IndexVT);
53398       return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
53399     }
53400   }
53401 
53402   // With vector masks we only demand the upper bit of the mask.
53403   SDValue Mask = GorS->getMask();
53404   if (Mask.getScalarValueSizeInBits() != 1) {
53405     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53406     APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
53407     if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI)) {
53408       if (N->getOpcode() != ISD::DELETED_NODE)
53409         DCI.AddToWorklist(N);
53410       return SDValue(N, 0);
53411     }
53412   }
53413 
53414   return SDValue();
53415 }
53416 
53417 // Optimize  RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
53418 static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG,
53419                                const X86Subtarget &Subtarget) {
53420   SDLoc DL(N);
53421   X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
53422   SDValue EFLAGS = N->getOperand(1);
53423 
53424   // Try to simplify the EFLAGS and condition code operands.
53425   if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget))
53426     return getSETCC(CC, Flags, DL, DAG);
53427 
53428   return SDValue();
53429 }
53430 
53431 /// Optimize branch condition evaluation.
53432 static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
53433                              const X86Subtarget &Subtarget) {
53434   SDLoc DL(N);
53435   SDValue EFLAGS = N->getOperand(3);
53436   X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
53437 
53438   // Try to simplify the EFLAGS and condition code operands.
53439   // Make sure to not keep references to operands, as combineSetCCEFLAGS can
53440   // RAUW them under us.
53441   if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
53442     SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
53443     return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
53444                        N->getOperand(1), Cond, Flags);
53445   }
53446 
53447   return SDValue();
53448 }
53449 
53450 // TODO: Could we move this to DAGCombine?
53451 static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
53452                                                   SelectionDAG &DAG) {
53453   // Take advantage of vector comparisons (etc.) producing 0 or -1 in each lane
53454   // to optimize away operation when it's from a constant.
53455   //
53456   // The general transformation is:
53457   //    UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
53458   //       AND(VECTOR_CMP(x,y), constant2)
53459   //    constant2 = UNARYOP(constant)
53460 
53461   // Early exit if this isn't a vector operation, the operand of the
53462   // unary operation isn't a bitwise AND, or if the sizes of the operations
53463   // aren't the same.
53464   EVT VT = N->getValueType(0);
53465   bool IsStrict = N->isStrictFPOpcode();
53466   unsigned NumEltBits = VT.getScalarSizeInBits();
53467   SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
53468   if (!VT.isVector() || Op0.getOpcode() != ISD::AND ||
53469       DAG.ComputeNumSignBits(Op0.getOperand(0)) != NumEltBits ||
53470       VT.getSizeInBits() != Op0.getValueSizeInBits())
53471     return SDValue();
53472 
53473   // Now check that the other operand of the AND is a constant. We could
53474   // make the transformation for non-constant splats as well, but it's unclear
53475   // that would be a benefit as it would not eliminate any operations, just
53476   // perform one more step in scalar code before moving to the vector unit.
53477   if (auto *BV = dyn_cast<BuildVectorSDNode>(Op0.getOperand(1))) {
53478     // Bail out if the vector isn't a constant.
53479     if (!BV->isConstant())
53480       return SDValue();
53481 
53482     // Everything checks out. Build up the new and improved node.
53483     SDLoc DL(N);
53484     EVT IntVT = BV->getValueType(0);
53485     // Create a new constant of the appropriate type for the transformed
53486     // DAG.
53487     SDValue SourceConst;
53488     if (IsStrict)
53489       SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other},
53490                                 {N->getOperand(0), SDValue(BV, 0)});
53491     else
53492       SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
53493     // The AND node needs bitcasts to/from an integer vector type around it.
53494     SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
53495     SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
53496                                  MaskConst);
53497     SDValue Res = DAG.getBitcast(VT, NewAnd);
53498     if (IsStrict)
53499       return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL);
53500     return Res;
53501   }
53502 
53503   return SDValue();
53504 }
53505 
53506 /// If we are converting a value to floating-point, try to replace scalar
53507 /// truncate of an extracted vector element with a bitcast. This tries to keep
53508 /// the sequence on XMM registers rather than moving between vector and GPRs.
53509 static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) {
53510   // TODO: This is currently only used by combineSIntToFP, but it is generalized
53511   //       to allow being called by any similar cast opcode.
53512   // TODO: Consider merging this into lowering: vectorizeExtractedCast().
53513   SDValue Trunc = N->getOperand(0);
53514   if (!Trunc.hasOneUse() || Trunc.getOpcode() != ISD::TRUNCATE)
53515     return SDValue();
53516 
53517   SDValue ExtElt = Trunc.getOperand(0);
53518   if (!ExtElt.hasOneUse() || ExtElt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
53519       !isNullConstant(ExtElt.getOperand(1)))
53520     return SDValue();
53521 
53522   EVT TruncVT = Trunc.getValueType();
53523   EVT SrcVT = ExtElt.getValueType();
53524   unsigned DestWidth = TruncVT.getSizeInBits();
53525   unsigned SrcWidth = SrcVT.getSizeInBits();
53526   if (SrcWidth % DestWidth != 0)
53527     return SDValue();
53528 
53529   // inttofp (trunc (extelt X, 0)) --> inttofp (extelt (bitcast X), 0)
53530   EVT SrcVecVT = ExtElt.getOperand(0).getValueType();
53531   unsigned VecWidth = SrcVecVT.getSizeInBits();
53532   unsigned NumElts = VecWidth / DestWidth;
53533   EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), TruncVT, NumElts);
53534   SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0));
53535   SDLoc DL(N);
53536   SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
53537                                   BitcastVec, ExtElt.getOperand(1));
53538   return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt);
53539 }
53540 
53541 static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG,
53542                                const X86Subtarget &Subtarget) {
53543   bool IsStrict = N->isStrictFPOpcode();
53544   SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
53545   EVT VT = N->getValueType(0);
53546   EVT InVT = Op0.getValueType();
53547 
53548   // Using i16 as an intermediate type is a bad idea, unless we have HW support
53549   // for it. Therefore for type sizes equal or smaller than 32 just go with i32.
53550   // if hasFP16 support:
53551   //   UINT_TO_FP(vXi1~15)  -> SINT_TO_FP(ZEXT(vXi1~15  to vXi16))
53552   //   UINT_TO_FP(vXi17~31) -> SINT_TO_FP(ZEXT(vXi17~31 to vXi32))
53553   // else
53554   //   UINT_TO_FP(vXi1~31) -> SINT_TO_FP(ZEXT(vXi1~31 to vXi32))
53555   // UINT_TO_FP(vXi33~63) -> SINT_TO_FP(ZEXT(vXi33~63 to vXi64))
53556   if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) {
53557     unsigned ScalarSize = InVT.getScalarSizeInBits();
53558     if ((ScalarSize == 16 && Subtarget.hasFP16()) || ScalarSize == 32 ||
53559         ScalarSize >= 64)
53560       return SDValue();
53561     SDLoc dl(N);
53562     EVT DstVT =
53563         EVT::getVectorVT(*DAG.getContext(),
53564                          (Subtarget.hasFP16() && ScalarSize < 16) ? MVT::i16
53565                          : ScalarSize < 32                        ? MVT::i32
53566                                                                   : MVT::i64,
53567                          InVT.getVectorNumElements());
53568     SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
53569     if (IsStrict)
53570       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53571                          {N->getOperand(0), P});
53572     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
53573   }
53574 
53575   // UINT_TO_FP(vXi1) -> SINT_TO_FP(ZEXT(vXi1 to vXi32))
53576   // UINT_TO_FP(vXi8) -> SINT_TO_FP(ZEXT(vXi8 to vXi32))
53577   // UINT_TO_FP(vXi16) -> SINT_TO_FP(ZEXT(vXi16 to vXi32))
53578   if (InVT.isVector() && InVT.getScalarSizeInBits() < 32 &&
53579       VT.getScalarType() != MVT::f16) {
53580     SDLoc dl(N);
53581     EVT DstVT = InVT.changeVectorElementType(MVT::i32);
53582     SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
53583 
53584     // UINT_TO_FP isn't legal without AVX512 so use SINT_TO_FP.
53585     if (IsStrict)
53586       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53587                          {N->getOperand(0), P});
53588     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
53589   }
53590 
53591   // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't
53592   // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
53593   // the optimization here.
53594   if (DAG.SignBitIsZero(Op0)) {
53595     if (IsStrict)
53596       return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other},
53597                          {N->getOperand(0), Op0});
53598     return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);
53599   }
53600 
53601   return SDValue();
53602 }
53603 
53604 static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
53605                                TargetLowering::DAGCombinerInfo &DCI,
53606                                const X86Subtarget &Subtarget) {
53607   // First try to optimize away the conversion entirely when it's
53608   // conditionally from a constant. Vectors only.
53609   bool IsStrict = N->isStrictFPOpcode();
53610   if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG))
53611     return Res;
53612 
53613   // Now move on to more general possibilities.
53614   SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
53615   EVT VT = N->getValueType(0);
53616   EVT InVT = Op0.getValueType();
53617 
53618   // Using i16 as an intermediate type is a bad idea, unless we have HW support
53619   // for it. Therefore for type sizes equal or smaller than 32 just go with i32.
53620   // if hasFP16 support:
53621   //   SINT_TO_FP(vXi1~15)  -> SINT_TO_FP(SEXT(vXi1~15  to vXi16))
53622   //   SINT_TO_FP(vXi17~31) -> SINT_TO_FP(SEXT(vXi17~31 to vXi32))
53623   // else
53624   //   SINT_TO_FP(vXi1~31) -> SINT_TO_FP(ZEXT(vXi1~31 to vXi32))
53625   // SINT_TO_FP(vXi33~63) -> SINT_TO_FP(SEXT(vXi33~63 to vXi64))
53626   if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) {
53627     unsigned ScalarSize = InVT.getScalarSizeInBits();
53628     if ((ScalarSize == 16 && Subtarget.hasFP16()) || ScalarSize == 32 ||
53629         ScalarSize >= 64)
53630       return SDValue();
53631     SDLoc dl(N);
53632     EVT DstVT =
53633         EVT::getVectorVT(*DAG.getContext(),
53634                          (Subtarget.hasFP16() && ScalarSize < 16) ? MVT::i16
53635                          : ScalarSize < 32                        ? MVT::i32
53636                                                                   : MVT::i64,
53637                          InVT.getVectorNumElements());
53638     SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
53639     if (IsStrict)
53640       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53641                          {N->getOperand(0), P});
53642     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
53643   }
53644 
53645   // SINT_TO_FP(vXi1) -> SINT_TO_FP(SEXT(vXi1 to vXi32))
53646   // SINT_TO_FP(vXi8) -> SINT_TO_FP(SEXT(vXi8 to vXi32))
53647   // SINT_TO_FP(vXi16) -> SINT_TO_FP(SEXT(vXi16 to vXi32))
53648   if (InVT.isVector() && InVT.getScalarSizeInBits() < 32 &&
53649       VT.getScalarType() != MVT::f16) {
53650     SDLoc dl(N);
53651     EVT DstVT = InVT.changeVectorElementType(MVT::i32);
53652     SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
53653     if (IsStrict)
53654       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53655                          {N->getOperand(0), P});
53656     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
53657   }
53658 
53659   // Without AVX512DQ we only support i64 to float scalar conversion. For both
53660   // vectors and scalars, see if we know that the upper bits are all the sign
53661   // bit, in which case we can truncate the input to i32 and convert from that.
53662   if (InVT.getScalarSizeInBits() > 32 && !Subtarget.hasDQI()) {
53663     unsigned BitWidth = InVT.getScalarSizeInBits();
53664     unsigned NumSignBits = DAG.ComputeNumSignBits(Op0);
53665     if (NumSignBits >= (BitWidth - 31)) {
53666       EVT TruncVT = MVT::i32;
53667       if (InVT.isVector())
53668         TruncVT = InVT.changeVectorElementType(TruncVT);
53669       SDLoc dl(N);
53670       if (DCI.isBeforeLegalize() || TruncVT != MVT::v2i32) {
53671         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0);
53672         if (IsStrict)
53673           return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
53674                              {N->getOperand(0), Trunc});
53675         return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc);
53676       }
53677       // If we're after legalize and the type is v2i32 we need to shuffle and
53678       // use CVTSI2P.
53679       assert(InVT == MVT::v2i64 && "Unexpected VT!");
53680       SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
53681       SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
53682                                           { 0, 2, -1, -1 });
53683       if (IsStrict)
53684         return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
53685                            {N->getOperand(0), Shuf});
53686       return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf);
53687     }
53688   }
53689 
53690   // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have
53691   // a 32-bit target where SSE doesn't support i64->FP operations.
53692   if (!Subtarget.useSoftFloat() && Subtarget.hasX87() &&
53693       Op0.getOpcode() == ISD::LOAD) {
53694     LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode());
53695 
53696     // This transformation is not supported if the result type is f16 or f128.
53697     if (VT == MVT::f16 || VT == MVT::f128)
53698       return SDValue();
53699 
53700     // If we have AVX512DQ we can use packed conversion instructions unless
53701     // the VT is f80.
53702     if (Subtarget.hasDQI() && VT != MVT::f80)
53703       return SDValue();
53704 
53705     if (Ld->isSimple() && !VT.isVector() && ISD::isNormalLoad(Op0.getNode()) &&
53706         Op0.hasOneUse() && !Subtarget.is64Bit() && InVT == MVT::i64) {
53707       std::pair<SDValue, SDValue> Tmp =
53708           Subtarget.getTargetLowering()->BuildFILD(
53709               VT, InVT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(),
53710               Ld->getPointerInfo(), Ld->getOriginalAlign(), DAG);
53711       DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Tmp.second);
53712       return Tmp.first;
53713     }
53714   }
53715 
53716   if (IsStrict)
53717     return SDValue();
53718 
53719   if (SDValue V = combineToFPTruncExtElt(N, DAG))
53720     return V;
53721 
53722   return SDValue();
53723 }
53724 
53725 static bool needCarryOrOverflowFlag(SDValue Flags) {
53726   assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!");
53727 
53728   for (const SDNode *User : Flags->uses()) {
53729     X86::CondCode CC;
53730     switch (User->getOpcode()) {
53731     default:
53732       // Be conservative.
53733       return true;
53734     case X86ISD::SETCC:
53735     case X86ISD::SETCC_CARRY:
53736       CC = (X86::CondCode)User->getConstantOperandVal(0);
53737       break;
53738     case X86ISD::BRCOND:
53739     case X86ISD::CMOV:
53740       CC = (X86::CondCode)User->getConstantOperandVal(2);
53741       break;
53742     }
53743 
53744     switch (CC) {
53745     default: break;
53746     case X86::COND_A: case X86::COND_AE:
53747     case X86::COND_B: case X86::COND_BE:
53748     case X86::COND_O: case X86::COND_NO:
53749     case X86::COND_G: case X86::COND_GE:
53750     case X86::COND_L: case X86::COND_LE:
53751       return true;
53752     }
53753   }
53754 
53755   return false;
53756 }
53757 
53758 static bool onlyZeroFlagUsed(SDValue Flags) {
53759   assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!");
53760 
53761   for (const SDNode *User : Flags->uses()) {
53762     unsigned CCOpNo;
53763     switch (User->getOpcode()) {
53764     default:
53765       // Be conservative.
53766       return false;
53767     case X86ISD::SETCC:
53768     case X86ISD::SETCC_CARRY:
53769       CCOpNo = 0;
53770       break;
53771     case X86ISD::BRCOND:
53772     case X86ISD::CMOV:
53773       CCOpNo = 2;
53774       break;
53775     }
53776 
53777     X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(CCOpNo);
53778     if (CC != X86::COND_E && CC != X86::COND_NE)
53779       return false;
53780   }
53781 
53782   return true;
53783 }
53784 
53785 static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
53786                           const X86Subtarget &Subtarget) {
53787   // Only handle test patterns.
53788   if (!isNullConstant(N->getOperand(1)))
53789     return SDValue();
53790 
53791   // If we have a CMP of a truncated binop, see if we can make a smaller binop
53792   // and use its flags directly.
53793   // TODO: Maybe we should try promoting compares that only use the zero flag
53794   // first if we can prove the upper bits with computeKnownBits?
53795   SDLoc dl(N);
53796   SDValue Op = N->getOperand(0);
53797   EVT VT = Op.getValueType();
53798   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53799 
53800   // If we have a constant logical shift that's only used in a comparison
53801   // against zero turn it into an equivalent AND. This allows turning it into
53802   // a TEST instruction later.
53803   if ((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) &&
53804       Op.hasOneUse() && isa<ConstantSDNode>(Op.getOperand(1)) &&
53805       onlyZeroFlagUsed(SDValue(N, 0))) {
53806     unsigned BitWidth = VT.getSizeInBits();
53807     const APInt &ShAmt = Op.getConstantOperandAPInt(1);
53808     if (ShAmt.ult(BitWidth)) { // Avoid undefined shifts.
53809       unsigned MaskBits = BitWidth - ShAmt.getZExtValue();
53810       APInt Mask = Op.getOpcode() == ISD::SRL
53811                        ? APInt::getHighBitsSet(BitWidth, MaskBits)
53812                        : APInt::getLowBitsSet(BitWidth, MaskBits);
53813       if (Mask.isSignedIntN(32)) {
53814         Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0),
53815                          DAG.getConstant(Mask, dl, VT));
53816         return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53817                            DAG.getConstant(0, dl, VT));
53818       }
53819     }
53820   }
53821 
53822   // If we're extracting from a avx512 bool vector and comparing against zero,
53823   // then try to just bitcast the vector to an integer to use TEST/BT directly.
53824   // (and (extract_elt (kshiftr vXi1, C), 0), 1) -> (and (bc vXi1), 1<<C)
53825   if (Op.getOpcode() == ISD::AND && isOneConstant(Op.getOperand(1)) &&
53826       Op.hasOneUse() && onlyZeroFlagUsed(SDValue(N, 0))) {
53827     SDValue Src = Op.getOperand(0);
53828     if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
53829         isNullConstant(Src.getOperand(1)) &&
53830         Src.getOperand(0).getValueType().getScalarType() == MVT::i1) {
53831       SDValue BoolVec = Src.getOperand(0);
53832       unsigned ShAmt = 0;
53833       if (BoolVec.getOpcode() == X86ISD::KSHIFTR) {
53834         ShAmt = BoolVec.getConstantOperandVal(1);
53835         BoolVec = BoolVec.getOperand(0);
53836       }
53837       BoolVec = widenMaskVector(BoolVec, false, Subtarget, DAG, dl);
53838       EVT VecVT = BoolVec.getValueType();
53839       unsigned BitWidth = VecVT.getVectorNumElements();
53840       EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), BitWidth);
53841       if (TLI.isTypeLegal(VecVT) && TLI.isTypeLegal(BCVT)) {
53842         APInt Mask = APInt::getOneBitSet(BitWidth, ShAmt);
53843         Op = DAG.getBitcast(BCVT, BoolVec);
53844         Op = DAG.getNode(ISD::AND, dl, BCVT, Op,
53845                          DAG.getConstant(Mask, dl, BCVT));
53846         return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53847                            DAG.getConstant(0, dl, BCVT));
53848       }
53849     }
53850   }
53851 
53852   // Peek through any zero-extend if we're only testing for a zero result.
53853   if (Op.getOpcode() == ISD::ZERO_EXTEND && onlyZeroFlagUsed(SDValue(N, 0))) {
53854     SDValue Src = Op.getOperand(0);
53855     EVT SrcVT = Src.getValueType();
53856     if (SrcVT.getScalarSizeInBits() >= 8 && TLI.isTypeLegal(SrcVT))
53857       return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Src,
53858                          DAG.getConstant(0, dl, SrcVT));
53859   }
53860 
53861   // Look for a truncate.
53862   if (Op.getOpcode() != ISD::TRUNCATE)
53863     return SDValue();
53864 
53865   SDValue Trunc = Op;
53866   Op = Op.getOperand(0);
53867 
53868   // See if we can compare with zero against the truncation source,
53869   // which should help using the Z flag from many ops. Only do this for
53870   // i32 truncated op to prevent partial-reg compares of promoted ops.
53871   EVT OpVT = Op.getValueType();
53872   APInt UpperBits =
53873       APInt::getBitsSetFrom(OpVT.getSizeInBits(), VT.getSizeInBits());
53874   if (OpVT == MVT::i32 && DAG.MaskedValueIsZero(Op, UpperBits) &&
53875       onlyZeroFlagUsed(SDValue(N, 0))) {
53876     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53877                        DAG.getConstant(0, dl, OpVT));
53878   }
53879 
53880   // After this the truncate and arithmetic op must have a single use.
53881   if (!Trunc.hasOneUse() || !Op.hasOneUse())
53882       return SDValue();
53883 
53884   unsigned NewOpc;
53885   switch (Op.getOpcode()) {
53886   default: return SDValue();
53887   case ISD::AND:
53888     // Skip and with constant. We have special handling for and with immediate
53889     // during isel to generate test instructions.
53890     if (isa<ConstantSDNode>(Op.getOperand(1)))
53891       return SDValue();
53892     NewOpc = X86ISD::AND;
53893     break;
53894   case ISD::OR:  NewOpc = X86ISD::OR;  break;
53895   case ISD::XOR: NewOpc = X86ISD::XOR; break;
53896   case ISD::ADD:
53897     // If the carry or overflow flag is used, we can't truncate.
53898     if (needCarryOrOverflowFlag(SDValue(N, 0)))
53899       return SDValue();
53900     NewOpc = X86ISD::ADD;
53901     break;
53902   case ISD::SUB:
53903     // If the carry or overflow flag is used, we can't truncate.
53904     if (needCarryOrOverflowFlag(SDValue(N, 0)))
53905       return SDValue();
53906     NewOpc = X86ISD::SUB;
53907     break;
53908   }
53909 
53910   // We found an op we can narrow. Truncate its inputs.
53911   SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));
53912   SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));
53913 
53914   // Use a X86 specific opcode to avoid DAG combine messing with it.
53915   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
53916   Op = DAG.getNode(NewOpc, dl, VTs, Op0, Op1);
53917 
53918   // For AND, keep a CMP so that we can match the test pattern.
53919   if (NewOpc == X86ISD::AND)
53920     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
53921                        DAG.getConstant(0, dl, VT));
53922 
53923   // Return the flags.
53924   return Op.getValue(1);
53925 }
53926 
53927 static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
53928                                 TargetLowering::DAGCombinerInfo &DCI) {
53929   assert((X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode()) &&
53930          "Expected X86ISD::ADD or X86ISD::SUB");
53931 
53932   SDLoc DL(N);
53933   SDValue LHS = N->getOperand(0);
53934   SDValue RHS = N->getOperand(1);
53935   MVT VT = LHS.getSimpleValueType();
53936   bool IsSub = X86ISD::SUB == N->getOpcode();
53937   unsigned GenericOpc = IsSub ? ISD::SUB : ISD::ADD;
53938 
53939   // If we don't use the flag result, simplify back to a generic ADD/SUB.
53940   if (!N->hasAnyUseOfValue(1)) {
53941     SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
53942     return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL);
53943   }
53944 
53945   // Fold any similar generic ADD/SUB opcodes to reuse this node.
53946   auto MatchGeneric = [&](SDValue N0, SDValue N1, bool Negate) {
53947     SDValue Ops[] = {N0, N1};
53948     SDVTList VTs = DAG.getVTList(N->getValueType(0));
53949     if (SDNode *GenericAddSub = DAG.getNodeIfExists(GenericOpc, VTs, Ops)) {
53950       SDValue Op(N, 0);
53951       if (Negate)
53952         Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
53953       DCI.CombineTo(GenericAddSub, Op);
53954     }
53955   };
53956   MatchGeneric(LHS, RHS, false);
53957   MatchGeneric(RHS, LHS, X86ISD::SUB == N->getOpcode());
53958 
53959   // TODO: Can we drop the ZeroSecondOpOnly limit? This is to guarantee that the
53960   // EFLAGS result doesn't change.
53961   return combineAddOrSubToADCOrSBB(IsSub, DL, VT, LHS, RHS, DAG,
53962                                    /*ZeroSecondOpOnly*/ true);
53963 }
53964 
53965 static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
53966   SDValue LHS = N->getOperand(0);
53967   SDValue RHS = N->getOperand(1);
53968   SDValue BorrowIn = N->getOperand(2);
53969 
53970   if (SDValue Flags = combineCarryThroughADD(BorrowIn, DAG)) {
53971     MVT VT = N->getSimpleValueType(0);
53972     SDVTList VTs = DAG.getVTList(VT, MVT::i32);
53973     return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs, LHS, RHS, Flags);
53974   }
53975 
53976   // Fold SBB(SUB(X,Y),0,Carry) -> SBB(X,Y,Carry)
53977   // iff the flag result is dead.
53978   if (LHS.getOpcode() == ISD::SUB && isNullConstant(RHS) &&
53979       !N->hasAnyUseOfValue(1))
53980     return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), LHS.getOperand(0),
53981                        LHS.getOperand(1), BorrowIn);
53982 
53983   return SDValue();
53984 }
53985 
53986 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
53987 static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
53988                           TargetLowering::DAGCombinerInfo &DCI) {
53989   SDValue LHS = N->getOperand(0);
53990   SDValue RHS = N->getOperand(1);
53991   SDValue CarryIn = N->getOperand(2);
53992   auto *LHSC = dyn_cast<ConstantSDNode>(LHS);
53993   auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
53994 
53995   // Canonicalize constant to RHS.
53996   if (LHSC && !RHSC)
53997     return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), RHS, LHS,
53998                        CarryIn);
53999 
54000   // If the LHS and RHS of the ADC node are zero, then it can't overflow and
54001   // the result is either zero or one (depending on the input carry bit).
54002   // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1.
54003   if (LHSC && RHSC && LHSC->isZero() && RHSC->isZero() &&
54004       // We don't have a good way to replace an EFLAGS use, so only do this when
54005       // dead right now.
54006       SDValue(N, 1).use_empty()) {
54007     SDLoc DL(N);
54008     EVT VT = N->getValueType(0);
54009     SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
54010     SDValue Res1 = DAG.getNode(
54011         ISD::AND, DL, VT,
54012         DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
54013                     DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), CarryIn),
54014         DAG.getConstant(1, DL, VT));
54015     return DCI.CombineTo(N, Res1, CarryOut);
54016   }
54017 
54018   // Fold ADC(C1,C2,Carry) -> ADC(0,C1+C2,Carry)
54019   // iff the flag result is dead.
54020   // TODO: Allow flag result if C1+C2 doesn't signed/unsigned overflow.
54021   if (LHSC && RHSC && !LHSC->isZero() && !N->hasAnyUseOfValue(1)) {
54022     SDLoc DL(N);
54023     APInt Sum = LHSC->getAPIntValue() + RHSC->getAPIntValue();
54024     return DAG.getNode(X86ISD::ADC, DL, N->getVTList(),
54025                        DAG.getConstant(0, DL, LHS.getValueType()),
54026                        DAG.getConstant(Sum, DL, LHS.getValueType()), CarryIn);
54027   }
54028 
54029   if (SDValue Flags = combineCarryThroughADD(CarryIn, DAG)) {
54030     MVT VT = N->getSimpleValueType(0);
54031     SDVTList VTs = DAG.getVTList(VT, MVT::i32);
54032     return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs, LHS, RHS, Flags);
54033   }
54034 
54035   // Fold ADC(ADD(X,Y),0,Carry) -> ADC(X,Y,Carry)
54036   // iff the flag result is dead.
54037   if (LHS.getOpcode() == ISD::ADD && RHSC && RHSC->isZero() &&
54038       !N->hasAnyUseOfValue(1))
54039     return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), LHS.getOperand(0),
54040                        LHS.getOperand(1), CarryIn);
54041 
54042   return SDValue();
54043 }
54044 
54045 static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
54046                             const SDLoc &DL, EVT VT,
54047                             const X86Subtarget &Subtarget) {
54048   // Example of pattern we try to detect:
54049   // t := (v8i32 mul (sext (v8i16 x0), (sext (v8i16 x1))))
54050   //(add (build_vector (extract_elt t, 0),
54051   //                   (extract_elt t, 2),
54052   //                   (extract_elt t, 4),
54053   //                   (extract_elt t, 6)),
54054   //     (build_vector (extract_elt t, 1),
54055   //                   (extract_elt t, 3),
54056   //                   (extract_elt t, 5),
54057   //                   (extract_elt t, 7)))
54058 
54059   if (!Subtarget.hasSSE2())
54060     return SDValue();
54061 
54062   if (Op0.getOpcode() != ISD::BUILD_VECTOR ||
54063       Op1.getOpcode() != ISD::BUILD_VECTOR)
54064     return SDValue();
54065 
54066   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
54067       VT.getVectorNumElements() < 4 ||
54068       !isPowerOf2_32(VT.getVectorNumElements()))
54069     return SDValue();
54070 
54071   // Check if one of Op0,Op1 is of the form:
54072   // (build_vector (extract_elt Mul, 0),
54073   //               (extract_elt Mul, 2),
54074   //               (extract_elt Mul, 4),
54075   //                   ...
54076   // the other is of the form:
54077   // (build_vector (extract_elt Mul, 1),
54078   //               (extract_elt Mul, 3),
54079   //               (extract_elt Mul, 5),
54080   //                   ...
54081   // and identify Mul.
54082   SDValue Mul;
54083   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; i += 2) {
54084     SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i),
54085             Op0H = Op0->getOperand(i + 1), Op1H = Op1->getOperand(i + 1);
54086     // TODO: Be more tolerant to undefs.
54087     if (Op0L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54088         Op1L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54089         Op0H.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54090         Op1H.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
54091       return SDValue();
54092     auto *Const0L = dyn_cast<ConstantSDNode>(Op0L->getOperand(1));
54093     auto *Const1L = dyn_cast<ConstantSDNode>(Op1L->getOperand(1));
54094     auto *Const0H = dyn_cast<ConstantSDNode>(Op0H->getOperand(1));
54095     auto *Const1H = dyn_cast<ConstantSDNode>(Op1H->getOperand(1));
54096     if (!Const0L || !Const1L || !Const0H || !Const1H)
54097       return SDValue();
54098     unsigned Idx0L = Const0L->getZExtValue(), Idx1L = Const1L->getZExtValue(),
54099              Idx0H = Const0H->getZExtValue(), Idx1H = Const1H->getZExtValue();
54100     // Commutativity of mul allows factors of a product to reorder.
54101     if (Idx0L > Idx1L)
54102       std::swap(Idx0L, Idx1L);
54103     if (Idx0H > Idx1H)
54104       std::swap(Idx0H, Idx1H);
54105     // Commutativity of add allows pairs of factors to reorder.
54106     if (Idx0L > Idx0H) {
54107       std::swap(Idx0L, Idx0H);
54108       std::swap(Idx1L, Idx1H);
54109     }
54110     if (Idx0L != 2 * i || Idx1L != 2 * i + 1 || Idx0H != 2 * i + 2 ||
54111         Idx1H != 2 * i + 3)
54112       return SDValue();
54113     if (!Mul) {
54114       // First time an extract_elt's source vector is visited. Must be a MUL
54115       // with 2X number of vector elements than the BUILD_VECTOR.
54116       // Both extracts must be from same MUL.
54117       Mul = Op0L->getOperand(0);
54118       if (Mul->getOpcode() != ISD::MUL ||
54119           Mul.getValueType().getVectorNumElements() != 2 * e)
54120         return SDValue();
54121     }
54122     // Check that the extract is from the same MUL previously seen.
54123     if (Mul != Op0L->getOperand(0) || Mul != Op1L->getOperand(0) ||
54124         Mul != Op0H->getOperand(0) || Mul != Op1H->getOperand(0))
54125       return SDValue();
54126   }
54127 
54128   // Check if the Mul source can be safely shrunk.
54129   ShrinkMode Mode;
54130   if (!canReduceVMulWidth(Mul.getNode(), DAG, Mode) ||
54131       Mode == ShrinkMode::MULU16)
54132     return SDValue();
54133 
54134   EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
54135                                  VT.getVectorNumElements() * 2);
54136   SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0));
54137   SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1));
54138 
54139   auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
54140                          ArrayRef<SDValue> Ops) {
54141     EVT InVT = Ops[0].getValueType();
54142     assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
54143     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
54144                                  InVT.getVectorNumElements() / 2);
54145     return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
54146   };
54147   return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder);
54148 }
54149 
54150 // Attempt to turn this pattern into PMADDWD.
54151 // (add (mul (sext (build_vector)), (sext (build_vector))),
54152 //      (mul (sext (build_vector)), (sext (build_vector)))
54153 static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
54154                               const SDLoc &DL, EVT VT,
54155                               const X86Subtarget &Subtarget) {
54156   if (!Subtarget.hasSSE2())
54157     return SDValue();
54158 
54159   if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
54160     return SDValue();
54161 
54162   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
54163       VT.getVectorNumElements() < 4 ||
54164       !isPowerOf2_32(VT.getVectorNumElements()))
54165     return SDValue();
54166 
54167   SDValue N00 = N0.getOperand(0);
54168   SDValue N01 = N0.getOperand(1);
54169   SDValue N10 = N1.getOperand(0);
54170   SDValue N11 = N1.getOperand(1);
54171 
54172   // All inputs need to be sign extends.
54173   // TODO: Support ZERO_EXTEND from known positive?
54174   if (N00.getOpcode() != ISD::SIGN_EXTEND ||
54175       N01.getOpcode() != ISD::SIGN_EXTEND ||
54176       N10.getOpcode() != ISD::SIGN_EXTEND ||
54177       N11.getOpcode() != ISD::SIGN_EXTEND)
54178     return SDValue();
54179 
54180   // Peek through the extends.
54181   N00 = N00.getOperand(0);
54182   N01 = N01.getOperand(0);
54183   N10 = N10.getOperand(0);
54184   N11 = N11.getOperand(0);
54185 
54186   // Must be extending from vXi16.
54187   EVT InVT = N00.getValueType();
54188   if (InVT.getVectorElementType() != MVT::i16 || N01.getValueType() != InVT ||
54189       N10.getValueType() != InVT || N11.getValueType() != InVT)
54190     return SDValue();
54191 
54192   // All inputs should be build_vectors.
54193   if (N00.getOpcode() != ISD::BUILD_VECTOR ||
54194       N01.getOpcode() != ISD::BUILD_VECTOR ||
54195       N10.getOpcode() != ISD::BUILD_VECTOR ||
54196       N11.getOpcode() != ISD::BUILD_VECTOR)
54197     return SDValue();
54198 
54199   // For each element, we need to ensure we have an odd element from one vector
54200   // multiplied by the odd element of another vector and the even element from
54201   // one of the same vectors being multiplied by the even element from the
54202   // other vector. So we need to make sure for each element i, this operator
54203   // is being performed:
54204   //  A[2 * i] * B[2 * i] + A[2 * i + 1] * B[2 * i + 1]
54205   SDValue In0, In1;
54206   for (unsigned i = 0; i != N00.getNumOperands(); ++i) {
54207     SDValue N00Elt = N00.getOperand(i);
54208     SDValue N01Elt = N01.getOperand(i);
54209     SDValue N10Elt = N10.getOperand(i);
54210     SDValue N11Elt = N11.getOperand(i);
54211     // TODO: Be more tolerant to undefs.
54212     if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54213         N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54214         N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54215         N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
54216       return SDValue();
54217     auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
54218     auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
54219     auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
54220     auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
54221     if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
54222       return SDValue();
54223     unsigned IdxN00 = ConstN00Elt->getZExtValue();
54224     unsigned IdxN01 = ConstN01Elt->getZExtValue();
54225     unsigned IdxN10 = ConstN10Elt->getZExtValue();
54226     unsigned IdxN11 = ConstN11Elt->getZExtValue();
54227     // Add is commutative so indices can be reordered.
54228     if (IdxN00 > IdxN10) {
54229       std::swap(IdxN00, IdxN10);
54230       std::swap(IdxN01, IdxN11);
54231     }
54232     // N0 indices be the even element. N1 indices must be the next odd element.
54233     if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
54234         IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
54235       return SDValue();
54236     SDValue N00In = N00Elt.getOperand(0);
54237     SDValue N01In = N01Elt.getOperand(0);
54238     SDValue N10In = N10Elt.getOperand(0);
54239     SDValue N11In = N11Elt.getOperand(0);
54240 
54241     // First time we find an input capture it.
54242     if (!In0) {
54243       In0 = N00In;
54244       In1 = N01In;
54245 
54246       // The input vectors must be at least as wide as the output.
54247       // If they are larger than the output, we extract subvector below.
54248       if (In0.getValueSizeInBits() < VT.getSizeInBits() ||
54249           In1.getValueSizeInBits() < VT.getSizeInBits())
54250         return SDValue();
54251     }
54252     // Mul is commutative so the input vectors can be in any order.
54253     // Canonicalize to make the compares easier.
54254     if (In0 != N00In)
54255       std::swap(N00In, N01In);
54256     if (In0 != N10In)
54257       std::swap(N10In, N11In);
54258     if (In0 != N00In || In1 != N01In || In0 != N10In || In1 != N11In)
54259       return SDValue();
54260   }
54261 
54262   auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
54263                          ArrayRef<SDValue> Ops) {
54264     EVT OpVT = Ops[0].getValueType();
54265     assert(OpVT.getScalarType() == MVT::i16 &&
54266            "Unexpected scalar element type");
54267     assert(OpVT == Ops[1].getValueType() && "Operands' types mismatch");
54268     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
54269                                  OpVT.getVectorNumElements() / 2);
54270     return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
54271   };
54272 
54273   // If the output is narrower than an input, extract the low part of the input
54274   // vector.
54275   EVT OutVT16 = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
54276                                VT.getVectorNumElements() * 2);
54277   if (OutVT16.bitsLT(In0.getValueType())) {
54278     In0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In0,
54279                       DAG.getIntPtrConstant(0, DL));
54280   }
54281   if (OutVT16.bitsLT(In1.getValueType())) {
54282     In1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In1,
54283                       DAG.getIntPtrConstant(0, DL));
54284   }
54285   return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },
54286                           PMADDBuilder);
54287 }
54288 
54289 // ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
54290 // If upper element in each pair of both VPMADDWD are zero then we can merge
54291 // the operand elements and use the implicit add of VPMADDWD.
54292 // TODO: Add support for VPMADDUBSW (which isn't commutable).
54293 static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
54294                                    const SDLoc &DL, EVT VT) {
54295   if (N0.getOpcode() != N1.getOpcode() || N0.getOpcode() != X86ISD::VPMADDWD)
54296     return SDValue();
54297 
54298   // TODO: Add 256/512-bit support once VPMADDWD combines with shuffles.
54299   if (VT.getSizeInBits() > 128)
54300     return SDValue();
54301 
54302   unsigned NumElts = VT.getVectorNumElements();
54303   MVT OpVT = N0.getOperand(0).getSimpleValueType();
54304   APInt DemandedBits = APInt::getAllOnes(OpVT.getScalarSizeInBits());
54305   APInt DemandedHiElts = APInt::getSplat(2 * NumElts, APInt(2, 2));
54306 
54307   bool Op0HiZero =
54308       DAG.MaskedValueIsZero(N0.getOperand(0), DemandedBits, DemandedHiElts) ||
54309       DAG.MaskedValueIsZero(N0.getOperand(1), DemandedBits, DemandedHiElts);
54310   bool Op1HiZero =
54311       DAG.MaskedValueIsZero(N1.getOperand(0), DemandedBits, DemandedHiElts) ||
54312       DAG.MaskedValueIsZero(N1.getOperand(1), DemandedBits, DemandedHiElts);
54313 
54314   // TODO: Check for zero lower elements once we have actual codegen that
54315   // creates them.
54316   if (!Op0HiZero || !Op1HiZero)
54317     return SDValue();
54318 
54319   // Create a shuffle mask packing the lower elements from each VPMADDWD.
54320   SmallVector<int> Mask;
54321   for (int i = 0; i != (int)NumElts; ++i) {
54322     Mask.push_back(2 * i);
54323     Mask.push_back(2 * (i + NumElts));
54324   }
54325 
54326   SDValue LHS =
54327       DAG.getVectorShuffle(OpVT, DL, N0.getOperand(0), N1.getOperand(0), Mask);
54328   SDValue RHS =
54329       DAG.getVectorShuffle(OpVT, DL, N0.getOperand(1), N1.getOperand(1), Mask);
54330   return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS);
54331 }
54332 
54333 /// CMOV of constants requires materializing constant operands in registers.
54334 /// Try to fold those constants into an 'add' instruction to reduce instruction
54335 /// count. We do this with CMOV rather the generic 'select' because there are
54336 /// earlier folds that may be used to turn select-of-constants into logic hacks.
54337 static SDValue pushAddIntoCmovOfConsts(SDNode *N, SelectionDAG &DAG,
54338                                        const X86Subtarget &Subtarget) {
54339   // If an operand is zero, add-of-0 gets simplified away, so that's clearly
54340   // better because we eliminate 1-2 instructions. This transform is still
54341   // an improvement without zero operands because we trade 2 move constants and
54342   // 1 add for 2 adds (LEA) as long as the constants can be represented as
54343   // immediate asm operands (fit in 32-bits).
54344   auto isSuitableCmov = [](SDValue V) {
54345     if (V.getOpcode() != X86ISD::CMOV || !V.hasOneUse())
54346       return false;
54347     if (!isa<ConstantSDNode>(V.getOperand(0)) ||
54348         !isa<ConstantSDNode>(V.getOperand(1)))
54349       return false;
54350     return isNullConstant(V.getOperand(0)) || isNullConstant(V.getOperand(1)) ||
54351            (V.getConstantOperandAPInt(0).isSignedIntN(32) &&
54352             V.getConstantOperandAPInt(1).isSignedIntN(32));
54353   };
54354 
54355   // Match an appropriate CMOV as the first operand of the add.
54356   SDValue Cmov = N->getOperand(0);
54357   SDValue OtherOp = N->getOperand(1);
54358   if (!isSuitableCmov(Cmov))
54359     std::swap(Cmov, OtherOp);
54360   if (!isSuitableCmov(Cmov))
54361     return SDValue();
54362 
54363   // Don't remove a load folding opportunity for the add. That would neutralize
54364   // any improvements from removing constant materializations.
54365   if (X86::mayFoldLoad(OtherOp, Subtarget))
54366     return SDValue();
54367 
54368   EVT VT = N->getValueType(0);
54369   SDLoc DL(N);
54370   SDValue FalseOp = Cmov.getOperand(0);
54371   SDValue TrueOp = Cmov.getOperand(1);
54372 
54373   // We will push the add through the select, but we can potentially do better
54374   // if we know there is another add in the sequence and this is pointer math.
54375   // In that case, we can absorb an add into the trailing memory op and avoid
54376   // a 3-operand LEA which is likely slower than a 2-operand LEA.
54377   // TODO: If target has "slow3OpsLEA", do this even without the trailing memop?
54378   if (OtherOp.getOpcode() == ISD::ADD && OtherOp.hasOneUse() &&
54379       !isa<ConstantSDNode>(OtherOp.getOperand(0)) &&
54380       all_of(N->uses(), [&](SDNode *Use) {
54381         auto *MemNode = dyn_cast<MemSDNode>(Use);
54382         return MemNode && MemNode->getBasePtr().getNode() == N;
54383       })) {
54384     // add (cmov C1, C2), add (X, Y) --> add (cmov (add X, C1), (add X, C2)), Y
54385     // TODO: We are arbitrarily choosing op0 as the 1st piece of the sum, but
54386     //       it is possible that choosing op1 might be better.
54387     SDValue X = OtherOp.getOperand(0), Y = OtherOp.getOperand(1);
54388     FalseOp = DAG.getNode(ISD::ADD, DL, VT, X, FalseOp);
54389     TrueOp = DAG.getNode(ISD::ADD, DL, VT, X, TrueOp);
54390     Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp,
54391                        Cmov.getOperand(2), Cmov.getOperand(3));
54392     return DAG.getNode(ISD::ADD, DL, VT, Cmov, Y);
54393   }
54394 
54395   // add (cmov C1, C2), OtherOp --> cmov (add OtherOp, C1), (add OtherOp, C2)
54396   FalseOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, FalseOp);
54397   TrueOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, TrueOp);
54398   return DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, Cmov.getOperand(2),
54399                      Cmov.getOperand(3));
54400 }
54401 
54402 static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
54403                           TargetLowering::DAGCombinerInfo &DCI,
54404                           const X86Subtarget &Subtarget) {
54405   EVT VT = N->getValueType(0);
54406   SDValue Op0 = N->getOperand(0);
54407   SDValue Op1 = N->getOperand(1);
54408   SDLoc DL(N);
54409 
54410   if (SDValue Select = pushAddIntoCmovOfConsts(N, DAG, Subtarget))
54411     return Select;
54412 
54413   if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget))
54414     return MAdd;
54415   if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget))
54416     return MAdd;
54417   if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
54418     return MAdd;
54419 
54420   // Try to synthesize horizontal adds from adds of shuffles.
54421   if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
54422     return V;
54423 
54424   // If vectors of i1 are legal, turn (add (zext (vXi1 X)), Y) into
54425   // (sub Y, (sext (vXi1 X))).
54426   // FIXME: We have the (sub Y, (zext (vXi1 X))) -> (add (sext (vXi1 X)), Y) in
54427   // generic DAG combine without a legal type check, but adding this there
54428   // caused regressions.
54429   if (VT.isVector()) {
54430     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54431     if (Op0.getOpcode() == ISD::ZERO_EXTEND &&
54432         Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
54433         TLI.isTypeLegal(Op0.getOperand(0).getValueType())) {
54434       SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
54435       return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt);
54436     }
54437 
54438     if (Op1.getOpcode() == ISD::ZERO_EXTEND &&
54439         Op1.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
54440         TLI.isTypeLegal(Op1.getOperand(0).getValueType())) {
54441       SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
54442       return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt);
54443     }
54444   }
54445 
54446   // Fold ADD(ADC(Y,0,W),X) -> ADC(X,Y,W)
54447   if (Op0.getOpcode() == X86ISD::ADC && Op0->hasOneUse() &&
54448       X86::isZeroNode(Op0.getOperand(1))) {
54449     assert(!Op0->hasAnyUseOfValue(1) && "Overflow bit in use");
54450     return DAG.getNode(X86ISD::ADC, SDLoc(Op0), Op0->getVTList(), Op1,
54451                        Op0.getOperand(0), Op0.getOperand(2));
54452   }
54453 
54454   return combineAddOrSubToADCOrSBB(N, DAG);
54455 }
54456 
54457 // Try to fold (sub Y, cmovns X, -X) -> (add Y, cmovns -X, X) if the cmov
54458 // condition comes from the subtract node that produced -X. This matches the
54459 // cmov expansion for absolute value. By swapping the operands we convert abs
54460 // to nabs.
54461 static SDValue combineSubABS(SDNode *N, SelectionDAG &DAG) {
54462   SDValue N0 = N->getOperand(0);
54463   SDValue N1 = N->getOperand(1);
54464 
54465   if (N1.getOpcode() != X86ISD::CMOV || !N1.hasOneUse())
54466     return SDValue();
54467 
54468   X86::CondCode CC = (X86::CondCode)N1.getConstantOperandVal(2);
54469   if (CC != X86::COND_S && CC != X86::COND_NS)
54470     return SDValue();
54471 
54472   // Condition should come from a negate operation.
54473   SDValue Cond = N1.getOperand(3);
54474   if (Cond.getOpcode() != X86ISD::SUB || !isNullConstant(Cond.getOperand(0)))
54475     return SDValue();
54476   assert(Cond.getResNo() == 1 && "Unexpected result number");
54477 
54478   // Get the X and -X from the negate.
54479   SDValue NegX = Cond.getValue(0);
54480   SDValue X = Cond.getOperand(1);
54481 
54482   SDValue FalseOp = N1.getOperand(0);
54483   SDValue TrueOp = N1.getOperand(1);
54484 
54485   // Cmov operands should be X and NegX. Order doesn't matter.
54486   if (!(TrueOp == X && FalseOp == NegX) && !(TrueOp == NegX && FalseOp == X))
54487     return SDValue();
54488 
54489   // Build a new CMOV with the operands swapped.
54490   SDLoc DL(N);
54491   MVT VT = N->getSimpleValueType(0);
54492   SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp,
54493                              N1.getOperand(2), Cond);
54494   // Convert sub to add.
54495   return DAG.getNode(ISD::ADD, DL, VT, N0, Cmov);
54496 }
54497 
54498 static SDValue combineSubSetcc(SDNode *N, SelectionDAG &DAG) {
54499   SDValue Op0 = N->getOperand(0);
54500   SDValue Op1 = N->getOperand(1);
54501 
54502   // (sub C (zero_extend (setcc)))
54503   // =>
54504   // (add (zero_extend (setcc inverted) C-1))   if C is a nonzero immediate
54505   // Don't disturb (sub 0 setcc), which is easily done with neg.
54506   EVT VT = N->getValueType(0);
54507   auto *Op0C = dyn_cast<ConstantSDNode>(Op0);
54508   if (Op1.getOpcode() == ISD::ZERO_EXTEND && Op1.hasOneUse() && Op0C &&
54509       !Op0C->isZero() && Op1.getOperand(0).getOpcode() == X86ISD::SETCC &&
54510       Op1.getOperand(0).hasOneUse()) {
54511     SDValue SetCC = Op1.getOperand(0);
54512     X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0);
54513     X86::CondCode NewCC = X86::GetOppositeBranchCondition(CC);
54514     APInt NewImm = Op0C->getAPIntValue() - 1;
54515     SDLoc DL(Op1);
54516     SDValue NewSetCC = getSETCC(NewCC, SetCC.getOperand(1), DL, DAG);
54517     NewSetCC = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NewSetCC);
54518     return DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(VT, VT), NewSetCC,
54519                        DAG.getConstant(NewImm, DL, VT));
54520   }
54521 
54522   return SDValue();
54523 }
54524 
54525 static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
54526                           TargetLowering::DAGCombinerInfo &DCI,
54527                           const X86Subtarget &Subtarget) {
54528   SDValue Op0 = N->getOperand(0);
54529   SDValue Op1 = N->getOperand(1);
54530 
54531   // TODO: Add NoOpaque handling to isConstantIntBuildVectorOrConstantInt.
54532   auto IsNonOpaqueConstant = [&](SDValue Op) {
54533     if (SDNode *C = DAG.isConstantIntBuildVectorOrConstantInt(Op)) {
54534       if (auto *Cst = dyn_cast<ConstantSDNode>(C))
54535         return !Cst->isOpaque();
54536       return true;
54537     }
54538     return false;
54539   };
54540 
54541   // X86 can't encode an immediate LHS of a sub. See if we can push the
54542   // negation into a preceding instruction. If the RHS of the sub is a XOR with
54543   // one use and a constant, invert the immediate, saving one register.
54544   // However, ignore cases where C1 is 0, as those will become a NEG.
54545   // sub(C1, xor(X, C2)) -> add(xor(X, ~C2), C1+1)
54546   if (Op1.getOpcode() == ISD::XOR && IsNonOpaqueConstant(Op0) &&
54547       !isNullConstant(Op0) && IsNonOpaqueConstant(Op1.getOperand(1)) &&
54548       Op1->hasOneUse()) {
54549     SDLoc DL(N);
54550     EVT VT = Op0.getValueType();
54551     SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0),
54552                                  DAG.getNOT(SDLoc(Op1), Op1.getOperand(1), VT));
54553     SDValue NewAdd =
54554         DAG.getNode(ISD::ADD, DL, VT, Op0, DAG.getConstant(1, DL, VT));
54555     return DAG.getNode(ISD::ADD, DL, VT, NewXor, NewAdd);
54556   }
54557 
54558   if (SDValue V = combineSubABS(N, DAG))
54559     return V;
54560 
54561   // Try to synthesize horizontal subs from subs of shuffles.
54562   if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
54563     return V;
54564 
54565   // Fold SUB(X,ADC(Y,0,W)) -> SBB(X,Y,W)
54566   if (Op1.getOpcode() == X86ISD::ADC && Op1->hasOneUse() &&
54567       X86::isZeroNode(Op1.getOperand(1))) {
54568     assert(!Op1->hasAnyUseOfValue(1) && "Overflow bit in use");
54569     return DAG.getNode(X86ISD::SBB, SDLoc(Op1), Op1->getVTList(), Op0,
54570                        Op1.getOperand(0), Op1.getOperand(2));
54571   }
54572 
54573   // Fold SUB(X,SBB(Y,Z,W)) -> SUB(ADC(X,Z,W),Y)
54574   // Don't fold to ADC(0,0,W)/SETCC_CARRY pattern which will prevent more folds.
54575   if (Op1.getOpcode() == X86ISD::SBB && Op1->hasOneUse() &&
54576       !(X86::isZeroNode(Op0) && X86::isZeroNode(Op1.getOperand(1)))) {
54577     assert(!Op1->hasAnyUseOfValue(1) && "Overflow bit in use");
54578     SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0,
54579                               Op1.getOperand(1), Op1.getOperand(2));
54580     return DAG.getNode(ISD::SUB, SDLoc(N), Op0.getValueType(), ADC.getValue(0),
54581                        Op1.getOperand(0));
54582   }
54583 
54584   if (SDValue V = combineXorSubCTLZ(N, DAG, Subtarget))
54585     return V;
54586 
54587   if (SDValue V = combineAddOrSubToADCOrSBB(N, DAG))
54588     return V;
54589 
54590   return combineSubSetcc(N, DAG);
54591 }
54592 
54593 static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG,
54594                                     const X86Subtarget &Subtarget) {
54595   MVT VT = N->getSimpleValueType(0);
54596   SDLoc DL(N);
54597 
54598   if (N->getOperand(0) == N->getOperand(1)) {
54599     if (N->getOpcode() == X86ISD::PCMPEQ)
54600       return DAG.getConstant(-1, DL, VT);
54601     if (N->getOpcode() == X86ISD::PCMPGT)
54602       return DAG.getConstant(0, DL, VT);
54603   }
54604 
54605   return SDValue();
54606 }
54607 
54608 /// Helper that combines an array of subvector ops as if they were the operands
54609 /// of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.
54610 /// ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.
54611 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
54612                                       ArrayRef<SDValue> Ops, SelectionDAG &DAG,
54613                                       TargetLowering::DAGCombinerInfo &DCI,
54614                                       const X86Subtarget &Subtarget) {
54615   assert(Subtarget.hasAVX() && "AVX assumed for concat_vectors");
54616   unsigned EltSizeInBits = VT.getScalarSizeInBits();
54617 
54618   if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
54619     return DAG.getUNDEF(VT);
54620 
54621   if (llvm::all_of(Ops, [](SDValue Op) {
54622         return ISD::isBuildVectorAllZeros(Op.getNode());
54623       }))
54624     return getZeroVector(VT, Subtarget, DAG, DL);
54625 
54626   SDValue Op0 = Ops[0];
54627   bool IsSplat = llvm::all_equal(Ops);
54628   unsigned NumOps = Ops.size();
54629   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54630   LLVMContext &Ctx = *DAG.getContext();
54631 
54632   // Repeated subvectors.
54633   if (IsSplat &&
54634       (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) {
54635     // If this broadcast is inserted into both halves, use a larger broadcast.
54636     if (Op0.getOpcode() == X86ISD::VBROADCAST)
54637       return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0));
54638 
54639     // concat_vectors(movddup(x),movddup(x)) -> broadcast(x)
54640     if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 &&
54641         (Subtarget.hasAVX2() ||
54642          X86::mayFoldLoadIntoBroadcastFromMem(Op0.getOperand(0),
54643                                               VT.getScalarType(), Subtarget)))
54644       return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
54645                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64,
54646                                      Op0.getOperand(0),
54647                                      DAG.getIntPtrConstant(0, DL)));
54648 
54649     // concat_vectors(scalar_to_vector(x),scalar_to_vector(x)) -> broadcast(x)
54650     if (Op0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
54651         (Subtarget.hasAVX2() ||
54652          (EltSizeInBits >= 32 &&
54653           X86::mayFoldLoad(Op0.getOperand(0), Subtarget))) &&
54654         Op0.getOperand(0).getValueType() == VT.getScalarType())
54655       return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0));
54656 
54657     // concat_vectors(extract_subvector(broadcast(x)),
54658     //                extract_subvector(broadcast(x))) -> broadcast(x)
54659     // concat_vectors(extract_subvector(subv_broadcast(x)),
54660     //                extract_subvector(subv_broadcast(x))) -> subv_broadcast(x)
54661     if (Op0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
54662         Op0.getOperand(0).getValueType() == VT) {
54663       SDValue SrcVec = Op0.getOperand(0);
54664       if (SrcVec.getOpcode() == X86ISD::VBROADCAST ||
54665           SrcVec.getOpcode() == X86ISD::VBROADCAST_LOAD)
54666         return Op0.getOperand(0);
54667       if (SrcVec.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
54668           Op0.getValueType() == cast<MemSDNode>(SrcVec)->getMemoryVT())
54669         return Op0.getOperand(0);
54670     }
54671 
54672     // concat_vectors(permq(x),permq(x)) -> permq(concat_vectors(x,x))
54673     if (Op0.getOpcode() == X86ISD::VPERMI && Subtarget.useAVX512Regs() &&
54674         !X86::mayFoldLoad(Op0.getOperand(0), Subtarget))
54675       return DAG.getNode(Op0.getOpcode(), DL, VT,
54676                          DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
54677                                      Op0.getOperand(0), Op0.getOperand(0)),
54678                          Op0.getOperand(1));
54679   }
54680 
54681   // concat(extract_subvector(v0,c0), extract_subvector(v1,c1)) -> vperm2x128.
54682   // Only concat of subvector high halves which vperm2x128 is best at.
54683   // TODO: This should go in combineX86ShufflesRecursively eventually.
54684   if (VT.is256BitVector() && NumOps == 2) {
54685     SDValue Src0 = peekThroughBitcasts(Ops[0]);
54686     SDValue Src1 = peekThroughBitcasts(Ops[1]);
54687     if (Src0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
54688         Src1.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
54689       EVT SrcVT0 = Src0.getOperand(0).getValueType();
54690       EVT SrcVT1 = Src1.getOperand(0).getValueType();
54691       unsigned NumSrcElts0 = SrcVT0.getVectorNumElements();
54692       unsigned NumSrcElts1 = SrcVT1.getVectorNumElements();
54693       if (SrcVT0.is256BitVector() && SrcVT1.is256BitVector() &&
54694           Src0.getConstantOperandAPInt(1) == (NumSrcElts0 / 2) &&
54695           Src1.getConstantOperandAPInt(1) == (NumSrcElts1 / 2)) {
54696         return DAG.getNode(X86ISD::VPERM2X128, DL, VT,
54697                            DAG.getBitcast(VT, Src0.getOperand(0)),
54698                            DAG.getBitcast(VT, Src1.getOperand(0)),
54699                            DAG.getTargetConstant(0x31, DL, MVT::i8));
54700       }
54701     }
54702   }
54703 
54704   // Repeated opcode.
54705   // TODO - combineX86ShufflesRecursively should handle shuffle concatenation
54706   // but it currently struggles with different vector widths.
54707   if (llvm::all_of(Ops, [Op0](SDValue Op) {
54708         return Op.getOpcode() == Op0.getOpcode() && Op.hasOneUse();
54709       })) {
54710     auto ConcatSubOperand = [&](EVT VT, ArrayRef<SDValue> SubOps, unsigned I) {
54711       SmallVector<SDValue> Subs;
54712       for (SDValue SubOp : SubOps)
54713         Subs.push_back(SubOp.getOperand(I));
54714       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
54715     };
54716     auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) {
54717       bool AllConstants = true;
54718       bool AllSubVectors = true;
54719       for (unsigned I = 0, E = SubOps.size(); I != E; ++I) {
54720         SDValue Sub = SubOps[I].getOperand(Op);
54721         unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
54722         SDValue BC = peekThroughBitcasts(Sub);
54723         AllConstants &= ISD::isBuildVectorOfConstantSDNodes(BC.getNode()) ||
54724                         ISD::isBuildVectorOfConstantFPSDNodes(BC.getNode());
54725         AllSubVectors &= Sub.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
54726                          Sub.getOperand(0).getValueType() == VT &&
54727                          Sub.getConstantOperandAPInt(1) == (I * NumSubElts);
54728       }
54729       return AllConstants || AllSubVectors;
54730     };
54731 
54732     switch (Op0.getOpcode()) {
54733     case X86ISD::VBROADCAST: {
54734       if (!IsSplat && llvm::all_of(Ops, [](SDValue Op) {
54735             return Op.getOperand(0).getValueType().is128BitVector();
54736           })) {
54737         if (VT == MVT::v4f64 || VT == MVT::v4i64)
54738           return DAG.getNode(X86ISD::UNPCKL, DL, VT,
54739                              ConcatSubOperand(VT, Ops, 0),
54740                              ConcatSubOperand(VT, Ops, 0));
54741         // TODO: Add pseudo v8i32 PSHUFD handling to AVX1Only targets.
54742         if (VT == MVT::v8f32 || (VT == MVT::v8i32 && Subtarget.hasInt256()))
54743           return DAG.getNode(VT == MVT::v8f32 ? X86ISD::VPERMILPI
54744                                               : X86ISD::PSHUFD,
54745                              DL, VT, ConcatSubOperand(VT, Ops, 0),
54746                              getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG));
54747       }
54748       break;
54749     }
54750     case X86ISD::MOVDDUP:
54751     case X86ISD::MOVSHDUP:
54752     case X86ISD::MOVSLDUP: {
54753       if (!IsSplat)
54754         return DAG.getNode(Op0.getOpcode(), DL, VT,
54755                            ConcatSubOperand(VT, Ops, 0));
54756       break;
54757     }
54758     case X86ISD::SHUFP: {
54759       // Add SHUFPD support if/when necessary.
54760       if (!IsSplat && VT.getScalarType() == MVT::f32 &&
54761           llvm::all_of(Ops, [Op0](SDValue Op) {
54762             return Op.getOperand(2) == Op0.getOperand(2);
54763           })) {
54764         return DAG.getNode(Op0.getOpcode(), DL, VT,
54765                            ConcatSubOperand(VT, Ops, 0),
54766                            ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
54767       }
54768       break;
54769     }
54770     case X86ISD::UNPCKH:
54771     case X86ISD::UNPCKL: {
54772       // Don't concatenate build_vector patterns.
54773       if (!IsSplat && EltSizeInBits >= 32 &&
54774           ((VT.is256BitVector() && Subtarget.hasInt256()) ||
54775            (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
54776           none_of(Ops, [](SDValue Op) {
54777             return peekThroughBitcasts(Op.getOperand(0)).getOpcode() ==
54778                        ISD::SCALAR_TO_VECTOR ||
54779                    peekThroughBitcasts(Op.getOperand(1)).getOpcode() ==
54780                        ISD::SCALAR_TO_VECTOR;
54781           })) {
54782         return DAG.getNode(Op0.getOpcode(), DL, VT,
54783                            ConcatSubOperand(VT, Ops, 0),
54784                            ConcatSubOperand(VT, Ops, 1));
54785       }
54786       break;
54787     }
54788     case X86ISD::PSHUFHW:
54789     case X86ISD::PSHUFLW:
54790     case X86ISD::PSHUFD:
54791       if (!IsSplat && NumOps == 2 && VT.is256BitVector() &&
54792           Subtarget.hasInt256() && Op0.getOperand(1) == Ops[1].getOperand(1)) {
54793         return DAG.getNode(Op0.getOpcode(), DL, VT,
54794                            ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
54795       }
54796       [[fallthrough]];
54797     case X86ISD::VPERMILPI:
54798       if (!IsSplat && EltSizeInBits == 32 &&
54799           (VT.is256BitVector() ||
54800            (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
54801           all_of(Ops, [&Op0](SDValue Op) {
54802             return Op0.getOperand(1) == Op.getOperand(1);
54803           })) {
54804         MVT FloatVT = VT.changeVectorElementType(MVT::f32);
54805         SDValue Res = DAG.getBitcast(FloatVT, ConcatSubOperand(VT, Ops, 0));
54806         Res =
54807             DAG.getNode(X86ISD::VPERMILPI, DL, FloatVT, Res, Op0.getOperand(1));
54808         return DAG.getBitcast(VT, Res);
54809       }
54810       if (!IsSplat && NumOps == 2 && VT == MVT::v4f64) {
54811         uint64_t Idx0 = Ops[0].getConstantOperandVal(1);
54812         uint64_t Idx1 = Ops[1].getConstantOperandVal(1);
54813         uint64_t Idx = ((Idx1 & 3) << 2) | (Idx0 & 3);
54814         return DAG.getNode(Op0.getOpcode(), DL, VT,
54815                            ConcatSubOperand(VT, Ops, 0),
54816                            DAG.getTargetConstant(Idx, DL, MVT::i8));
54817       }
54818       break;
54819     case X86ISD::PSHUFB:
54820     case X86ISD::PSADBW:
54821       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
54822                        (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
54823         MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
54824         SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
54825                                  NumOps * SrcVT.getVectorNumElements());
54826         return DAG.getNode(Op0.getOpcode(), DL, VT,
54827                            ConcatSubOperand(SrcVT, Ops, 0),
54828                            ConcatSubOperand(SrcVT, Ops, 1));
54829       }
54830       break;
54831     case X86ISD::VPERMV:
54832       if (!IsSplat && NumOps == 2 &&
54833           (VT.is512BitVector() && Subtarget.useAVX512Regs())) {
54834         MVT OpVT = Op0.getSimpleValueType();
54835         int NumSrcElts = OpVT.getVectorNumElements();
54836         SmallVector<int, 64> ConcatMask;
54837         for (unsigned i = 0; i != NumOps; ++i) {
54838           SmallVector<int, 64> SubMask;
54839           SmallVector<SDValue, 2> SubOps;
54840           if (!getTargetShuffleMask(Ops[i].getNode(), OpVT, false, SubOps,
54841                                     SubMask))
54842             break;
54843           for (int M : SubMask) {
54844             if (0 <= M)
54845               M += i * NumSrcElts;
54846             ConcatMask.push_back(M);
54847           }
54848         }
54849         if (ConcatMask.size() == (NumOps * NumSrcElts)) {
54850           SDValue Src = concatSubVectors(Ops[0].getOperand(1),
54851                                          Ops[1].getOperand(1), DAG, DL);
54852           MVT IntMaskSVT = MVT::getIntegerVT(EltSizeInBits);
54853           MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts);
54854           SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
54855           return DAG.getNode(X86ISD::VPERMV, DL, VT, Mask, Src);
54856         }
54857       }
54858       break;
54859     case X86ISD::VPERMV3:
54860       if (!IsSplat && NumOps == 2 && VT.is512BitVector()) {
54861         MVT OpVT = Op0.getSimpleValueType();
54862         int NumSrcElts = OpVT.getVectorNumElements();
54863         SmallVector<int, 64> ConcatMask;
54864         for (unsigned i = 0; i != NumOps; ++i) {
54865           SmallVector<int, 64> SubMask;
54866           SmallVector<SDValue, 2> SubOps;
54867           if (!getTargetShuffleMask(Ops[i].getNode(), OpVT, false, SubOps,
54868                                     SubMask))
54869             break;
54870           for (int M : SubMask) {
54871             if (0 <= M) {
54872               M += M < NumSrcElts ? 0 : NumSrcElts;
54873               M += i * NumSrcElts;
54874             }
54875             ConcatMask.push_back(M);
54876           }
54877         }
54878         if (ConcatMask.size() == (NumOps * NumSrcElts)) {
54879           SDValue Src0 = concatSubVectors(Ops[0].getOperand(0),
54880                                           Ops[1].getOperand(0), DAG, DL);
54881           SDValue Src1 = concatSubVectors(Ops[0].getOperand(2),
54882                                           Ops[1].getOperand(2), DAG, DL);
54883           MVT IntMaskSVT = MVT::getIntegerVT(EltSizeInBits);
54884           MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts);
54885           SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
54886           return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1);
54887         }
54888       }
54889       break;
54890     case X86ISD::VPERM2X128: {
54891       if (!IsSplat && VT.is512BitVector() && Subtarget.useAVX512Regs()) {
54892         assert(NumOps == 2 && "Bad concat_vectors operands");
54893         unsigned Imm0 = Ops[0].getConstantOperandVal(2);
54894         unsigned Imm1 = Ops[1].getConstantOperandVal(2);
54895         // TODO: Handle zero'd subvectors.
54896         if ((Imm0 & 0x88) == 0 && (Imm1 & 0x88) == 0) {
54897           int Mask[4] = {(int)(Imm0 & 0x03), (int)((Imm0 >> 4) & 0x3), (int)(Imm1 & 0x03),
54898                          (int)((Imm1 >> 4) & 0x3)};
54899           MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64;
54900           SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
54901                                          Ops[0].getOperand(1), DAG, DL);
54902           SDValue RHS = concatSubVectors(Ops[1].getOperand(0),
54903                                          Ops[1].getOperand(1), DAG, DL);
54904           SDValue Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
54905                                     DAG.getBitcast(ShuffleVT, LHS),
54906                                     DAG.getBitcast(ShuffleVT, RHS),
54907                                     getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
54908           return DAG.getBitcast(VT, Res);
54909         }
54910       }
54911       break;
54912     }
54913     case X86ISD::SHUF128: {
54914       if (!IsSplat && NumOps == 2 && VT.is512BitVector()) {
54915         unsigned Imm0 = Ops[0].getConstantOperandVal(2);
54916         unsigned Imm1 = Ops[1].getConstantOperandVal(2);
54917         unsigned Imm = ((Imm0 & 1) << 0) | ((Imm0 & 2) << 1) | 0x08 |
54918                        ((Imm1 & 1) << 4) | ((Imm1 & 2) << 5) | 0x80;
54919         SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
54920                                        Ops[0].getOperand(1), DAG, DL);
54921         SDValue RHS = concatSubVectors(Ops[1].getOperand(0),
54922                                        Ops[1].getOperand(1), DAG, DL);
54923         return DAG.getNode(X86ISD::SHUF128, DL, VT, LHS, RHS,
54924                            DAG.getTargetConstant(Imm, DL, MVT::i8));
54925       }
54926       break;
54927     }
54928     case ISD::TRUNCATE:
54929       if (!IsSplat && NumOps == 2 && VT.is256BitVector()) {
54930         EVT SrcVT = Ops[0].getOperand(0).getValueType();
54931         if (SrcVT.is256BitVector() && SrcVT.isSimple() &&
54932             SrcVT == Ops[1].getOperand(0).getValueType() &&
54933             Subtarget.useAVX512Regs() &&
54934             Subtarget.getPreferVectorWidth() >= 512 &&
54935             (SrcVT.getScalarSizeInBits() > 16 || Subtarget.useBWIRegs())) {
54936           EVT NewSrcVT = SrcVT.getDoubleNumVectorElementsVT(Ctx);
54937           return DAG.getNode(ISD::TRUNCATE, DL, VT,
54938                              ConcatSubOperand(NewSrcVT, Ops, 0));
54939         }
54940       }
54941       break;
54942     case X86ISD::VSHLI:
54943     case X86ISD::VSRLI:
54944       // Special case: SHL/SRL AVX1 V4i64 by 32-bits can lower as a shuffle.
54945       // TODO: Move this to LowerShiftByScalarImmediate?
54946       if (VT == MVT::v4i64 && !Subtarget.hasInt256() &&
54947           llvm::all_of(Ops, [](SDValue Op) {
54948             return Op.getConstantOperandAPInt(1) == 32;
54949           })) {
54950         SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0));
54951         SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL);
54952         if (Op0.getOpcode() == X86ISD::VSHLI) {
54953           Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
54954                                      {8, 0, 8, 2, 8, 4, 8, 6});
54955         } else {
54956           Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
54957                                      {1, 8, 3, 8, 5, 8, 7, 8});
54958         }
54959         return DAG.getBitcast(VT, Res);
54960       }
54961       [[fallthrough]];
54962     case X86ISD::VSRAI:
54963     case X86ISD::VSHL:
54964     case X86ISD::VSRL:
54965     case X86ISD::VSRA:
54966       if (((VT.is256BitVector() && Subtarget.hasInt256()) ||
54967            (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
54968             (EltSizeInBits >= 32 || Subtarget.useBWIRegs()))) &&
54969           llvm::all_of(Ops, [Op0](SDValue Op) {
54970             return Op0.getOperand(1) == Op.getOperand(1);
54971           })) {
54972         return DAG.getNode(Op0.getOpcode(), DL, VT,
54973                            ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
54974       }
54975       break;
54976     case X86ISD::VPERMI:
54977     case X86ISD::VROTLI:
54978     case X86ISD::VROTRI:
54979       if (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
54980           llvm::all_of(Ops, [Op0](SDValue Op) {
54981             return Op0.getOperand(1) == Op.getOperand(1);
54982           })) {
54983         return DAG.getNode(Op0.getOpcode(), DL, VT,
54984                            ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
54985       }
54986       break;
54987     case ISD::AND:
54988     case ISD::OR:
54989     case ISD::XOR:
54990     case X86ISD::ANDNP:
54991       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
54992                        (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
54993         return DAG.getNode(Op0.getOpcode(), DL, VT,
54994                            ConcatSubOperand(VT, Ops, 0),
54995                            ConcatSubOperand(VT, Ops, 1));
54996       }
54997       break;
54998     case X86ISD::PCMPEQ:
54999     case X86ISD::PCMPGT:
55000       if (!IsSplat && VT.is256BitVector() && Subtarget.hasInt256() &&
55001           (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1))) {
55002         return DAG.getNode(Op0.getOpcode(), DL, VT,
55003                            ConcatSubOperand(VT, Ops, 0),
55004                            ConcatSubOperand(VT, Ops, 1));
55005       }
55006       break;
55007     case ISD::CTPOP:
55008     case ISD::CTTZ:
55009     case ISD::CTLZ:
55010     case ISD::CTTZ_ZERO_UNDEF:
55011     case ISD::CTLZ_ZERO_UNDEF:
55012       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
55013                        (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
55014         return DAG.getNode(Op0.getOpcode(), DL, VT,
55015                            ConcatSubOperand(VT, Ops, 0));
55016       }
55017       break;
55018     case X86ISD::GF2P8AFFINEQB:
55019       if (!IsSplat &&
55020           (VT.is256BitVector() ||
55021            (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
55022           llvm::all_of(Ops, [Op0](SDValue Op) {
55023             return Op0.getOperand(2) == Op.getOperand(2);
55024           })) {
55025         return DAG.getNode(Op0.getOpcode(), DL, VT,
55026                            ConcatSubOperand(VT, Ops, 0),
55027                            ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
55028       }
55029       break;
55030     case ISD::ADD:
55031     case ISD::SUB:
55032     case ISD::MUL:
55033       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
55034                        (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
55035                         (EltSizeInBits >= 32 || Subtarget.useBWIRegs())))) {
55036         return DAG.getNode(Op0.getOpcode(), DL, VT,
55037                            ConcatSubOperand(VT, Ops, 0),
55038                            ConcatSubOperand(VT, Ops, 1));
55039       }
55040       break;
55041     // Due to VADD, VSUB, VMUL can executed on more ports than VINSERT and
55042     // their latency are short, so here we don't replace them.
55043     case ISD::FDIV:
55044       if (!IsSplat && (VT.is256BitVector() ||
55045                        (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
55046         return DAG.getNode(Op0.getOpcode(), DL, VT,
55047                            ConcatSubOperand(VT, Ops, 0),
55048                            ConcatSubOperand(VT, Ops, 1));
55049       }
55050       break;
55051     case X86ISD::HADD:
55052     case X86ISD::HSUB:
55053     case X86ISD::FHADD:
55054     case X86ISD::FHSUB:
55055       if (!IsSplat && VT.is256BitVector() &&
55056           (VT.isFloatingPoint() || Subtarget.hasInt256())) {
55057         return DAG.getNode(Op0.getOpcode(), DL, VT,
55058                            ConcatSubOperand(VT, Ops, 0),
55059                            ConcatSubOperand(VT, Ops, 1));
55060       }
55061       break;
55062     case X86ISD::PACKSS:
55063     case X86ISD::PACKUS:
55064       if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
55065                        (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
55066         MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
55067         SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
55068                                  NumOps * SrcVT.getVectorNumElements());
55069         return DAG.getNode(Op0.getOpcode(), DL, VT,
55070                            ConcatSubOperand(SrcVT, Ops, 0),
55071                            ConcatSubOperand(SrcVT, Ops, 1));
55072       }
55073       break;
55074     case X86ISD::PALIGNR:
55075       if (!IsSplat &&
55076           ((VT.is256BitVector() && Subtarget.hasInt256()) ||
55077            (VT.is512BitVector() && Subtarget.useBWIRegs())) &&
55078           llvm::all_of(Ops, [Op0](SDValue Op) {
55079             return Op0.getOperand(2) == Op.getOperand(2);
55080           })) {
55081         return DAG.getNode(Op0.getOpcode(), DL, VT,
55082                            ConcatSubOperand(VT, Ops, 0),
55083                            ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
55084       }
55085       break;
55086     case X86ISD::BLENDI:
55087       if (NumOps == 2 && VT.is512BitVector() && Subtarget.useBWIRegs()) {
55088         uint64_t Mask0 = Ops[0].getConstantOperandVal(2);
55089         uint64_t Mask1 = Ops[1].getConstantOperandVal(2);
55090         uint64_t Mask = (Mask1 << (VT.getVectorNumElements() / 2)) | Mask0;
55091         MVT MaskSVT = MVT::getIntegerVT(VT.getVectorNumElements());
55092         MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
55093         SDValue Sel =
55094             DAG.getBitcast(MaskVT, DAG.getConstant(Mask, DL, MaskSVT));
55095         return DAG.getSelect(DL, VT, Sel, ConcatSubOperand(VT, Ops, 1),
55096                              ConcatSubOperand(VT, Ops, 0));
55097       }
55098       break;
55099     case ISD::VSELECT:
55100       if (!IsSplat && Subtarget.hasAVX512() &&
55101           (VT.is256BitVector() ||
55102            (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
55103           (EltSizeInBits >= 32 || Subtarget.hasBWI())) {
55104         EVT SelVT = Ops[0].getOperand(0).getValueType();
55105         if (SelVT.getVectorElementType() == MVT::i1) {
55106           SelVT = EVT::getVectorVT(Ctx, MVT::i1,
55107                                    NumOps * SelVT.getVectorNumElements());
55108           if (TLI.isTypeLegal(SelVT))
55109             return DAG.getNode(Op0.getOpcode(), DL, VT,
55110                                ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0),
55111                                ConcatSubOperand(VT, Ops, 1),
55112                                ConcatSubOperand(VT, Ops, 2));
55113         }
55114       }
55115       [[fallthrough]];
55116     case X86ISD::BLENDV:
55117       if (!IsSplat && VT.is256BitVector() && NumOps == 2 &&
55118           (EltSizeInBits >= 32 || Subtarget.hasInt256()) &&
55119           IsConcatFree(VT, Ops, 1) && IsConcatFree(VT, Ops, 2)) {
55120         EVT SelVT = Ops[0].getOperand(0).getValueType();
55121         SelVT = SelVT.getDoubleNumVectorElementsVT(Ctx);
55122         if (TLI.isTypeLegal(SelVT))
55123           return DAG.getNode(Op0.getOpcode(), DL, VT,
55124                              ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0),
55125                              ConcatSubOperand(VT, Ops, 1),
55126                              ConcatSubOperand(VT, Ops, 2));
55127       }
55128       break;
55129     }
55130   }
55131 
55132   // Fold subvector loads into one.
55133   // If needed, look through bitcasts to get to the load.
55134   if (auto *FirstLd = dyn_cast<LoadSDNode>(peekThroughBitcasts(Op0))) {
55135     unsigned Fast;
55136     const X86TargetLowering *TLI = Subtarget.getTargetLowering();
55137     if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT,
55138                                 *FirstLd->getMemOperand(), &Fast) &&
55139         Fast) {
55140       if (SDValue Ld =
55141               EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false))
55142         return Ld;
55143     }
55144   }
55145 
55146   // Attempt to fold target constant loads.
55147   if (all_of(Ops, [](SDValue Op) { return getTargetConstantFromNode(Op); })) {
55148     SmallVector<APInt> EltBits;
55149     APInt UndefElts = APInt::getZero(VT.getVectorNumElements());
55150     for (unsigned I = 0; I != NumOps; ++I) {
55151       APInt OpUndefElts;
55152       SmallVector<APInt> OpEltBits;
55153       if (!getTargetConstantBitsFromNode(Ops[I], EltSizeInBits, OpUndefElts,
55154                                          OpEltBits, true, false))
55155         break;
55156       EltBits.append(OpEltBits);
55157       UndefElts.insertBits(OpUndefElts, I * OpUndefElts.getBitWidth());
55158     }
55159     if (EltBits.size() == VT.getVectorNumElements()) {
55160       Constant *C = getConstantVector(VT, EltBits, UndefElts, Ctx);
55161       MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
55162       SDValue CV = DAG.getConstantPool(C, PVT);
55163       MachineFunction &MF = DAG.getMachineFunction();
55164       MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF);
55165       SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI);
55166       SDValue Sub = extractSubVector(Ld, 0, DAG, DL, Op0.getValueSizeInBits());
55167       DAG.ReplaceAllUsesOfValueWith(Op0, Sub);
55168       return Ld;
55169     }
55170   }
55171 
55172   // If this simple subvector or scalar/subvector broadcast_load is inserted
55173   // into both halves, use a larger broadcast_load. Update other uses to use
55174   // an extracted subvector.
55175   if (IsSplat &&
55176       (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) {
55177     if (ISD::isNormalLoad(Op0.getNode()) ||
55178         Op0.getOpcode() == X86ISD::VBROADCAST_LOAD ||
55179         Op0.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) {
55180       auto *Mem = cast<MemSDNode>(Op0);
55181       unsigned Opc = Op0.getOpcode() == X86ISD::VBROADCAST_LOAD
55182                          ? X86ISD::VBROADCAST_LOAD
55183                          : X86ISD::SUBV_BROADCAST_LOAD;
55184       if (SDValue BcastLd =
55185               getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) {
55186         SDValue BcastSrc =
55187             extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits());
55188         DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc);
55189         return BcastLd;
55190       }
55191     }
55192   }
55193 
55194   // If we're splatting a 128-bit subvector to 512-bits, use SHUF128 directly.
55195   if (IsSplat && NumOps == 4 && VT.is512BitVector() &&
55196       Subtarget.useAVX512Regs()) {
55197     MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64;
55198     SDValue Res = widenSubVector(Op0, false, Subtarget, DAG, DL, 512);
55199     Res = DAG.getBitcast(ShuffleVT, Res);
55200     Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, Res, Res,
55201                       getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG));
55202     return DAG.getBitcast(VT, Res);
55203   }
55204 
55205   return SDValue();
55206 }
55207 
55208 static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG,
55209                                      TargetLowering::DAGCombinerInfo &DCI,
55210                                      const X86Subtarget &Subtarget) {
55211   EVT VT = N->getValueType(0);
55212   EVT SrcVT = N->getOperand(0).getValueType();
55213   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55214   SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end());
55215 
55216   if (VT.getVectorElementType() == MVT::i1) {
55217     // Attempt to constant fold.
55218     unsigned SubSizeInBits = SrcVT.getSizeInBits();
55219     APInt Constant = APInt::getZero(VT.getSizeInBits());
55220     for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
55221       auto *C = dyn_cast<ConstantSDNode>(peekThroughBitcasts(Ops[I]));
55222       if (!C) break;
55223       Constant.insertBits(C->getAPIntValue(), I * SubSizeInBits);
55224       if (I == (E - 1)) {
55225         EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
55226         if (TLI.isTypeLegal(IntVT))
55227           return DAG.getBitcast(VT, DAG.getConstant(Constant, SDLoc(N), IntVT));
55228       }
55229     }
55230 
55231     // Don't do anything else for i1 vectors.
55232     return SDValue();
55233   }
55234 
55235   if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) {
55236     if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
55237                                            DCI, Subtarget))
55238       return R;
55239   }
55240 
55241   return SDValue();
55242 }
55243 
55244 static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
55245                                        TargetLowering::DAGCombinerInfo &DCI,
55246                                        const X86Subtarget &Subtarget) {
55247   if (DCI.isBeforeLegalizeOps())
55248     return SDValue();
55249 
55250   MVT OpVT = N->getSimpleValueType(0);
55251 
55252   bool IsI1Vector = OpVT.getVectorElementType() == MVT::i1;
55253 
55254   SDLoc dl(N);
55255   SDValue Vec = N->getOperand(0);
55256   SDValue SubVec = N->getOperand(1);
55257 
55258   uint64_t IdxVal = N->getConstantOperandVal(2);
55259   MVT SubVecVT = SubVec.getSimpleValueType();
55260 
55261   if (Vec.isUndef() && SubVec.isUndef())
55262     return DAG.getUNDEF(OpVT);
55263 
55264   // Inserting undefs/zeros into zeros/undefs is a zero vector.
55265   if ((Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())) &&
55266       (SubVec.isUndef() || ISD::isBuildVectorAllZeros(SubVec.getNode())))
55267     return getZeroVector(OpVT, Subtarget, DAG, dl);
55268 
55269   if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
55270     // If we're inserting into a zero vector and then into a larger zero vector,
55271     // just insert into the larger zero vector directly.
55272     if (SubVec.getOpcode() == ISD::INSERT_SUBVECTOR &&
55273         ISD::isBuildVectorAllZeros(SubVec.getOperand(0).getNode())) {
55274       uint64_t Idx2Val = SubVec.getConstantOperandVal(2);
55275       return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
55276                          getZeroVector(OpVT, Subtarget, DAG, dl),
55277                          SubVec.getOperand(1),
55278                          DAG.getIntPtrConstant(IdxVal + Idx2Val, dl));
55279     }
55280 
55281     // If we're inserting into a zero vector and our input was extracted from an
55282     // insert into a zero vector of the same type and the extraction was at
55283     // least as large as the original insertion. Just insert the original
55284     // subvector into a zero vector.
55285     if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR && IdxVal == 0 &&
55286         isNullConstant(SubVec.getOperand(1)) &&
55287         SubVec.getOperand(0).getOpcode() == ISD::INSERT_SUBVECTOR) {
55288       SDValue Ins = SubVec.getOperand(0);
55289       if (isNullConstant(Ins.getOperand(2)) &&
55290           ISD::isBuildVectorAllZeros(Ins.getOperand(0).getNode()) &&
55291           Ins.getOperand(1).getValueSizeInBits().getFixedValue() <=
55292               SubVecVT.getFixedSizeInBits())
55293           return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
55294                              getZeroVector(OpVT, Subtarget, DAG, dl),
55295                              Ins.getOperand(1), N->getOperand(2));
55296     }
55297   }
55298 
55299   // Stop here if this is an i1 vector.
55300   if (IsI1Vector)
55301     return SDValue();
55302 
55303   // Eliminate an intermediate vector widening:
55304   // insert_subvector X, (insert_subvector undef, Y, 0), Idx -->
55305   // insert_subvector X, Y, Idx
55306   // TODO: This is a more general version of a DAGCombiner fold, can we move it
55307   // there?
55308   if (SubVec.getOpcode() == ISD::INSERT_SUBVECTOR &&
55309       SubVec.getOperand(0).isUndef() && isNullConstant(SubVec.getOperand(2)))
55310     return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, Vec,
55311                        SubVec.getOperand(1), N->getOperand(2));
55312 
55313   // If this is an insert of an extract, combine to a shuffle. Don't do this
55314   // if the insert or extract can be represented with a subregister operation.
55315   if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
55316       SubVec.getOperand(0).getSimpleValueType() == OpVT &&
55317       (IdxVal != 0 ||
55318        !(Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())))) {
55319     int ExtIdxVal = SubVec.getConstantOperandVal(1);
55320     if (ExtIdxVal != 0) {
55321       int VecNumElts = OpVT.getVectorNumElements();
55322       int SubVecNumElts = SubVecVT.getVectorNumElements();
55323       SmallVector<int, 64> Mask(VecNumElts);
55324       // First create an identity shuffle mask.
55325       for (int i = 0; i != VecNumElts; ++i)
55326         Mask[i] = i;
55327       // Now insert the extracted portion.
55328       for (int i = 0; i != SubVecNumElts; ++i)
55329         Mask[i + IdxVal] = i + ExtIdxVal + VecNumElts;
55330 
55331       return DAG.getVectorShuffle(OpVT, dl, Vec, SubVec.getOperand(0), Mask);
55332     }
55333   }
55334 
55335   // Match concat_vector style patterns.
55336   SmallVector<SDValue, 2> SubVectorOps;
55337   if (collectConcatOps(N, SubVectorOps, DAG)) {
55338     if (SDValue Fold =
55339             combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, DCI, Subtarget))
55340       return Fold;
55341 
55342     // If we're inserting all zeros into the upper half, change this to
55343     // a concat with zero. We will match this to a move
55344     // with implicit upper bit zeroing during isel.
55345     // We do this here because we don't want combineConcatVectorOps to
55346     // create INSERT_SUBVECTOR from CONCAT_VECTORS.
55347     if (SubVectorOps.size() == 2 &&
55348         ISD::isBuildVectorAllZeros(SubVectorOps[1].getNode()))
55349       return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
55350                          getZeroVector(OpVT, Subtarget, DAG, dl),
55351                          SubVectorOps[0], DAG.getIntPtrConstant(0, dl));
55352 
55353     // Attempt to recursively combine to a shuffle.
55354     if (all_of(SubVectorOps, [](SDValue SubOp) {
55355           return isTargetShuffle(SubOp.getOpcode());
55356         })) {
55357       SDValue Op(N, 0);
55358       if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
55359         return Res;
55360     }
55361   }
55362 
55363   // If this is a broadcast insert into an upper undef, use a larger broadcast.
55364   if (Vec.isUndef() && IdxVal != 0 && SubVec.getOpcode() == X86ISD::VBROADCAST)
55365     return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0));
55366 
55367   // If this is a broadcast load inserted into an upper undef, use a larger
55368   // broadcast load.
55369   if (Vec.isUndef() && IdxVal != 0 && SubVec.hasOneUse() &&
55370       SubVec.getOpcode() == X86ISD::VBROADCAST_LOAD) {
55371     auto *MemIntr = cast<MemIntrinsicSDNode>(SubVec);
55372     SDVTList Tys = DAG.getVTList(OpVT, MVT::Other);
55373     SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
55374     SDValue BcastLd =
55375         DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
55376                                 MemIntr->getMemoryVT(),
55377                                 MemIntr->getMemOperand());
55378     DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
55379     return BcastLd;
55380   }
55381 
55382   // If we're splatting the lower half subvector of a full vector load into the
55383   // upper half, attempt to create a subvector broadcast.
55384   if (IdxVal == (OpVT.getVectorNumElements() / 2) && SubVec.hasOneUse() &&
55385       Vec.getValueSizeInBits() == (2 * SubVec.getValueSizeInBits())) {
55386     auto *VecLd = dyn_cast<LoadSDNode>(Vec);
55387     auto *SubLd = dyn_cast<LoadSDNode>(SubVec);
55388     if (VecLd && SubLd &&
55389         DAG.areNonVolatileConsecutiveLoads(SubLd, VecLd,
55390                                            SubVec.getValueSizeInBits() / 8, 0))
55391       return getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, dl, OpVT, SubVecVT,
55392                                SubLd, 0, DAG);
55393   }
55394 
55395   return SDValue();
55396 }
55397 
55398 /// If we are extracting a subvector of a vector select and the select condition
55399 /// is composed of concatenated vectors, try to narrow the select width. This
55400 /// is a common pattern for AVX1 integer code because 256-bit selects may be
55401 /// legal, but there is almost no integer math/logic available for 256-bit.
55402 /// This function should only be called with legal types (otherwise, the calls
55403 /// to get simple value types will assert).
55404 static SDValue narrowExtractedVectorSelect(SDNode *Ext, SelectionDAG &DAG) {
55405   SDValue Sel = Ext->getOperand(0);
55406   if (Sel.getOpcode() != ISD::VSELECT ||
55407       !isFreeToSplitVector(Sel.getOperand(0).getNode(), DAG))
55408     return SDValue();
55409 
55410   // Note: We assume simple value types because this should only be called with
55411   //       legal operations/types.
55412   // TODO: This can be extended to handle extraction to 256-bits.
55413   MVT VT = Ext->getSimpleValueType(0);
55414   if (!VT.is128BitVector())
55415     return SDValue();
55416 
55417   MVT SelCondVT = Sel.getOperand(0).getSimpleValueType();
55418   if (!SelCondVT.is256BitVector() && !SelCondVT.is512BitVector())
55419     return SDValue();
55420 
55421   MVT WideVT = Ext->getOperand(0).getSimpleValueType();
55422   MVT SelVT = Sel.getSimpleValueType();
55423   assert((SelVT.is256BitVector() || SelVT.is512BitVector()) &&
55424          "Unexpected vector type with legal operations");
55425 
55426   unsigned SelElts = SelVT.getVectorNumElements();
55427   unsigned CastedElts = WideVT.getVectorNumElements();
55428   unsigned ExtIdx = Ext->getConstantOperandVal(1);
55429   if (SelElts % CastedElts == 0) {
55430     // The select has the same or more (narrower) elements than the extract
55431     // operand. The extraction index gets scaled by that factor.
55432     ExtIdx *= (SelElts / CastedElts);
55433   } else if (CastedElts % SelElts == 0) {
55434     // The select has less (wider) elements than the extract operand. Make sure
55435     // that the extraction index can be divided evenly.
55436     unsigned IndexDivisor = CastedElts / SelElts;
55437     if (ExtIdx % IndexDivisor != 0)
55438       return SDValue();
55439     ExtIdx /= IndexDivisor;
55440   } else {
55441     llvm_unreachable("Element count of simple vector types are not divisible?");
55442   }
55443 
55444   unsigned NarrowingFactor = WideVT.getSizeInBits() / VT.getSizeInBits();
55445   unsigned NarrowElts = SelElts / NarrowingFactor;
55446   MVT NarrowSelVT = MVT::getVectorVT(SelVT.getVectorElementType(), NarrowElts);
55447   SDLoc DL(Ext);
55448   SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
55449   SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
55450   SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
55451   SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
55452   return DAG.getBitcast(VT, NarrowSel);
55453 }
55454 
55455 static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
55456                                         TargetLowering::DAGCombinerInfo &DCI,
55457                                         const X86Subtarget &Subtarget) {
55458   // For AVX1 only, if we are extracting from a 256-bit and+not (which will
55459   // eventually get combined/lowered into ANDNP) with a concatenated operand,
55460   // split the 'and' into 128-bit ops to avoid the concatenate and extract.
55461   // We let generic combining take over from there to simplify the
55462   // insert/extract and 'not'.
55463   // This pattern emerges during AVX1 legalization. We handle it before lowering
55464   // to avoid complications like splitting constant vector loads.
55465 
55466   // Capture the original wide type in the likely case that we need to bitcast
55467   // back to this type.
55468   if (!N->getValueType(0).isSimple())
55469     return SDValue();
55470 
55471   MVT VT = N->getSimpleValueType(0);
55472   SDValue InVec = N->getOperand(0);
55473   unsigned IdxVal = N->getConstantOperandVal(1);
55474   SDValue InVecBC = peekThroughBitcasts(InVec);
55475   EVT InVecVT = InVec.getValueType();
55476   unsigned SizeInBits = VT.getSizeInBits();
55477   unsigned InSizeInBits = InVecVT.getSizeInBits();
55478   unsigned NumSubElts = VT.getVectorNumElements();
55479   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55480 
55481   if (Subtarget.hasAVX() && !Subtarget.hasAVX2() &&
55482       TLI.isTypeLegal(InVecVT) &&
55483       InSizeInBits == 256 && InVecBC.getOpcode() == ISD::AND) {
55484     auto isConcatenatedNot = [](SDValue V) {
55485       V = peekThroughBitcasts(V);
55486       if (!isBitwiseNot(V))
55487         return false;
55488       SDValue NotOp = V->getOperand(0);
55489       return peekThroughBitcasts(NotOp).getOpcode() == ISD::CONCAT_VECTORS;
55490     };
55491     if (isConcatenatedNot(InVecBC.getOperand(0)) ||
55492         isConcatenatedNot(InVecBC.getOperand(1))) {
55493       // extract (and v4i64 X, (not (concat Y1, Y2))), n -> andnp v2i64 X(n), Y1
55494       SDValue Concat = splitVectorIntBinary(InVecBC, DAG);
55495       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT,
55496                          DAG.getBitcast(InVecVT, Concat), N->getOperand(1));
55497     }
55498   }
55499 
55500   if (DCI.isBeforeLegalizeOps())
55501     return SDValue();
55502 
55503   if (SDValue V = narrowExtractedVectorSelect(N, DAG))
55504     return V;
55505 
55506   if (ISD::isBuildVectorAllZeros(InVec.getNode()))
55507     return getZeroVector(VT, Subtarget, DAG, SDLoc(N));
55508 
55509   if (ISD::isBuildVectorAllOnes(InVec.getNode())) {
55510     if (VT.getScalarType() == MVT::i1)
55511       return DAG.getConstant(1, SDLoc(N), VT);
55512     return getOnesVector(VT, DAG, SDLoc(N));
55513   }
55514 
55515   if (InVec.getOpcode() == ISD::BUILD_VECTOR)
55516     return DAG.getBuildVector(VT, SDLoc(N),
55517                               InVec->ops().slice(IdxVal, NumSubElts));
55518 
55519   // If we are extracting from an insert into a larger vector, replace with a
55520   // smaller insert if we don't access less than the original subvector. Don't
55521   // do this for i1 vectors.
55522   // TODO: Relax the matching indices requirement?
55523   if (VT.getVectorElementType() != MVT::i1 &&
55524       InVec.getOpcode() == ISD::INSERT_SUBVECTOR && InVec.hasOneUse() &&
55525       IdxVal == InVec.getConstantOperandVal(2) &&
55526       InVec.getOperand(1).getValueSizeInBits() <= SizeInBits) {
55527     SDLoc DL(N);
55528     SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT,
55529                                  InVec.getOperand(0), N->getOperand(1));
55530     unsigned NewIdxVal = InVec.getConstantOperandVal(2) - IdxVal;
55531     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, NewExt,
55532                        InVec.getOperand(1),
55533                        DAG.getVectorIdxConstant(NewIdxVal, DL));
55534   }
55535 
55536   // If we're extracting an upper subvector from a broadcast we should just
55537   // extract the lowest subvector instead which should allow
55538   // SimplifyDemandedVectorElts do more simplifications.
55539   if (IdxVal != 0 && (InVec.getOpcode() == X86ISD::VBROADCAST ||
55540                       InVec.getOpcode() == X86ISD::VBROADCAST_LOAD ||
55541                       DAG.isSplatValue(InVec, /*AllowUndefs*/ false)))
55542     return extractSubVector(InVec, 0, DAG, SDLoc(N), SizeInBits);
55543 
55544   // If we're extracting a broadcasted subvector, just use the lowest subvector.
55545   if (IdxVal != 0 && InVec.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
55546       cast<MemIntrinsicSDNode>(InVec)->getMemoryVT() == VT)
55547     return extractSubVector(InVec, 0, DAG, SDLoc(N), SizeInBits);
55548 
55549   // Attempt to extract from the source of a shuffle vector.
55550   if ((InSizeInBits % SizeInBits) == 0 && (IdxVal % NumSubElts) == 0) {
55551     SmallVector<int, 32> ShuffleMask;
55552     SmallVector<int, 32> ScaledMask;
55553     SmallVector<SDValue, 2> ShuffleInputs;
55554     unsigned NumSubVecs = InSizeInBits / SizeInBits;
55555     // Decode the shuffle mask and scale it so its shuffling subvectors.
55556     if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) &&
55557         scaleShuffleElements(ShuffleMask, NumSubVecs, ScaledMask)) {
55558       unsigned SubVecIdx = IdxVal / NumSubElts;
55559       if (ScaledMask[SubVecIdx] == SM_SentinelUndef)
55560         return DAG.getUNDEF(VT);
55561       if (ScaledMask[SubVecIdx] == SM_SentinelZero)
55562         return getZeroVector(VT, Subtarget, DAG, SDLoc(N));
55563       SDValue Src = ShuffleInputs[ScaledMask[SubVecIdx] / NumSubVecs];
55564       if (Src.getValueSizeInBits() == InSizeInBits) {
55565         unsigned SrcSubVecIdx = ScaledMask[SubVecIdx] % NumSubVecs;
55566         unsigned SrcEltIdx = SrcSubVecIdx * NumSubElts;
55567         return extractSubVector(DAG.getBitcast(InVecVT, Src), SrcEltIdx, DAG,
55568                                 SDLoc(N), SizeInBits);
55569       }
55570     }
55571   }
55572 
55573   // If we're extracting the lowest subvector and we're the only user,
55574   // we may be able to perform this with a smaller vector width.
55575   unsigned InOpcode = InVec.getOpcode();
55576   if (InVec.hasOneUse()) {
55577     if (IdxVal == 0 && VT == MVT::v2f64 && InVecVT == MVT::v4f64) {
55578       // v2f64 CVTDQ2PD(v4i32).
55579       if (InOpcode == ISD::SINT_TO_FP &&
55580           InVec.getOperand(0).getValueType() == MVT::v4i32) {
55581         return DAG.getNode(X86ISD::CVTSI2P, SDLoc(N), VT, InVec.getOperand(0));
55582       }
55583       // v2f64 CVTUDQ2PD(v4i32).
55584       if (InOpcode == ISD::UINT_TO_FP && Subtarget.hasVLX() &&
55585           InVec.getOperand(0).getValueType() == MVT::v4i32) {
55586         return DAG.getNode(X86ISD::CVTUI2P, SDLoc(N), VT, InVec.getOperand(0));
55587       }
55588       // v2f64 CVTPS2PD(v4f32).
55589       if (InOpcode == ISD::FP_EXTEND &&
55590           InVec.getOperand(0).getValueType() == MVT::v4f32) {
55591         return DAG.getNode(X86ISD::VFPEXT, SDLoc(N), VT, InVec.getOperand(0));
55592       }
55593     }
55594     if (IdxVal == 0 &&
55595         (ISD::isExtOpcode(InOpcode) || ISD::isExtVecInRegOpcode(InOpcode)) &&
55596         (SizeInBits == 128 || SizeInBits == 256) &&
55597         InVec.getOperand(0).getValueSizeInBits() >= SizeInBits) {
55598       SDLoc DL(N);
55599       SDValue Ext = InVec.getOperand(0);
55600       if (Ext.getValueSizeInBits() > SizeInBits)
55601         Ext = extractSubVector(Ext, 0, DAG, DL, SizeInBits);
55602       unsigned ExtOp = DAG.getOpcode_EXTEND_VECTOR_INREG(InOpcode);
55603       return DAG.getNode(ExtOp, DL, VT, Ext);
55604     }
55605     if (IdxVal == 0 && InOpcode == ISD::VSELECT &&
55606         InVec.getOperand(0).getValueType().is256BitVector() &&
55607         InVec.getOperand(1).getValueType().is256BitVector() &&
55608         InVec.getOperand(2).getValueType().is256BitVector()) {
55609       SDLoc DL(N);
55610       SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
55611       SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
55612       SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
55613       return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2);
55614     }
55615     if (IdxVal == 0 && InOpcode == ISD::TRUNCATE && Subtarget.hasVLX() &&
55616         (VT.is128BitVector() || VT.is256BitVector())) {
55617       SDLoc DL(N);
55618       SDValue InVecSrc = InVec.getOperand(0);
55619       unsigned Scale = InVecSrc.getValueSizeInBits() / InSizeInBits;
55620       SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits);
55621       return DAG.getNode(InOpcode, DL, VT, Ext);
55622     }
55623     if (InOpcode == X86ISD::MOVDDUP &&
55624         (VT.is128BitVector() || VT.is256BitVector())) {
55625       SDLoc DL(N);
55626       SDValue Ext0 =
55627           extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits);
55628       return DAG.getNode(InOpcode, DL, VT, Ext0);
55629     }
55630   }
55631 
55632   // Always split vXi64 logical shifts where we're extracting the upper 32-bits
55633   // as this is very likely to fold into a shuffle/truncation.
55634   if ((InOpcode == X86ISD::VSHLI || InOpcode == X86ISD::VSRLI) &&
55635       InVecVT.getScalarSizeInBits() == 64 &&
55636       InVec.getConstantOperandAPInt(1) == 32) {
55637     SDLoc DL(N);
55638     SDValue Ext =
55639         extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits);
55640     return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1));
55641   }
55642 
55643   return SDValue();
55644 }
55645 
55646 static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
55647   EVT VT = N->getValueType(0);
55648   SDValue Src = N->getOperand(0);
55649   SDLoc DL(N);
55650 
55651   // If this is a scalar to vector to v1i1 from an AND with 1, bypass the and.
55652   // This occurs frequently in our masked scalar intrinsic code and our
55653   // floating point select lowering with AVX512.
55654   // TODO: SimplifyDemandedBits instead?
55655   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse() &&
55656       isOneConstant(Src.getOperand(1)))
55657     return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Src.getOperand(0));
55658 
55659   // Combine scalar_to_vector of an extract_vector_elt into an extract_subvec.
55660   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
55661       Src.hasOneUse() && Src.getOperand(0).getValueType().isVector() &&
55662       Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
55663       isNullConstant(Src.getOperand(1)))
55664     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0),
55665                        Src.getOperand(1));
55666 
55667   // Reduce v2i64 to v4i32 if we don't need the upper bits or are known zero.
55668   // TODO: Move to DAGCombine/SimplifyDemandedBits?
55669   if ((VT == MVT::v2i64 || VT == MVT::v2f64) && Src.hasOneUse()) {
55670     auto IsExt64 = [&DAG](SDValue Op, bool IsZeroExt) {
55671       if (Op.getValueType() != MVT::i64)
55672         return SDValue();
55673       unsigned Opc = IsZeroExt ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND;
55674       if (Op.getOpcode() == Opc &&
55675           Op.getOperand(0).getScalarValueSizeInBits() <= 32)
55676         return Op.getOperand(0);
55677       unsigned Ext = IsZeroExt ? ISD::ZEXTLOAD : ISD::EXTLOAD;
55678       if (auto *Ld = dyn_cast<LoadSDNode>(Op))
55679         if (Ld->getExtensionType() == Ext &&
55680             Ld->getMemoryVT().getScalarSizeInBits() <= 32)
55681           return Op;
55682       if (IsZeroExt) {
55683         KnownBits Known = DAG.computeKnownBits(Op);
55684         if (!Known.isConstant() && Known.countMinLeadingZeros() >= 32)
55685           return Op;
55686       }
55687       return SDValue();
55688     };
55689 
55690     if (SDValue AnyExt = IsExt64(peekThroughOneUseBitcasts(Src), false))
55691       return DAG.getBitcast(
55692           VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
55693                           DAG.getAnyExtOrTrunc(AnyExt, DL, MVT::i32)));
55694 
55695     if (SDValue ZeroExt = IsExt64(peekThroughOneUseBitcasts(Src), true))
55696       return DAG.getBitcast(
55697           VT,
55698           DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v4i32,
55699                       DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
55700                                   DAG.getZExtOrTrunc(ZeroExt, DL, MVT::i32))));
55701   }
55702 
55703   // Combine (v2i64 (scalar_to_vector (i64 (bitconvert (mmx))))) to MOVQ2DQ.
55704   if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST &&
55705       Src.getOperand(0).getValueType() == MVT::x86mmx)
55706     return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0));
55707 
55708   // See if we're broadcasting the scalar value, in which case just reuse that.
55709   // Ensure the same SDValue from the SDNode use is being used.
55710   if (VT.getScalarType() == Src.getValueType())
55711     for (SDNode *User : Src->uses())
55712       if (User->getOpcode() == X86ISD::VBROADCAST &&
55713           Src == User->getOperand(0)) {
55714         unsigned SizeInBits = VT.getFixedSizeInBits();
55715         unsigned BroadcastSizeInBits =
55716             User->getValueSizeInBits(0).getFixedValue();
55717         if (BroadcastSizeInBits == SizeInBits)
55718           return SDValue(User, 0);
55719         if (BroadcastSizeInBits > SizeInBits)
55720           return extractSubVector(SDValue(User, 0), 0, DAG, DL, SizeInBits);
55721         // TODO: Handle BroadcastSizeInBits < SizeInBits when we have test
55722         // coverage.
55723       }
55724 
55725   return SDValue();
55726 }
55727 
55728 // Simplify PMULDQ and PMULUDQ operations.
55729 static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
55730                              TargetLowering::DAGCombinerInfo &DCI,
55731                              const X86Subtarget &Subtarget) {
55732   SDValue LHS = N->getOperand(0);
55733   SDValue RHS = N->getOperand(1);
55734 
55735   // Canonicalize constant to RHS.
55736   if (DAG.isConstantIntBuildVectorOrConstantInt(LHS) &&
55737       !DAG.isConstantIntBuildVectorOrConstantInt(RHS))
55738     return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), RHS, LHS);
55739 
55740   // Multiply by zero.
55741   // Don't return RHS as it may contain UNDEFs.
55742   if (ISD::isBuildVectorAllZeros(RHS.getNode()))
55743     return DAG.getConstant(0, SDLoc(N), N->getValueType(0));
55744 
55745   // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
55746   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55747   if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(64), DCI))
55748     return SDValue(N, 0);
55749 
55750   // If the input is an extend_invec and the SimplifyDemandedBits call didn't
55751   // convert it to any_extend_invec, due to the LegalOperations check, do the
55752   // conversion directly to a vector shuffle manually. This exposes combine
55753   // opportunities missed by combineEXTEND_VECTOR_INREG not calling
55754   // combineX86ShufflesRecursively on SSE4.1 targets.
55755   // FIXME: This is basically a hack around several other issues related to
55756   // ANY_EXTEND_VECTOR_INREG.
55757   if (N->getValueType(0) == MVT::v2i64 && LHS.hasOneUse() &&
55758       (LHS.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
55759        LHS.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) &&
55760       LHS.getOperand(0).getValueType() == MVT::v4i32) {
55761     SDLoc dl(N);
55762     LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0),
55763                                LHS.getOperand(0), { 0, -1, 1, -1 });
55764     LHS = DAG.getBitcast(MVT::v2i64, LHS);
55765     return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
55766   }
55767   if (N->getValueType(0) == MVT::v2i64 && RHS.hasOneUse() &&
55768       (RHS.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
55769        RHS.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) &&
55770       RHS.getOperand(0).getValueType() == MVT::v4i32) {
55771     SDLoc dl(N);
55772     RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0),
55773                                RHS.getOperand(0), { 0, -1, 1, -1 });
55774     RHS = DAG.getBitcast(MVT::v2i64, RHS);
55775     return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
55776   }
55777 
55778   return SDValue();
55779 }
55780 
55781 // Simplify VPMADDUBSW/VPMADDWD operations.
55782 static SDValue combineVPMADD(SDNode *N, SelectionDAG &DAG,
55783                              TargetLowering::DAGCombinerInfo &DCI) {
55784   EVT VT = N->getValueType(0);
55785   SDValue LHS = N->getOperand(0);
55786   SDValue RHS = N->getOperand(1);
55787 
55788   // Multiply by zero.
55789   // Don't return LHS/RHS as it may contain UNDEFs.
55790   if (ISD::isBuildVectorAllZeros(LHS.getNode()) ||
55791       ISD::isBuildVectorAllZeros(RHS.getNode()))
55792     return DAG.getConstant(0, SDLoc(N), VT);
55793 
55794   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55795   APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
55796   if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
55797     return SDValue(N, 0);
55798 
55799   return SDValue();
55800 }
55801 
55802 static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG,
55803                                           TargetLowering::DAGCombinerInfo &DCI,
55804                                           const X86Subtarget &Subtarget) {
55805   EVT VT = N->getValueType(0);
55806   SDValue In = N->getOperand(0);
55807   unsigned Opcode = N->getOpcode();
55808   unsigned InOpcode = In.getOpcode();
55809   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55810   SDLoc DL(N);
55811 
55812   // Try to merge vector loads and extend_inreg to an extload.
55813   if (!DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(In.getNode()) &&
55814       In.hasOneUse()) {
55815     auto *Ld = cast<LoadSDNode>(In);
55816     if (Ld->isSimple()) {
55817       MVT SVT = In.getSimpleValueType().getVectorElementType();
55818       ISD::LoadExtType Ext = Opcode == ISD::SIGN_EXTEND_VECTOR_INREG
55819                                  ? ISD::SEXTLOAD
55820                                  : ISD::ZEXTLOAD;
55821       EVT MemVT = VT.changeVectorElementType(SVT);
55822       if (TLI.isLoadExtLegal(Ext, VT, MemVT)) {
55823         SDValue Load = DAG.getExtLoad(
55824             Ext, DL, VT, Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(),
55825             MemVT, Ld->getOriginalAlign(), Ld->getMemOperand()->getFlags());
55826         DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
55827         return Load;
55828       }
55829     }
55830   }
55831 
55832   // Fold EXTEND_VECTOR_INREG(EXTEND_VECTOR_INREG(X)) -> EXTEND_VECTOR_INREG(X).
55833   if (Opcode == InOpcode)
55834     return DAG.getNode(Opcode, DL, VT, In.getOperand(0));
55835 
55836   // Fold EXTEND_VECTOR_INREG(EXTRACT_SUBVECTOR(EXTEND(X),0))
55837   // -> EXTEND_VECTOR_INREG(X).
55838   // TODO: Handle non-zero subvector indices.
55839   if (InOpcode == ISD::EXTRACT_SUBVECTOR && In.getConstantOperandVal(1) == 0 &&
55840       In.getOperand(0).getOpcode() == DAG.getOpcode_EXTEND(Opcode) &&
55841       In.getOperand(0).getOperand(0).getValueSizeInBits() ==
55842           In.getValueSizeInBits())
55843     return DAG.getNode(Opcode, DL, VT, In.getOperand(0).getOperand(0));
55844 
55845   // Fold EXTEND_VECTOR_INREG(BUILD_VECTOR(X,Y,?,?)) -> BUILD_VECTOR(X,0,Y,0).
55846   // TODO: Move to DAGCombine?
55847   if (!DCI.isBeforeLegalizeOps() && Opcode == ISD::ZERO_EXTEND_VECTOR_INREG &&
55848       In.getOpcode() == ISD::BUILD_VECTOR && In.hasOneUse() &&
55849       In.getValueSizeInBits() == VT.getSizeInBits()) {
55850     unsigned NumElts = VT.getVectorNumElements();
55851     unsigned Scale = VT.getScalarSizeInBits() / In.getScalarValueSizeInBits();
55852     EVT EltVT = In.getOperand(0).getValueType();
55853     SmallVector<SDValue> Elts(Scale * NumElts, DAG.getConstant(0, DL, EltVT));
55854     for (unsigned I = 0; I != NumElts; ++I)
55855       Elts[I * Scale] = In.getOperand(I);
55856     return DAG.getBitcast(VT, DAG.getBuildVector(In.getValueType(), DL, Elts));
55857   }
55858 
55859   // Attempt to combine as a shuffle on SSE41+ targets.
55860   if (Subtarget.hasSSE41()) {
55861     SDValue Op(N, 0);
55862     if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getValueType()))
55863       if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
55864         return Res;
55865   }
55866 
55867   return SDValue();
55868 }
55869 
55870 static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
55871                              TargetLowering::DAGCombinerInfo &DCI) {
55872   EVT VT = N->getValueType(0);
55873 
55874   if (ISD::isBuildVectorAllZeros(N->getOperand(0).getNode()))
55875     return DAG.getConstant(0, SDLoc(N), VT);
55876 
55877   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55878   APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
55879   if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
55880     return SDValue(N, 0);
55881 
55882   return SDValue();
55883 }
55884 
55885 // Optimize (fp16_to_fp (fp_to_fp16 X)) to VCVTPS2PH followed by VCVTPH2PS.
55886 // Done as a combine because the lowering for fp16_to_fp and fp_to_fp16 produce
55887 // extra instructions between the conversion due to going to scalar and back.
55888 static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG,
55889                                  const X86Subtarget &Subtarget) {
55890   if (Subtarget.useSoftFloat() || !Subtarget.hasF16C())
55891     return SDValue();
55892 
55893   if (N->getOperand(0).getOpcode() != ISD::FP_TO_FP16)
55894     return SDValue();
55895 
55896   if (N->getValueType(0) != MVT::f32 ||
55897       N->getOperand(0).getOperand(0).getValueType() != MVT::f32)
55898     return SDValue();
55899 
55900   SDLoc dl(N);
55901   SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32,
55902                             N->getOperand(0).getOperand(0));
55903   Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
55904                     DAG.getTargetConstant(4, dl, MVT::i32));
55905   Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
55906   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
55907                      DAG.getIntPtrConstant(0, dl));
55908 }
55909 
55910 static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
55911                                 const X86Subtarget &Subtarget) {
55912   if (!Subtarget.hasF16C() || Subtarget.useSoftFloat())
55913     return SDValue();
55914 
55915   if (Subtarget.hasFP16())
55916     return SDValue();
55917 
55918   bool IsStrict = N->isStrictFPOpcode();
55919   EVT VT = N->getValueType(0);
55920   SDValue Src = N->getOperand(IsStrict ? 1 : 0);
55921   EVT SrcVT = Src.getValueType();
55922 
55923   if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::f16)
55924     return SDValue();
55925 
55926   if (VT.getVectorElementType() != MVT::f32 &&
55927       VT.getVectorElementType() != MVT::f64)
55928     return SDValue();
55929 
55930   unsigned NumElts = VT.getVectorNumElements();
55931   if (NumElts == 1 || !isPowerOf2_32(NumElts))
55932     return SDValue();
55933 
55934   SDLoc dl(N);
55935 
55936   // Convert the input to vXi16.
55937   EVT IntVT = SrcVT.changeVectorElementTypeToInteger();
55938   Src = DAG.getBitcast(IntVT, Src);
55939 
55940   // Widen to at least 8 input elements.
55941   if (NumElts < 8) {
55942     unsigned NumConcats = 8 / NumElts;
55943     SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT)
55944                                 : DAG.getConstant(0, dl, IntVT);
55945     SmallVector<SDValue, 4> Ops(NumConcats, Fill);
55946     Ops[0] = Src;
55947     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops);
55948   }
55949 
55950   // Destination is vXf32 with at least 4 elements.
55951   EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32,
55952                                std::max(4U, NumElts));
55953   SDValue Cvt, Chain;
55954   if (IsStrict) {
55955     Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other},
55956                       {N->getOperand(0), Src});
55957     Chain = Cvt.getValue(1);
55958   } else {
55959     Cvt = DAG.getNode(X86ISD::CVTPH2PS, dl, CvtVT, Src);
55960   }
55961 
55962   if (NumElts < 4) {
55963     assert(NumElts == 2 && "Unexpected size");
55964     Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Cvt,
55965                       DAG.getIntPtrConstant(0, dl));
55966   }
55967 
55968   if (IsStrict) {
55969     // Extend to the original VT if necessary.
55970     if (Cvt.getValueType() != VT) {
55971       Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other},
55972                         {Chain, Cvt});
55973       Chain = Cvt.getValue(1);
55974     }
55975     return DAG.getMergeValues({Cvt, Chain}, dl);
55976   }
55977 
55978   // Extend to the original VT if necessary.
55979   return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt);
55980 }
55981 
55982 // Try to find a larger VBROADCAST_LOAD/SUBV_BROADCAST_LOAD that we can extract
55983 // from. Limit this to cases where the loads have the same input chain and the
55984 // output chains are unused. This avoids any memory ordering issues.
55985 static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG,
55986                                      TargetLowering::DAGCombinerInfo &DCI) {
55987   assert((N->getOpcode() == X86ISD::VBROADCAST_LOAD ||
55988           N->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) &&
55989          "Unknown broadcast load type");
55990 
55991   // Only do this if the chain result is unused.
55992   if (N->hasAnyUseOfValue(1))
55993     return SDValue();
55994 
55995   auto *MemIntrin = cast<MemIntrinsicSDNode>(N);
55996 
55997   SDValue Ptr = MemIntrin->getBasePtr();
55998   SDValue Chain = MemIntrin->getChain();
55999   EVT VT = N->getSimpleValueType(0);
56000   EVT MemVT = MemIntrin->getMemoryVT();
56001 
56002   // Look at other users of our base pointer and try to find a wider broadcast.
56003   // The input chain and the size of the memory VT must match.
56004   for (SDNode *User : Ptr->uses())
56005     if (User != N && User->getOpcode() == N->getOpcode() &&
56006         cast<MemIntrinsicSDNode>(User)->getBasePtr() == Ptr &&
56007         cast<MemIntrinsicSDNode>(User)->getChain() == Chain &&
56008         cast<MemIntrinsicSDNode>(User)->getMemoryVT().getSizeInBits() ==
56009             MemVT.getSizeInBits() &&
56010         !User->hasAnyUseOfValue(1) &&
56011         User->getValueSizeInBits(0).getFixedValue() > VT.getFixedSizeInBits()) {
56012       SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N),
56013                                          VT.getSizeInBits());
56014       Extract = DAG.getBitcast(VT, Extract);
56015       return DCI.CombineTo(N, Extract, SDValue(User, 1));
56016     }
56017 
56018   return SDValue();
56019 }
56020 
56021 static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG,
56022                                const X86Subtarget &Subtarget) {
56023   if (!Subtarget.hasF16C() || Subtarget.useSoftFloat())
56024     return SDValue();
56025 
56026   bool IsStrict = N->isStrictFPOpcode();
56027   EVT VT = N->getValueType(0);
56028   SDValue Src = N->getOperand(IsStrict ? 1 : 0);
56029   EVT SrcVT = Src.getValueType();
56030 
56031   if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
56032       SrcVT.getVectorElementType() != MVT::f32)
56033     return SDValue();
56034 
56035   SDLoc dl(N);
56036 
56037   SDValue Cvt, Chain;
56038   unsigned NumElts = VT.getVectorNumElements();
56039   if (Subtarget.hasFP16()) {
56040     // Combine (v8f16 fp_round(concat_vectors(v4f32 (xint_to_fp v4i64), ..)))
56041     // into (v8f16 vector_shuffle(v8f16 (CVTXI2P v4i64), ..))
56042     if (NumElts == 8 && Src.getOpcode() == ISD::CONCAT_VECTORS) {
56043       SDValue Cvt0, Cvt1;
56044       SDValue Op0 = Src.getOperand(0);
56045       SDValue Op1 = Src.getOperand(1);
56046       bool IsOp0Strict = Op0->isStrictFPOpcode();
56047       if (Op0.getOpcode() != Op1.getOpcode() ||
56048           Op0.getOperand(IsOp0Strict ? 1 : 0).getValueType() != MVT::v4i64 ||
56049           Op1.getOperand(IsOp0Strict ? 1 : 0).getValueType() != MVT::v4i64) {
56050         return SDValue();
56051       }
56052       int Mask[8] = {0, 1, 2, 3, 8, 9, 10, 11};
56053       if (IsStrict) {
56054         assert(IsOp0Strict && "Op0 must be strict node");
56055         unsigned Opc = Op0.getOpcode() == ISD::STRICT_SINT_TO_FP
56056                            ? X86ISD::STRICT_CVTSI2P
56057                            : X86ISD::STRICT_CVTUI2P;
56058         Cvt0 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
56059                            {Op0.getOperand(0), Op0.getOperand(1)});
56060         Cvt1 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
56061                            {Op1.getOperand(0), Op1.getOperand(1)});
56062         Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
56063         return DAG.getMergeValues({Cvt, Cvt0.getValue(1)}, dl);
56064       }
56065       unsigned Opc = Op0.getOpcode() == ISD::SINT_TO_FP ? X86ISD::CVTSI2P
56066                                                         : X86ISD::CVTUI2P;
56067       Cvt0 = DAG.getNode(Opc, dl, MVT::v8f16, Op0.getOperand(0));
56068       Cvt1 = DAG.getNode(Opc, dl, MVT::v8f16, Op1.getOperand(0));
56069       return Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
56070     }
56071     return SDValue();
56072   }
56073 
56074   if (NumElts == 1 || !isPowerOf2_32(NumElts))
56075     return SDValue();
56076 
56077   // Widen to at least 4 input elements.
56078   if (NumElts < 4)
56079     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
56080                       DAG.getConstantFP(0.0, dl, SrcVT));
56081 
56082   // Destination is v8i16 with at least 8 elements.
56083   EVT CvtVT =
56084       EVT::getVectorVT(*DAG.getContext(), MVT::i16, std::max(8U, NumElts));
56085   SDValue Rnd = DAG.getTargetConstant(4, dl, MVT::i32);
56086   if (IsStrict) {
56087     Cvt = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {CvtVT, MVT::Other},
56088                       {N->getOperand(0), Src, Rnd});
56089     Chain = Cvt.getValue(1);
56090   } else {
56091     Cvt = DAG.getNode(X86ISD::CVTPS2PH, dl, CvtVT, Src, Rnd);
56092   }
56093 
56094   // Extract down to real number of elements.
56095   if (NumElts < 8) {
56096     EVT IntVT = VT.changeVectorElementTypeToInteger();
56097     Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, IntVT, Cvt,
56098                       DAG.getIntPtrConstant(0, dl));
56099   }
56100 
56101   Cvt = DAG.getBitcast(VT, Cvt);
56102 
56103   if (IsStrict)
56104     return DAG.getMergeValues({Cvt, Chain}, dl);
56105 
56106   return Cvt;
56107 }
56108 
56109 static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG) {
56110   SDValue Src = N->getOperand(0);
56111 
56112   // Turn MOVDQ2Q+simple_load into an mmx load.
56113   if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) {
56114     LoadSDNode *LN = cast<LoadSDNode>(Src.getNode());
56115 
56116     if (LN->isSimple()) {
56117       SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(),
56118                                   LN->getBasePtr(),
56119                                   LN->getPointerInfo(),
56120                                   LN->getOriginalAlign(),
56121                                   LN->getMemOperand()->getFlags());
56122       DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), NewLd.getValue(1));
56123       return NewLd;
56124     }
56125   }
56126 
56127   return SDValue();
56128 }
56129 
56130 static SDValue combinePDEP(SDNode *N, SelectionDAG &DAG,
56131                            TargetLowering::DAGCombinerInfo &DCI) {
56132   unsigned NumBits = N->getSimpleValueType(0).getSizeInBits();
56133   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
56134   if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumBits), DCI))
56135     return SDValue(N, 0);
56136 
56137   return SDValue();
56138 }
56139 
56140 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
56141                                              DAGCombinerInfo &DCI) const {
56142   SelectionDAG &DAG = DCI.DAG;
56143   switch (N->getOpcode()) {
56144   default: break;
56145   case ISD::SCALAR_TO_VECTOR:
56146     return combineScalarToVector(N, DAG);
56147   case ISD::EXTRACT_VECTOR_ELT:
56148   case X86ISD::PEXTRW:
56149   case X86ISD::PEXTRB:
56150     return combineExtractVectorElt(N, DAG, DCI, Subtarget);
56151   case ISD::CONCAT_VECTORS:
56152     return combineCONCAT_VECTORS(N, DAG, DCI, Subtarget);
56153   case ISD::INSERT_SUBVECTOR:
56154     return combineINSERT_SUBVECTOR(N, DAG, DCI, Subtarget);
56155   case ISD::EXTRACT_SUBVECTOR:
56156     return combineEXTRACT_SUBVECTOR(N, DAG, DCI, Subtarget);
56157   case ISD::VSELECT:
56158   case ISD::SELECT:
56159   case X86ISD::BLENDV:      return combineSelect(N, DAG, DCI, Subtarget);
56160   case ISD::BITCAST:        return combineBitcast(N, DAG, DCI, Subtarget);
56161   case X86ISD::CMOV:        return combineCMov(N, DAG, DCI, Subtarget);
56162   case X86ISD::CMP:         return combineCMP(N, DAG, Subtarget);
56163   case ISD::ADD:            return combineAdd(N, DAG, DCI, Subtarget);
56164   case ISD::SUB:            return combineSub(N, DAG, DCI, Subtarget);
56165   case X86ISD::ADD:
56166   case X86ISD::SUB:         return combineX86AddSub(N, DAG, DCI);
56167   case X86ISD::SBB:         return combineSBB(N, DAG);
56168   case X86ISD::ADC:         return combineADC(N, DAG, DCI);
56169   case ISD::MUL:            return combineMul(N, DAG, DCI, Subtarget);
56170   case ISD::SHL:            return combineShiftLeft(N, DAG);
56171   case ISD::SRA:            return combineShiftRightArithmetic(N, DAG, Subtarget);
56172   case ISD::SRL:            return combineShiftRightLogical(N, DAG, DCI, Subtarget);
56173   case ISD::AND:            return combineAnd(N, DAG, DCI, Subtarget);
56174   case ISD::OR:             return combineOr(N, DAG, DCI, Subtarget);
56175   case ISD::XOR:            return combineXor(N, DAG, DCI, Subtarget);
56176   case ISD::BITREVERSE:     return combineBITREVERSE(N, DAG, DCI, Subtarget);
56177   case X86ISD::BEXTR:
56178   case X86ISD::BEXTRI:      return combineBEXTR(N, DAG, DCI, Subtarget);
56179   case ISD::LOAD:           return combineLoad(N, DAG, DCI, Subtarget);
56180   case ISD::MLOAD:          return combineMaskedLoad(N, DAG, DCI, Subtarget);
56181   case ISD::STORE:          return combineStore(N, DAG, DCI, Subtarget);
56182   case ISD::MSTORE:         return combineMaskedStore(N, DAG, DCI, Subtarget);
56183   case X86ISD::VEXTRACT_STORE:
56184     return combineVEXTRACT_STORE(N, DAG, DCI, Subtarget);
56185   case ISD::SINT_TO_FP:
56186   case ISD::STRICT_SINT_TO_FP:
56187     return combineSIntToFP(N, DAG, DCI, Subtarget);
56188   case ISD::UINT_TO_FP:
56189   case ISD::STRICT_UINT_TO_FP:
56190     return combineUIntToFP(N, DAG, Subtarget);
56191   case ISD::FADD:
56192   case ISD::FSUB:           return combineFaddFsub(N, DAG, Subtarget);
56193   case X86ISD::VFCMULC:
56194   case X86ISD::VFMULC:      return combineFMulcFCMulc(N, DAG, Subtarget);
56195   case ISD::FNEG:           return combineFneg(N, DAG, DCI, Subtarget);
56196   case ISD::TRUNCATE:       return combineTruncate(N, DAG, Subtarget);
56197   case X86ISD::VTRUNC:      return combineVTRUNC(N, DAG, DCI);
56198   case X86ISD::ANDNP:       return combineAndnp(N, DAG, DCI, Subtarget);
56199   case X86ISD::FAND:        return combineFAnd(N, DAG, Subtarget);
56200   case X86ISD::FANDN:       return combineFAndn(N, DAG, Subtarget);
56201   case X86ISD::FXOR:
56202   case X86ISD::FOR:         return combineFOr(N, DAG, DCI, Subtarget);
56203   case X86ISD::FMIN:
56204   case X86ISD::FMAX:        return combineFMinFMax(N, DAG);
56205   case ISD::FMINNUM:
56206   case ISD::FMAXNUM:        return combineFMinNumFMaxNum(N, DAG, Subtarget);
56207   case X86ISD::CVTSI2P:
56208   case X86ISD::CVTUI2P:     return combineX86INT_TO_FP(N, DAG, DCI);
56209   case X86ISD::CVTP2SI:
56210   case X86ISD::CVTP2UI:
56211   case X86ISD::STRICT_CVTTP2SI:
56212   case X86ISD::CVTTP2SI:
56213   case X86ISD::STRICT_CVTTP2UI:
56214   case X86ISD::CVTTP2UI:
56215                             return combineCVTP2I_CVTTP2I(N, DAG, DCI);
56216   case X86ISD::STRICT_CVTPH2PS:
56217   case X86ISD::CVTPH2PS:    return combineCVTPH2PS(N, DAG, DCI);
56218   case X86ISD::BT:          return combineBT(N, DAG, DCI);
56219   case ISD::ANY_EXTEND:
56220   case ISD::ZERO_EXTEND:    return combineZext(N, DAG, DCI, Subtarget);
56221   case ISD::SIGN_EXTEND:    return combineSext(N, DAG, DCI, Subtarget);
56222   case ISD::SIGN_EXTEND_INREG: return combineSignExtendInReg(N, DAG, Subtarget);
56223   case ISD::ANY_EXTEND_VECTOR_INREG:
56224   case ISD::SIGN_EXTEND_VECTOR_INREG:
56225   case ISD::ZERO_EXTEND_VECTOR_INREG:
56226     return combineEXTEND_VECTOR_INREG(N, DAG, DCI, Subtarget);
56227   case ISD::SETCC:          return combineSetCC(N, DAG, DCI, Subtarget);
56228   case X86ISD::SETCC:       return combineX86SetCC(N, DAG, Subtarget);
56229   case X86ISD::BRCOND:      return combineBrCond(N, DAG, Subtarget);
56230   case X86ISD::PACKSS:
56231   case X86ISD::PACKUS:      return combineVectorPack(N, DAG, DCI, Subtarget);
56232   case X86ISD::HADD:
56233   case X86ISD::HSUB:
56234   case X86ISD::FHADD:
56235   case X86ISD::FHSUB:       return combineVectorHADDSUB(N, DAG, DCI, Subtarget);
56236   case X86ISD::VSHL:
56237   case X86ISD::VSRA:
56238   case X86ISD::VSRL:
56239     return combineVectorShiftVar(N, DAG, DCI, Subtarget);
56240   case X86ISD::VSHLI:
56241   case X86ISD::VSRAI:
56242   case X86ISD::VSRLI:
56243     return combineVectorShiftImm(N, DAG, DCI, Subtarget);
56244   case ISD::INSERT_VECTOR_ELT:
56245   case X86ISD::PINSRB:
56246   case X86ISD::PINSRW:      return combineVectorInsert(N, DAG, DCI, Subtarget);
56247   case X86ISD::SHUFP:       // Handle all target specific shuffles
56248   case X86ISD::INSERTPS:
56249   case X86ISD::EXTRQI:
56250   case X86ISD::INSERTQI:
56251   case X86ISD::VALIGN:
56252   case X86ISD::PALIGNR:
56253   case X86ISD::VSHLDQ:
56254   case X86ISD::VSRLDQ:
56255   case X86ISD::BLENDI:
56256   case X86ISD::UNPCKH:
56257   case X86ISD::UNPCKL:
56258   case X86ISD::MOVHLPS:
56259   case X86ISD::MOVLHPS:
56260   case X86ISD::PSHUFB:
56261   case X86ISD::PSHUFD:
56262   case X86ISD::PSHUFHW:
56263   case X86ISD::PSHUFLW:
56264   case X86ISD::MOVSHDUP:
56265   case X86ISD::MOVSLDUP:
56266   case X86ISD::MOVDDUP:
56267   case X86ISD::MOVSS:
56268   case X86ISD::MOVSD:
56269   case X86ISD::MOVSH:
56270   case X86ISD::VBROADCAST:
56271   case X86ISD::VPPERM:
56272   case X86ISD::VPERMI:
56273   case X86ISD::VPERMV:
56274   case X86ISD::VPERMV3:
56275   case X86ISD::VPERMIL2:
56276   case X86ISD::VPERMILPI:
56277   case X86ISD::VPERMILPV:
56278   case X86ISD::VPERM2X128:
56279   case X86ISD::SHUF128:
56280   case X86ISD::VZEXT_MOVL:
56281   case ISD::VECTOR_SHUFFLE: return combineShuffle(N, DAG, DCI,Subtarget);
56282   case X86ISD::FMADD_RND:
56283   case X86ISD::FMSUB:
56284   case X86ISD::STRICT_FMSUB:
56285   case X86ISD::FMSUB_RND:
56286   case X86ISD::FNMADD:
56287   case X86ISD::STRICT_FNMADD:
56288   case X86ISD::FNMADD_RND:
56289   case X86ISD::FNMSUB:
56290   case X86ISD::STRICT_FNMSUB:
56291   case X86ISD::FNMSUB_RND:
56292   case ISD::FMA:
56293   case ISD::STRICT_FMA:     return combineFMA(N, DAG, DCI, Subtarget);
56294   case X86ISD::FMADDSUB_RND:
56295   case X86ISD::FMSUBADD_RND:
56296   case X86ISD::FMADDSUB:
56297   case X86ISD::FMSUBADD:    return combineFMADDSUB(N, DAG, DCI);
56298   case X86ISD::MOVMSK:      return combineMOVMSK(N, DAG, DCI, Subtarget);
56299   case X86ISD::TESTP:       return combineTESTP(N, DAG, DCI, Subtarget);
56300   case X86ISD::MGATHER:
56301   case X86ISD::MSCATTER:    return combineX86GatherScatter(N, DAG, DCI);
56302   case ISD::MGATHER:
56303   case ISD::MSCATTER:       return combineGatherScatter(N, DAG, DCI);
56304   case X86ISD::PCMPEQ:
56305   case X86ISD::PCMPGT:      return combineVectorCompare(N, DAG, Subtarget);
56306   case X86ISD::PMULDQ:
56307   case X86ISD::PMULUDQ:     return combinePMULDQ(N, DAG, DCI, Subtarget);
56308   case X86ISD::VPMADDUBSW:
56309   case X86ISD::VPMADDWD:    return combineVPMADD(N, DAG, DCI);
56310   case X86ISD::KSHIFTL:
56311   case X86ISD::KSHIFTR:     return combineKSHIFT(N, DAG, DCI);
56312   case ISD::FP16_TO_FP:     return combineFP16_TO_FP(N, DAG, Subtarget);
56313   case ISD::STRICT_FP_EXTEND:
56314   case ISD::FP_EXTEND:      return combineFP_EXTEND(N, DAG, Subtarget);
56315   case ISD::STRICT_FP_ROUND:
56316   case ISD::FP_ROUND:       return combineFP_ROUND(N, DAG, Subtarget);
56317   case X86ISD::VBROADCAST_LOAD:
56318   case X86ISD::SUBV_BROADCAST_LOAD: return combineBROADCAST_LOAD(N, DAG, DCI);
56319   case X86ISD::MOVDQ2Q:     return combineMOVDQ2Q(N, DAG);
56320   case X86ISD::PDEP:        return combinePDEP(N, DAG, DCI);
56321   }
56322 
56323   return SDValue();
56324 }
56325 
56326 bool X86TargetLowering::preferABDSToABSWithNSW(EVT VT) const {
56327   return false;
56328 }
56329 
56330 // Prefer (non-AVX512) vector TRUNCATE(SIGN_EXTEND_INREG(X)) to use of PACKSS.
56331 bool X86TargetLowering::preferSextInRegOfTruncate(EVT TruncVT, EVT VT,
56332                                                   EVT ExtVT) const {
56333   return Subtarget.hasAVX512() || !VT.isVector();
56334 }
56335 
56336 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
56337   if (!isTypeLegal(VT))
56338     return false;
56339 
56340   // There are no vXi8 shifts.
56341   if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8)
56342     return false;
56343 
56344   // TODO: Almost no 8-bit ops are desirable because they have no actual
56345   //       size/speed advantages vs. 32-bit ops, but they do have a major
56346   //       potential disadvantage by causing partial register stalls.
56347   //
56348   // 8-bit multiply/shl is probably not cheaper than 32-bit multiply/shl, and
56349   // we have specializations to turn 32-bit multiply/shl into LEA or other ops.
56350   // Also, see the comment in "IsDesirableToPromoteOp" - where we additionally
56351   // check for a constant operand to the multiply.
56352   if ((Opc == ISD::MUL || Opc == ISD::SHL) && VT == MVT::i8)
56353     return false;
56354 
56355   // i16 instruction encodings are longer and some i16 instructions are slow,
56356   // so those are not desirable.
56357   if (VT == MVT::i16) {
56358     switch (Opc) {
56359     default:
56360       break;
56361     case ISD::LOAD:
56362     case ISD::SIGN_EXTEND:
56363     case ISD::ZERO_EXTEND:
56364     case ISD::ANY_EXTEND:
56365     case ISD::SHL:
56366     case ISD::SRA:
56367     case ISD::SRL:
56368     case ISD::SUB:
56369     case ISD::ADD:
56370     case ISD::MUL:
56371     case ISD::AND:
56372     case ISD::OR:
56373     case ISD::XOR:
56374       return false;
56375     }
56376   }
56377 
56378   // Any legal type not explicitly accounted for above here is desirable.
56379   return true;
56380 }
56381 
56382 SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc &dl,
56383                                                   SDValue Value, SDValue Addr,
56384                                                   int JTI,
56385                                                   SelectionDAG &DAG) const {
56386   const Module *M = DAG.getMachineFunction().getMMI().getModule();
56387   Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
56388   if (IsCFProtectionSupported) {
56389     // In case control-flow branch protection is enabled, we need to add
56390     // notrack prefix to the indirect branch.
56391     // In order to do that we create NT_BRIND SDNode.
56392     // Upon ISEL, the pattern will convert it to jmp with NoTrack prefix.
56393     SDValue JTInfo = DAG.getJumpTableDebugInfo(JTI, Value, dl);
56394     return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, JTInfo, Addr);
56395   }
56396 
56397   return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);
56398 }
56399 
56400 TargetLowering::AndOrSETCCFoldKind
56401 X86TargetLowering::isDesirableToCombineLogicOpOfSETCC(
56402     const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const {
56403   using AndOrSETCCFoldKind = TargetLowering::AndOrSETCCFoldKind;
56404   EVT VT = LogicOp->getValueType(0);
56405   EVT OpVT = SETCC0->getOperand(0).getValueType();
56406   if (!VT.isInteger())
56407     return AndOrSETCCFoldKind::None;
56408 
56409   if (VT.isVector())
56410     return AndOrSETCCFoldKind(AndOrSETCCFoldKind::NotAnd |
56411                               (isOperationLegal(ISD::ABS, OpVT)
56412                                    ? AndOrSETCCFoldKind::ABS
56413                                    : AndOrSETCCFoldKind::None));
56414 
56415   // Don't use `NotAnd` as even though `not` is generally shorter code size than
56416   // `add`, `add` can lower to LEA which can save moves / spills. Any case where
56417   // `NotAnd` applies, `AddAnd` does as well.
56418   // TODO: Currently we lower (icmp eq/ne (and ~X, Y), 0) -> `test (not X), Y`,
56419   // if we change that to `andn Y, X` it may be worth prefering `NotAnd` here.
56420   return AndOrSETCCFoldKind::AddAnd;
56421 }
56422 
56423 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
56424   EVT VT = Op.getValueType();
56425   bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL &&
56426                              isa<ConstantSDNode>(Op.getOperand(1));
56427 
56428   // i16 is legal, but undesirable since i16 instruction encodings are longer
56429   // and some i16 instructions are slow.
56430   // 8-bit multiply-by-constant can usually be expanded to something cheaper
56431   // using LEA and/or other ALU ops.
56432   if (VT != MVT::i16 && !Is8BitMulByConstant)
56433     return false;
56434 
56435   auto IsFoldableRMW = [](SDValue Load, SDValue Op) {
56436     if (!Op.hasOneUse())
56437       return false;
56438     SDNode *User = *Op->use_begin();
56439     if (!ISD::isNormalStore(User))
56440       return false;
56441     auto *Ld = cast<LoadSDNode>(Load);
56442     auto *St = cast<StoreSDNode>(User);
56443     return Ld->getBasePtr() == St->getBasePtr();
56444   };
56445 
56446   auto IsFoldableAtomicRMW = [](SDValue Load, SDValue Op) {
56447     if (!Load.hasOneUse() || Load.getOpcode() != ISD::ATOMIC_LOAD)
56448       return false;
56449     if (!Op.hasOneUse())
56450       return false;
56451     SDNode *User = *Op->use_begin();
56452     if (User->getOpcode() != ISD::ATOMIC_STORE)
56453       return false;
56454     auto *Ld = cast<AtomicSDNode>(Load);
56455     auto *St = cast<AtomicSDNode>(User);
56456     return Ld->getBasePtr() == St->getBasePtr();
56457   };
56458 
56459   bool Commute = false;
56460   switch (Op.getOpcode()) {
56461   default: return false;
56462   case ISD::SIGN_EXTEND:
56463   case ISD::ZERO_EXTEND:
56464   case ISD::ANY_EXTEND:
56465     break;
56466   case ISD::SHL:
56467   case ISD::SRA:
56468   case ISD::SRL: {
56469     SDValue N0 = Op.getOperand(0);
56470     // Look out for (store (shl (load), x)).
56471     if (X86::mayFoldLoad(N0, Subtarget) && IsFoldableRMW(N0, Op))
56472       return false;
56473     break;
56474   }
56475   case ISD::ADD:
56476   case ISD::MUL:
56477   case ISD::AND:
56478   case ISD::OR:
56479   case ISD::XOR:
56480     Commute = true;
56481     [[fallthrough]];
56482   case ISD::SUB: {
56483     SDValue N0 = Op.getOperand(0);
56484     SDValue N1 = Op.getOperand(1);
56485     // Avoid disabling potential load folding opportunities.
56486     if (X86::mayFoldLoad(N1, Subtarget) &&
56487         (!Commute || !isa<ConstantSDNode>(N0) ||
56488          (Op.getOpcode() != ISD::MUL && IsFoldableRMW(N1, Op))))
56489       return false;
56490     if (X86::mayFoldLoad(N0, Subtarget) &&
56491         ((Commute && !isa<ConstantSDNode>(N1)) ||
56492          (Op.getOpcode() != ISD::MUL && IsFoldableRMW(N0, Op))))
56493       return false;
56494     if (IsFoldableAtomicRMW(N0, Op) ||
56495         (Commute && IsFoldableAtomicRMW(N1, Op)))
56496       return false;
56497   }
56498   }
56499 
56500   PVT = MVT::i32;
56501   return true;
56502 }
56503 
56504 //===----------------------------------------------------------------------===//
56505 //                           X86 Inline Assembly Support
56506 //===----------------------------------------------------------------------===//
56507 
56508 // Helper to match a string separated by whitespace.
56509 static bool matchAsm(StringRef S, ArrayRef<const char *> Pieces) {
56510   S = S.substr(S.find_first_not_of(" \t")); // Skip leading whitespace.
56511 
56512   for (StringRef Piece : Pieces) {
56513     if (!S.starts_with(Piece)) // Check if the piece matches.
56514       return false;
56515 
56516     S = S.substr(Piece.size());
56517     StringRef::size_type Pos = S.find_first_not_of(" \t");
56518     if (Pos == 0) // We matched a prefix.
56519       return false;
56520 
56521     S = S.substr(Pos);
56522   }
56523 
56524   return S.empty();
56525 }
56526 
56527 static bool clobbersFlagRegisters(const SmallVector<StringRef, 4> &AsmPieces) {
56528 
56529   if (AsmPieces.size() == 3 || AsmPieces.size() == 4) {
56530     if (llvm::is_contained(AsmPieces, "~{cc}") &&
56531         llvm::is_contained(AsmPieces, "~{flags}") &&
56532         llvm::is_contained(AsmPieces, "~{fpsr}")) {
56533 
56534       if (AsmPieces.size() == 3)
56535         return true;
56536       else if (llvm::is_contained(AsmPieces, "~{dirflag}"))
56537         return true;
56538     }
56539   }
56540   return false;
56541 }
56542 
56543 bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
56544   InlineAsm *IA = cast<InlineAsm>(CI->getCalledOperand());
56545 
56546   const std::string &AsmStr = IA->getAsmString();
56547 
56548   IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
56549   if (!Ty || Ty->getBitWidth() % 16 != 0)
56550     return false;
56551 
56552   // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a"
56553   SmallVector<StringRef, 4> AsmPieces;
56554   SplitString(AsmStr, AsmPieces, ";\n");
56555 
56556   switch (AsmPieces.size()) {
56557   default: return false;
56558   case 1:
56559     // FIXME: this should verify that we are targeting a 486 or better.  If not,
56560     // we will turn this bswap into something that will be lowered to logical
56561     // ops instead of emitting the bswap asm.  For now, we don't support 486 or
56562     // lower so don't worry about this.
56563     // bswap $0
56564     if (matchAsm(AsmPieces[0], {"bswap", "$0"}) ||
56565         matchAsm(AsmPieces[0], {"bswapl", "$0"}) ||
56566         matchAsm(AsmPieces[0], {"bswapq", "$0"}) ||
56567         matchAsm(AsmPieces[0], {"bswap", "${0:q}"}) ||
56568         matchAsm(AsmPieces[0], {"bswapl", "${0:q}"}) ||
56569         matchAsm(AsmPieces[0], {"bswapq", "${0:q}"})) {
56570       // No need to check constraints, nothing other than the equivalent of
56571       // "=r,0" would be valid here.
56572       return IntrinsicLowering::LowerToByteSwap(CI);
56573     }
56574 
56575     // rorw $$8, ${0:w}  -->  llvm.bswap.i16
56576     if (CI->getType()->isIntegerTy(16) &&
56577         IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
56578         (matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) ||
56579          matchAsm(AsmPieces[0], {"rolw", "$$8,", "${0:w}"}))) {
56580       AsmPieces.clear();
56581       StringRef ConstraintsStr = IA->getConstraintString();
56582       SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
56583       array_pod_sort(AsmPieces.begin(), AsmPieces.end());
56584       if (clobbersFlagRegisters(AsmPieces))
56585         return IntrinsicLowering::LowerToByteSwap(CI);
56586     }
56587     break;
56588   case 3:
56589     if (CI->getType()->isIntegerTy(32) &&
56590         IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
56591         matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) &&
56592         matchAsm(AsmPieces[1], {"rorl", "$$16,", "$0"}) &&
56593         matchAsm(AsmPieces[2], {"rorw", "$$8,", "${0:w}"})) {
56594       AsmPieces.clear();
56595       StringRef ConstraintsStr = IA->getConstraintString();
56596       SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
56597       array_pod_sort(AsmPieces.begin(), AsmPieces.end());
56598       if (clobbersFlagRegisters(AsmPieces))
56599         return IntrinsicLowering::LowerToByteSwap(CI);
56600     }
56601 
56602     if (CI->getType()->isIntegerTy(64)) {
56603       InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
56604       if (Constraints.size() >= 2 &&
56605           Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" &&
56606           Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") {
56607         // bswap %eax / bswap %edx / xchgl %eax, %edx  -> llvm.bswap.i64
56608         if (matchAsm(AsmPieces[0], {"bswap", "%eax"}) &&
56609             matchAsm(AsmPieces[1], {"bswap", "%edx"}) &&
56610             matchAsm(AsmPieces[2], {"xchgl", "%eax,", "%edx"}))
56611           return IntrinsicLowering::LowerToByteSwap(CI);
56612       }
56613     }
56614     break;
56615   }
56616   return false;
56617 }
56618 
56619 static X86::CondCode parseConstraintCode(llvm::StringRef Constraint) {
56620   X86::CondCode Cond = StringSwitch<X86::CondCode>(Constraint)
56621                            .Case("{@cca}", X86::COND_A)
56622                            .Case("{@ccae}", X86::COND_AE)
56623                            .Case("{@ccb}", X86::COND_B)
56624                            .Case("{@ccbe}", X86::COND_BE)
56625                            .Case("{@ccc}", X86::COND_B)
56626                            .Case("{@cce}", X86::COND_E)
56627                            .Case("{@ccz}", X86::COND_E)
56628                            .Case("{@ccg}", X86::COND_G)
56629                            .Case("{@ccge}", X86::COND_GE)
56630                            .Case("{@ccl}", X86::COND_L)
56631                            .Case("{@ccle}", X86::COND_LE)
56632                            .Case("{@ccna}", X86::COND_BE)
56633                            .Case("{@ccnae}", X86::COND_B)
56634                            .Case("{@ccnb}", X86::COND_AE)
56635                            .Case("{@ccnbe}", X86::COND_A)
56636                            .Case("{@ccnc}", X86::COND_AE)
56637                            .Case("{@ccne}", X86::COND_NE)
56638                            .Case("{@ccnz}", X86::COND_NE)
56639                            .Case("{@ccng}", X86::COND_LE)
56640                            .Case("{@ccnge}", X86::COND_L)
56641                            .Case("{@ccnl}", X86::COND_GE)
56642                            .Case("{@ccnle}", X86::COND_G)
56643                            .Case("{@ccno}", X86::COND_NO)
56644                            .Case("{@ccnp}", X86::COND_NP)
56645                            .Case("{@ccns}", X86::COND_NS)
56646                            .Case("{@cco}", X86::COND_O)
56647                            .Case("{@ccp}", X86::COND_P)
56648                            .Case("{@ccs}", X86::COND_S)
56649                            .Default(X86::COND_INVALID);
56650   return Cond;
56651 }
56652 
56653 /// Given a constraint letter, return the type of constraint for this target.
56654 X86TargetLowering::ConstraintType
56655 X86TargetLowering::getConstraintType(StringRef Constraint) const {
56656   if (Constraint.size() == 1) {
56657     switch (Constraint[0]) {
56658     case 'R':
56659     case 'q':
56660     case 'Q':
56661     case 'f':
56662     case 't':
56663     case 'u':
56664     case 'y':
56665     case 'x':
56666     case 'v':
56667     case 'l':
56668     case 'k': // AVX512 masking registers.
56669       return C_RegisterClass;
56670     case 'a':
56671     case 'b':
56672     case 'c':
56673     case 'd':
56674     case 'S':
56675     case 'D':
56676     case 'A':
56677       return C_Register;
56678     case 'I':
56679     case 'J':
56680     case 'K':
56681     case 'N':
56682     case 'G':
56683     case 'L':
56684     case 'M':
56685       return C_Immediate;
56686     case 'C':
56687     case 'e':
56688     case 'Z':
56689       return C_Other;
56690     default:
56691       break;
56692     }
56693   }
56694   else if (Constraint.size() == 2) {
56695     switch (Constraint[0]) {
56696     default:
56697       break;
56698     case 'W':
56699       if (Constraint[1] != 's')
56700         break;
56701       return C_Other;
56702     case 'Y':
56703       switch (Constraint[1]) {
56704       default:
56705         break;
56706       case 'z':
56707         return C_Register;
56708       case 'i':
56709       case 'm':
56710       case 'k':
56711       case 't':
56712       case '2':
56713         return C_RegisterClass;
56714       }
56715     }
56716   } else if (parseConstraintCode(Constraint) != X86::COND_INVALID)
56717     return C_Other;
56718   return TargetLowering::getConstraintType(Constraint);
56719 }
56720 
56721 /// Examine constraint type and operand type and determine a weight value.
56722 /// This object must already have been set up with the operand type
56723 /// and the current alternative constraint selected.
56724 TargetLowering::ConstraintWeight
56725 X86TargetLowering::getSingleConstraintMatchWeight(
56726     AsmOperandInfo &Info, const char *Constraint) const {
56727   ConstraintWeight Wt = CW_Invalid;
56728   Value *CallOperandVal = Info.CallOperandVal;
56729   // If we don't have a value, we can't do a match,
56730   // but allow it at the lowest weight.
56731   if (!CallOperandVal)
56732     return CW_Default;
56733   Type *Ty = CallOperandVal->getType();
56734   // Look at the constraint type.
56735   switch (*Constraint) {
56736   default:
56737     Wt = TargetLowering::getSingleConstraintMatchWeight(Info, Constraint);
56738     [[fallthrough]];
56739   case 'R':
56740   case 'q':
56741   case 'Q':
56742   case 'a':
56743   case 'b':
56744   case 'c':
56745   case 'd':
56746   case 'S':
56747   case 'D':
56748   case 'A':
56749     if (CallOperandVal->getType()->isIntegerTy())
56750       Wt = CW_SpecificReg;
56751     break;
56752   case 'f':
56753   case 't':
56754   case 'u':
56755     if (Ty->isFloatingPointTy())
56756       Wt = CW_SpecificReg;
56757     break;
56758   case 'y':
56759     if (Ty->isX86_MMXTy() && Subtarget.hasMMX())
56760       Wt = CW_SpecificReg;
56761     break;
56762   case 'Y':
56763     if (StringRef(Constraint).size() != 2)
56764       break;
56765     switch (Constraint[1]) {
56766     default:
56767       return CW_Invalid;
56768     // XMM0
56769     case 'z':
56770       if (((Ty->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) ||
56771           ((Ty->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()) ||
56772           ((Ty->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512()))
56773         return CW_SpecificReg;
56774       return CW_Invalid;
56775     // Conditional OpMask regs (AVX512)
56776     case 'k':
56777       if ((Ty->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512())
56778         return CW_Register;
56779       return CW_Invalid;
56780     // Any MMX reg
56781     case 'm':
56782       if (Ty->isX86_MMXTy() && Subtarget.hasMMX())
56783         return Wt;
56784       return CW_Invalid;
56785     // Any SSE reg when ISA >= SSE2, same as 'x'
56786     case 'i':
56787     case 't':
56788     case '2':
56789       if (!Subtarget.hasSSE2())
56790         return CW_Invalid;
56791       break;
56792     }
56793     break;
56794   case 'v':
56795     if ((Ty->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512())
56796       Wt = CW_Register;
56797     [[fallthrough]];
56798   case 'x':
56799     if (((Ty->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) ||
56800         ((Ty->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()))
56801       Wt = CW_Register;
56802     break;
56803   case 'k':
56804     // Enable conditional vector operations using %k<#> registers.
56805     if ((Ty->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512())
56806       Wt = CW_Register;
56807     break;
56808   case 'I':
56809     if (auto *C = dyn_cast<ConstantInt>(Info.CallOperandVal))
56810       if (C->getZExtValue() <= 31)
56811         Wt = CW_Constant;
56812     break;
56813   case 'J':
56814     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56815       if (C->getZExtValue() <= 63)
56816         Wt = CW_Constant;
56817     break;
56818   case 'K':
56819     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56820       if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f))
56821         Wt = CW_Constant;
56822     break;
56823   case 'L':
56824     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56825       if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff))
56826         Wt = CW_Constant;
56827     break;
56828   case 'M':
56829     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56830       if (C->getZExtValue() <= 3)
56831         Wt = CW_Constant;
56832     break;
56833   case 'N':
56834     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56835       if (C->getZExtValue() <= 0xff)
56836         Wt = CW_Constant;
56837     break;
56838   case 'G':
56839   case 'C':
56840     if (isa<ConstantFP>(CallOperandVal))
56841       Wt = CW_Constant;
56842     break;
56843   case 'e':
56844     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56845       if ((C->getSExtValue() >= -0x80000000LL) &&
56846           (C->getSExtValue() <= 0x7fffffffLL))
56847         Wt = CW_Constant;
56848     break;
56849   case 'Z':
56850     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
56851       if (C->getZExtValue() <= 0xffffffff)
56852         Wt = CW_Constant;
56853     break;
56854   }
56855   return Wt;
56856 }
56857 
56858 /// Try to replace an X constraint, which matches anything, with another that
56859 /// has more specific requirements based on the type of the corresponding
56860 /// operand.
56861 const char *X86TargetLowering::
56862 LowerXConstraint(EVT ConstraintVT) const {
56863   // FP X constraints get lowered to SSE1/2 registers if available, otherwise
56864   // 'f' like normal targets.
56865   if (ConstraintVT.isFloatingPoint()) {
56866     if (Subtarget.hasSSE1())
56867       return "x";
56868   }
56869 
56870   return TargetLowering::LowerXConstraint(ConstraintVT);
56871 }
56872 
56873 // Lower @cc targets via setcc.
56874 SDValue X86TargetLowering::LowerAsmOutputForConstraint(
56875     SDValue &Chain, SDValue &Glue, const SDLoc &DL,
56876     const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
56877   X86::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode);
56878   if (Cond == X86::COND_INVALID)
56879     return SDValue();
56880   // Check that return type is valid.
56881   if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
56882       OpInfo.ConstraintVT.getSizeInBits() < 8)
56883     report_fatal_error("Glue output operand is of invalid type");
56884 
56885   // Get EFLAGS register. Only update chain when copyfrom is glued.
56886   if (Glue.getNode()) {
56887     Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Glue);
56888     Chain = Glue.getValue(1);
56889   } else
56890     Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);
56891   // Extract CC code.
56892   SDValue CC = getSETCC(Cond, Glue, DL, DAG);
56893   // Extend to 32-bits
56894   SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
56895 
56896   return Result;
56897 }
56898 
56899 /// Lower the specified operand into the Ops vector.
56900 /// If it is invalid, don't add anything to Ops.
56901 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
56902                                                      StringRef Constraint,
56903                                                      std::vector<SDValue> &Ops,
56904                                                      SelectionDAG &DAG) const {
56905   SDValue Result;
56906   char ConstraintLetter = Constraint[0];
56907   switch (ConstraintLetter) {
56908   default: break;
56909   case 'I':
56910     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56911       if (C->getZExtValue() <= 31) {
56912         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56913                                        Op.getValueType());
56914         break;
56915       }
56916     }
56917     return;
56918   case 'J':
56919     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56920       if (C->getZExtValue() <= 63) {
56921         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56922                                        Op.getValueType());
56923         break;
56924       }
56925     }
56926     return;
56927   case 'K':
56928     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56929       if (isInt<8>(C->getSExtValue())) {
56930         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56931                                        Op.getValueType());
56932         break;
56933       }
56934     }
56935     return;
56936   case 'L':
56937     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56938       if (C->getZExtValue() == 0xff || C->getZExtValue() == 0xffff ||
56939           (Subtarget.is64Bit() && C->getZExtValue() == 0xffffffff)) {
56940         Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
56941                                        Op.getValueType());
56942         break;
56943       }
56944     }
56945     return;
56946   case 'M':
56947     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56948       if (C->getZExtValue() <= 3) {
56949         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56950                                        Op.getValueType());
56951         break;
56952       }
56953     }
56954     return;
56955   case 'N':
56956     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56957       if (C->getZExtValue() <= 255) {
56958         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56959                                        Op.getValueType());
56960         break;
56961       }
56962     }
56963     return;
56964   case 'O':
56965     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56966       if (C->getZExtValue() <= 127) {
56967         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
56968                                        Op.getValueType());
56969         break;
56970       }
56971     }
56972     return;
56973   case 'e': {
56974     // 32-bit signed value
56975     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
56976       if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
56977                                            C->getSExtValue())) {
56978         // Widen to 64 bits here to get it sign extended.
56979         Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
56980         break;
56981       }
56982     // FIXME gcc accepts some relocatable values here too, but only in certain
56983     // memory models; it's complicated.
56984     }
56985     return;
56986   }
56987   case 'W': {
56988     assert(Constraint[1] == 's');
56989     // Op is a BlockAddressSDNode or a GlobalAddressSDNode with an optional
56990     // offset.
56991     if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
56992       Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
56993                                               BA->getValueType(0)));
56994     } else {
56995       int64_t Offset = 0;
56996       if (Op->getOpcode() == ISD::ADD &&
56997           isa<ConstantSDNode>(Op->getOperand(1))) {
56998         Offset = cast<ConstantSDNode>(Op->getOperand(1))->getSExtValue();
56999         Op = Op->getOperand(0);
57000       }
57001       if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op))
57002         Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
57003                                                  GA->getValueType(0), Offset));
57004     }
57005     return;
57006   }
57007   case 'Z': {
57008     // 32-bit unsigned value
57009     if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
57010       if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
57011                                            C->getZExtValue())) {
57012         Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
57013                                        Op.getValueType());
57014         break;
57015       }
57016     }
57017     // FIXME gcc accepts some relocatable values here too, but only in certain
57018     // memory models; it's complicated.
57019     return;
57020   }
57021   case 'i': {
57022     // Literal immediates are always ok.
57023     if (auto *CST = dyn_cast<ConstantSDNode>(Op)) {
57024       bool IsBool = CST->getConstantIntValue()->getBitWidth() == 1;
57025       BooleanContent BCont = getBooleanContents(MVT::i64);
57026       ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
57027                                     : ISD::SIGN_EXTEND;
57028       int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? CST->getZExtValue()
57029                                                   : CST->getSExtValue();
57030       Result = DAG.getTargetConstant(ExtVal, SDLoc(Op), MVT::i64);
57031       break;
57032     }
57033 
57034     // In any sort of PIC mode addresses need to be computed at runtime by
57035     // adding in a register or some sort of table lookup.  These can't
57036     // be used as immediates. BlockAddresses and BasicBlocks are fine though.
57037     if ((Subtarget.isPICStyleGOT() || Subtarget.isPICStyleStubPIC()) &&
57038         !(isa<BlockAddressSDNode>(Op) || isa<BasicBlockSDNode>(Op)))
57039       return;
57040 
57041     // If we are in non-pic codegen mode, we allow the address of a global (with
57042     // an optional displacement) to be used with 'i'.
57043     if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op))
57044       // If we require an extra load to get this address, as in PIC mode, we
57045       // can't accept it.
57046       if (isGlobalStubReference(
57047               Subtarget.classifyGlobalReference(GA->getGlobal())))
57048         return;
57049     break;
57050   }
57051   }
57052 
57053   if (Result.getNode()) {
57054     Ops.push_back(Result);
57055     return;
57056   }
57057   return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
57058 }
57059 
57060 /// Check if \p RC is a general purpose register class.
57061 /// I.e., GR* or one of their variant.
57062 static bool isGRClass(const TargetRegisterClass &RC) {
57063   return RC.hasSuperClassEq(&X86::GR8RegClass) ||
57064          RC.hasSuperClassEq(&X86::GR16RegClass) ||
57065          RC.hasSuperClassEq(&X86::GR32RegClass) ||
57066          RC.hasSuperClassEq(&X86::GR64RegClass) ||
57067          RC.hasSuperClassEq(&X86::LOW32_ADDR_ACCESS_RBPRegClass);
57068 }
57069 
57070 /// Check if \p RC is a vector register class.
57071 /// I.e., FR* / VR* or one of their variant.
57072 static bool isFRClass(const TargetRegisterClass &RC) {
57073   return RC.hasSuperClassEq(&X86::FR16XRegClass) ||
57074          RC.hasSuperClassEq(&X86::FR32XRegClass) ||
57075          RC.hasSuperClassEq(&X86::FR64XRegClass) ||
57076          RC.hasSuperClassEq(&X86::VR128XRegClass) ||
57077          RC.hasSuperClassEq(&X86::VR256XRegClass) ||
57078          RC.hasSuperClassEq(&X86::VR512RegClass);
57079 }
57080 
57081 /// Check if \p RC is a mask register class.
57082 /// I.e., VK* or one of their variant.
57083 static bool isVKClass(const TargetRegisterClass &RC) {
57084   return RC.hasSuperClassEq(&X86::VK1RegClass) ||
57085          RC.hasSuperClassEq(&X86::VK2RegClass) ||
57086          RC.hasSuperClassEq(&X86::VK4RegClass) ||
57087          RC.hasSuperClassEq(&X86::VK8RegClass) ||
57088          RC.hasSuperClassEq(&X86::VK16RegClass) ||
57089          RC.hasSuperClassEq(&X86::VK32RegClass) ||
57090          RC.hasSuperClassEq(&X86::VK64RegClass);
57091 }
57092 
57093 std::pair<unsigned, const TargetRegisterClass *>
57094 X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
57095                                                 StringRef Constraint,
57096                                                 MVT VT) const {
57097   // First, see if this is a constraint that directly corresponds to an LLVM
57098   // register class.
57099   if (Constraint.size() == 1) {
57100     // GCC Constraint Letters
57101     switch (Constraint[0]) {
57102     default: break;
57103     // 'A' means [ER]AX + [ER]DX.
57104     case 'A':
57105       if (Subtarget.is64Bit())
57106         return std::make_pair(X86::RAX, &X86::GR64_ADRegClass);
57107       assert((Subtarget.is32Bit() || Subtarget.is16Bit()) &&
57108              "Expecting 64, 32 or 16 bit subtarget");
57109       return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
57110 
57111       // TODO: Slight differences here in allocation order and leaving
57112       // RIP in the class. Do they matter any more here than they do
57113       // in the normal allocation?
57114     case 'k':
57115       if (Subtarget.hasAVX512()) {
57116         if (VT == MVT::v1i1 || VT == MVT::i1)
57117           return std::make_pair(0U, &X86::VK1RegClass);
57118         if (VT == MVT::v8i1 || VT == MVT::i8)
57119           return std::make_pair(0U, &X86::VK8RegClass);
57120         if (VT == MVT::v16i1 || VT == MVT::i16)
57121           return std::make_pair(0U, &X86::VK16RegClass);
57122       }
57123       if (Subtarget.hasBWI()) {
57124         if (VT == MVT::v32i1 || VT == MVT::i32)
57125           return std::make_pair(0U, &X86::VK32RegClass);
57126         if (VT == MVT::v64i1 || VT == MVT::i64)
57127           return std::make_pair(0U, &X86::VK64RegClass);
57128       }
57129       break;
57130     case 'q':   // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
57131       if (Subtarget.is64Bit()) {
57132         if (VT == MVT::i8 || VT == MVT::i1)
57133           return std::make_pair(0U, &X86::GR8_NOREX2RegClass);
57134         if (VT == MVT::i16)
57135           return std::make_pair(0U, &X86::GR16_NOREX2RegClass);
57136         if (VT == MVT::i32 || VT == MVT::f32)
57137           return std::make_pair(0U, &X86::GR32_NOREX2RegClass);
57138         if (VT != MVT::f80 && !VT.isVector())
57139           return std::make_pair(0U, &X86::GR64_NOREX2RegClass);
57140         break;
57141       }
57142       [[fallthrough]];
57143       // 32-bit fallthrough
57144     case 'Q':   // Q_REGS
57145       if (VT == MVT::i8 || VT == MVT::i1)
57146         return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
57147       if (VT == MVT::i16)
57148         return std::make_pair(0U, &X86::GR16_ABCDRegClass);
57149       if (VT == MVT::i32 || VT == MVT::f32 ||
57150           (!VT.isVector() && !Subtarget.is64Bit()))
57151         return std::make_pair(0U, &X86::GR32_ABCDRegClass);
57152       if (VT != MVT::f80 && !VT.isVector())
57153         return std::make_pair(0U, &X86::GR64_ABCDRegClass);
57154       break;
57155     case 'r':   // GENERAL_REGS
57156     case 'l':   // INDEX_REGS
57157       if (VT == MVT::i8 || VT == MVT::i1)
57158         return std::make_pair(0U, &X86::GR8_NOREX2RegClass);
57159       if (VT == MVT::i16)
57160         return std::make_pair(0U, &X86::GR16_NOREX2RegClass);
57161       if (VT == MVT::i32 || VT == MVT::f32 ||
57162           (!VT.isVector() && !Subtarget.is64Bit()))
57163         return std::make_pair(0U, &X86::GR32_NOREX2RegClass);
57164       if (VT != MVT::f80 && !VT.isVector())
57165         return std::make_pair(0U, &X86::GR64_NOREX2RegClass);
57166       break;
57167     case 'R':   // LEGACY_REGS
57168       if (VT == MVT::i8 || VT == MVT::i1)
57169         return std::make_pair(0U, &X86::GR8_NOREXRegClass);
57170       if (VT == MVT::i16)
57171         return std::make_pair(0U, &X86::GR16_NOREXRegClass);
57172       if (VT == MVT::i32 || VT == MVT::f32 ||
57173           (!VT.isVector() && !Subtarget.is64Bit()))
57174         return std::make_pair(0U, &X86::GR32_NOREXRegClass);
57175       if (VT != MVT::f80 && !VT.isVector())
57176         return std::make_pair(0U, &X86::GR64_NOREXRegClass);
57177       break;
57178     case 'f':  // FP Stack registers.
57179       // If SSE is enabled for this VT, use f80 to ensure the isel moves the
57180       // value to the correct fpstack register class.
57181       if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
57182         return std::make_pair(0U, &X86::RFP32RegClass);
57183       if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
57184         return std::make_pair(0U, &X86::RFP64RegClass);
57185       if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80)
57186         return std::make_pair(0U, &X86::RFP80RegClass);
57187       break;
57188     case 'y':   // MMX_REGS if MMX allowed.
57189       if (!Subtarget.hasMMX()) break;
57190       return std::make_pair(0U, &X86::VR64RegClass);
57191     case 'v':
57192     case 'x':   // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
57193       if (!Subtarget.hasSSE1()) break;
57194       bool VConstraint = (Constraint[0] == 'v');
57195 
57196       switch (VT.SimpleTy) {
57197       default: break;
57198       // Scalar SSE types.
57199       case MVT::f16:
57200         if (VConstraint && Subtarget.hasFP16())
57201           return std::make_pair(0U, &X86::FR16XRegClass);
57202         break;
57203       case MVT::f32:
57204       case MVT::i32:
57205         if (VConstraint && Subtarget.hasVLX())
57206           return std::make_pair(0U, &X86::FR32XRegClass);
57207         return std::make_pair(0U, &X86::FR32RegClass);
57208       case MVT::f64:
57209       case MVT::i64:
57210         if (VConstraint && Subtarget.hasVLX())
57211           return std::make_pair(0U, &X86::FR64XRegClass);
57212         return std::make_pair(0U, &X86::FR64RegClass);
57213       case MVT::i128:
57214         if (Subtarget.is64Bit()) {
57215           if (VConstraint && Subtarget.hasVLX())
57216             return std::make_pair(0U, &X86::VR128XRegClass);
57217           return std::make_pair(0U, &X86::VR128RegClass);
57218         }
57219         break;
57220       // Vector types and fp128.
57221       case MVT::v8f16:
57222         if (!Subtarget.hasFP16())
57223           break;
57224         if (VConstraint)
57225           return std::make_pair(0U, &X86::VR128XRegClass);
57226         return std::make_pair(0U, &X86::VR128RegClass);
57227       case MVT::v8bf16:
57228         if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
57229           break;
57230         if (VConstraint)
57231           return std::make_pair(0U, &X86::VR128XRegClass);
57232         return std::make_pair(0U, &X86::VR128RegClass);
57233       case MVT::f128:
57234       case MVT::v16i8:
57235       case MVT::v8i16:
57236       case MVT::v4i32:
57237       case MVT::v2i64:
57238       case MVT::v4f32:
57239       case MVT::v2f64:
57240         if (VConstraint && Subtarget.hasVLX())
57241           return std::make_pair(0U, &X86::VR128XRegClass);
57242         return std::make_pair(0U, &X86::VR128RegClass);
57243       // AVX types.
57244       case MVT::v16f16:
57245         if (!Subtarget.hasFP16())
57246           break;
57247         if (VConstraint)
57248           return std::make_pair(0U, &X86::VR256XRegClass);
57249         return std::make_pair(0U, &X86::VR256RegClass);
57250       case MVT::v16bf16:
57251         if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
57252           break;
57253         if (VConstraint)
57254           return std::make_pair(0U, &X86::VR256XRegClass);
57255         return std::make_pair(0U, &X86::VR256RegClass);
57256       case MVT::v32i8:
57257       case MVT::v16i16:
57258       case MVT::v8i32:
57259       case MVT::v4i64:
57260       case MVT::v8f32:
57261       case MVT::v4f64:
57262         if (VConstraint && Subtarget.hasVLX())
57263           return std::make_pair(0U, &X86::VR256XRegClass);
57264         if (Subtarget.hasAVX())
57265           return std::make_pair(0U, &X86::VR256RegClass);
57266         break;
57267       case MVT::v32f16:
57268         if (!Subtarget.hasFP16())
57269           break;
57270         if (VConstraint)
57271           return std::make_pair(0U, &X86::VR512RegClass);
57272         return std::make_pair(0U, &X86::VR512_0_15RegClass);
57273       case MVT::v32bf16:
57274         if (!Subtarget.hasBF16())
57275           break;
57276         if (VConstraint)
57277           return std::make_pair(0U, &X86::VR512RegClass);
57278         return std::make_pair(0U, &X86::VR512_0_15RegClass);
57279       case MVT::v64i8:
57280       case MVT::v32i16:
57281       case MVT::v8f64:
57282       case MVT::v16f32:
57283       case MVT::v16i32:
57284       case MVT::v8i64:
57285         if (!Subtarget.hasAVX512()) break;
57286         if (VConstraint)
57287           return std::make_pair(0U, &X86::VR512RegClass);
57288         return std::make_pair(0U, &X86::VR512_0_15RegClass);
57289       }
57290       break;
57291     }
57292   } else if (Constraint.size() == 2 && Constraint[0] == 'Y') {
57293     switch (Constraint[1]) {
57294     default:
57295       break;
57296     case 'i':
57297     case 't':
57298     case '2':
57299       return getRegForInlineAsmConstraint(TRI, "x", VT);
57300     case 'm':
57301       if (!Subtarget.hasMMX()) break;
57302       return std::make_pair(0U, &X86::VR64RegClass);
57303     case 'z':
57304       if (!Subtarget.hasSSE1()) break;
57305       switch (VT.SimpleTy) {
57306       default: break;
57307       // Scalar SSE types.
57308       case MVT::f16:
57309         if (!Subtarget.hasFP16())
57310           break;
57311         return std::make_pair(X86::XMM0, &X86::FR16XRegClass);
57312       case MVT::f32:
57313       case MVT::i32:
57314         return std::make_pair(X86::XMM0, &X86::FR32RegClass);
57315       case MVT::f64:
57316       case MVT::i64:
57317         return std::make_pair(X86::XMM0, &X86::FR64RegClass);
57318       case MVT::v8f16:
57319         if (!Subtarget.hasFP16())
57320           break;
57321         return std::make_pair(X86::XMM0, &X86::VR128RegClass);
57322       case MVT::v8bf16:
57323         if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
57324           break;
57325         return std::make_pair(X86::XMM0, &X86::VR128RegClass);
57326       case MVT::f128:
57327       case MVT::v16i8:
57328       case MVT::v8i16:
57329       case MVT::v4i32:
57330       case MVT::v2i64:
57331       case MVT::v4f32:
57332       case MVT::v2f64:
57333         return std::make_pair(X86::XMM0, &X86::VR128RegClass);
57334       // AVX types.
57335       case MVT::v16f16:
57336         if (!Subtarget.hasFP16())
57337           break;
57338         return std::make_pair(X86::YMM0, &X86::VR256RegClass);
57339       case MVT::v16bf16:
57340         if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
57341           break;
57342         return std::make_pair(X86::YMM0, &X86::VR256RegClass);
57343       case MVT::v32i8:
57344       case MVT::v16i16:
57345       case MVT::v8i32:
57346       case MVT::v4i64:
57347       case MVT::v8f32:
57348       case MVT::v4f64:
57349         if (Subtarget.hasAVX())
57350           return std::make_pair(X86::YMM0, &X86::VR256RegClass);
57351         break;
57352       case MVT::v32f16:
57353         if (!Subtarget.hasFP16())
57354           break;
57355         return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
57356       case MVT::v32bf16:
57357         if (!Subtarget.hasBF16())
57358           break;
57359         return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
57360       case MVT::v64i8:
57361       case MVT::v32i16:
57362       case MVT::v8f64:
57363       case MVT::v16f32:
57364       case MVT::v16i32:
57365       case MVT::v8i64:
57366         if (Subtarget.hasAVX512())
57367           return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
57368         break;
57369       }
57370       break;
57371     case 'k':
57372       // This register class doesn't allocate k0 for masked vector operation.
57373       if (Subtarget.hasAVX512()) {
57374         if (VT == MVT::v1i1 || VT == MVT::i1)
57375           return std::make_pair(0U, &X86::VK1WMRegClass);
57376         if (VT == MVT::v8i1 || VT == MVT::i8)
57377           return std::make_pair(0U, &X86::VK8WMRegClass);
57378         if (VT == MVT::v16i1 || VT == MVT::i16)
57379           return std::make_pair(0U, &X86::VK16WMRegClass);
57380       }
57381       if (Subtarget.hasBWI()) {
57382         if (VT == MVT::v32i1 || VT == MVT::i32)
57383           return std::make_pair(0U, &X86::VK32WMRegClass);
57384         if (VT == MVT::v64i1 || VT == MVT::i64)
57385           return std::make_pair(0U, &X86::VK64WMRegClass);
57386       }
57387       break;
57388     }
57389   }
57390 
57391   if (parseConstraintCode(Constraint) != X86::COND_INVALID)
57392     return std::make_pair(0U, &X86::GR32RegClass);
57393 
57394   // Use the default implementation in TargetLowering to convert the register
57395   // constraint into a member of a register class.
57396   std::pair<Register, const TargetRegisterClass*> Res;
57397   Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
57398 
57399   // Not found as a standard register?
57400   if (!Res.second) {
57401     // Only match x87 registers if the VT is one SelectionDAGBuilder can convert
57402     // to/from f80.
57403     if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) {
57404       // Map st(0) -> st(7) -> ST0
57405       if (Constraint.size() == 7 && Constraint[0] == '{' &&
57406           tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' &&
57407           Constraint[3] == '(' &&
57408           (Constraint[4] >= '0' && Constraint[4] <= '7') &&
57409           Constraint[5] == ')' && Constraint[6] == '}') {
57410         // st(7) is not allocatable and thus not a member of RFP80. Return
57411         // singleton class in cases where we have a reference to it.
57412         if (Constraint[4] == '7')
57413           return std::make_pair(X86::FP7, &X86::RFP80_7RegClass);
57414         return std::make_pair(X86::FP0 + Constraint[4] - '0',
57415                               &X86::RFP80RegClass);
57416       }
57417 
57418       // GCC allows "st(0)" to be called just plain "st".
57419       if (StringRef("{st}").equals_insensitive(Constraint))
57420         return std::make_pair(X86::FP0, &X86::RFP80RegClass);
57421     }
57422 
57423     // flags -> EFLAGS
57424     if (StringRef("{flags}").equals_insensitive(Constraint))
57425       return std::make_pair(X86::EFLAGS, &X86::CCRRegClass);
57426 
57427     // dirflag -> DF
57428     // Only allow for clobber.
57429     if (StringRef("{dirflag}").equals_insensitive(Constraint) &&
57430         VT == MVT::Other)
57431       return std::make_pair(X86::DF, &X86::DFCCRRegClass);
57432 
57433     // fpsr -> FPSW
57434     // Only allow for clobber.
57435     if (StringRef("{fpsr}").equals_insensitive(Constraint) && VT == MVT::Other)
57436       return std::make_pair(X86::FPSW, &X86::FPCCRRegClass);
57437 
57438     return Res;
57439   }
57440 
57441   // Make sure it isn't a register that requires 64-bit mode.
57442   if (!Subtarget.is64Bit() &&
57443       (isFRClass(*Res.second) || isGRClass(*Res.second)) &&
57444       TRI->getEncodingValue(Res.first) >= 8) {
57445     // Register requires REX prefix, but we're in 32-bit mode.
57446     return std::make_pair(0, nullptr);
57447   }
57448 
57449   // Make sure it isn't a register that requires AVX512.
57450   if (!Subtarget.hasAVX512() && isFRClass(*Res.second) &&
57451       TRI->getEncodingValue(Res.first) & 0x10) {
57452     // Register requires EVEX prefix.
57453     return std::make_pair(0, nullptr);
57454   }
57455 
57456   // Otherwise, check to see if this is a register class of the wrong value
57457   // type.  For example, we want to map "{ax},i32" -> {eax}, we don't want it to
57458   // turn into {ax},{dx}.
57459   // MVT::Other is used to specify clobber names.
57460   if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other)
57461     return Res;   // Correct type already, nothing to do.
57462 
57463   // Get a matching integer of the correct size. i.e. "ax" with MVT::32 should
57464   // return "eax". This should even work for things like getting 64bit integer
57465   // registers when given an f64 type.
57466   const TargetRegisterClass *Class = Res.second;
57467   // The generic code will match the first register class that contains the
57468   // given register. Thus, based on the ordering of the tablegened file,
57469   // the "plain" GR classes might not come first.
57470   // Therefore, use a helper method.
57471   if (isGRClass(*Class)) {
57472     unsigned Size = VT.getSizeInBits();
57473     if (Size == 1) Size = 8;
57474     if (Size != 8 && Size != 16 && Size != 32 && Size != 64)
57475       return std::make_pair(0, nullptr);
57476     Register DestReg = getX86SubSuperRegister(Res.first, Size);
57477     if (DestReg.isValid()) {
57478       bool is64Bit = Subtarget.is64Bit();
57479       const TargetRegisterClass *RC =
57480           Size == 8 ? (is64Bit ? &X86::GR8RegClass : &X86::GR8_NOREXRegClass)
57481         : Size == 16 ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
57482         : Size == 32 ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
57483         : /*Size == 64*/ (is64Bit ? &X86::GR64RegClass : nullptr);
57484       if (Size == 64 && !is64Bit) {
57485         // Model GCC's behavior here and select a fixed pair of 32-bit
57486         // registers.
57487         switch (DestReg) {
57488         case X86::RAX:
57489           return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
57490         case X86::RDX:
57491           return std::make_pair(X86::EDX, &X86::GR32_DCRegClass);
57492         case X86::RCX:
57493           return std::make_pair(X86::ECX, &X86::GR32_CBRegClass);
57494         case X86::RBX:
57495           return std::make_pair(X86::EBX, &X86::GR32_BSIRegClass);
57496         case X86::RSI:
57497           return std::make_pair(X86::ESI, &X86::GR32_SIDIRegClass);
57498         case X86::RDI:
57499           return std::make_pair(X86::EDI, &X86::GR32_DIBPRegClass);
57500         case X86::RBP:
57501           return std::make_pair(X86::EBP, &X86::GR32_BPSPRegClass);
57502         default:
57503           return std::make_pair(0, nullptr);
57504         }
57505       }
57506       if (RC && RC->contains(DestReg))
57507         return std::make_pair(DestReg, RC);
57508       return Res;
57509     }
57510     // No register found/type mismatch.
57511     return std::make_pair(0, nullptr);
57512   } else if (isFRClass(*Class)) {
57513     // Handle references to XMM physical registers that got mapped into the
57514     // wrong class.  This can happen with constraints like {xmm0} where the
57515     // target independent register mapper will just pick the first match it can
57516     // find, ignoring the required type.
57517 
57518     // TODO: Handle f128 and i128 in FR128RegClass after it is tested well.
57519     if (VT == MVT::f16)
57520       Res.second = &X86::FR16XRegClass;
57521     else if (VT == MVT::f32 || VT == MVT::i32)
57522       Res.second = &X86::FR32XRegClass;
57523     else if (VT == MVT::f64 || VT == MVT::i64)
57524       Res.second = &X86::FR64XRegClass;
57525     else if (TRI->isTypeLegalForClass(X86::VR128XRegClass, VT))
57526       Res.second = &X86::VR128XRegClass;
57527     else if (TRI->isTypeLegalForClass(X86::VR256XRegClass, VT))
57528       Res.second = &X86::VR256XRegClass;
57529     else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT))
57530       Res.second = &X86::VR512RegClass;
57531     else {
57532       // Type mismatch and not a clobber: Return an error;
57533       Res.first = 0;
57534       Res.second = nullptr;
57535     }
57536   } else if (isVKClass(*Class)) {
57537     if (VT == MVT::v1i1 || VT == MVT::i1)
57538       Res.second = &X86::VK1RegClass;
57539     else if (VT == MVT::v8i1 || VT == MVT::i8)
57540       Res.second = &X86::VK8RegClass;
57541     else if (VT == MVT::v16i1 || VT == MVT::i16)
57542       Res.second = &X86::VK16RegClass;
57543     else if (VT == MVT::v32i1 || VT == MVT::i32)
57544       Res.second = &X86::VK32RegClass;
57545     else if (VT == MVT::v64i1 || VT == MVT::i64)
57546       Res.second = &X86::VK64RegClass;
57547     else {
57548       // Type mismatch and not a clobber: Return an error;
57549       Res.first = 0;
57550       Res.second = nullptr;
57551     }
57552   }
57553 
57554   return Res;
57555 }
57556 
57557 bool X86TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
57558   // Integer division on x86 is expensive. However, when aggressively optimizing
57559   // for code size, we prefer to use a div instruction, as it is usually smaller
57560   // than the alternative sequence.
57561   // The exception to this is vector division. Since x86 doesn't have vector
57562   // integer division, leaving the division as-is is a loss even in terms of
57563   // size, because it will have to be scalarized, while the alternative code
57564   // sequence can be performed in vector form.
57565   bool OptSize = Attr.hasFnAttr(Attribute::MinSize);
57566   return OptSize && !VT.isVector();
57567 }
57568 
57569 void X86TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
57570   if (!Subtarget.is64Bit())
57571     return;
57572 
57573   // Update IsSplitCSR in X86MachineFunctionInfo.
57574   X86MachineFunctionInfo *AFI =
57575       Entry->getParent()->getInfo<X86MachineFunctionInfo>();
57576   AFI->setIsSplitCSR(true);
57577 }
57578 
57579 void X86TargetLowering::insertCopiesSplitCSR(
57580     MachineBasicBlock *Entry,
57581     const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
57582   const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
57583   const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
57584   if (!IStart)
57585     return;
57586 
57587   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
57588   MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
57589   MachineBasicBlock::iterator MBBI = Entry->begin();
57590   for (const MCPhysReg *I = IStart; *I; ++I) {
57591     const TargetRegisterClass *RC = nullptr;
57592     if (X86::GR64RegClass.contains(*I))
57593       RC = &X86::GR64RegClass;
57594     else
57595       llvm_unreachable("Unexpected register class in CSRsViaCopy!");
57596 
57597     Register NewVR = MRI->createVirtualRegister(RC);
57598     // Create copy from CSR to a virtual register.
57599     // FIXME: this currently does not emit CFI pseudo-instructions, it works
57600     // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
57601     // nounwind. If we want to generalize this later, we may need to emit
57602     // CFI pseudo-instructions.
57603     assert(
57604         Entry->getParent()->getFunction().hasFnAttribute(Attribute::NoUnwind) &&
57605         "Function should be nounwind in insertCopiesSplitCSR!");
57606     Entry->addLiveIn(*I);
57607     BuildMI(*Entry, MBBI, MIMetadata(), TII->get(TargetOpcode::COPY), NewVR)
57608         .addReg(*I);
57609 
57610     // Insert the copy-back instructions right before the terminator.
57611     for (auto *Exit : Exits)
57612       BuildMI(*Exit, Exit->getFirstTerminator(), MIMetadata(),
57613               TII->get(TargetOpcode::COPY), *I)
57614           .addReg(NewVR);
57615   }
57616 }
57617 
57618 bool X86TargetLowering::supportSwiftError() const {
57619   return Subtarget.is64Bit();
57620 }
57621 
57622 MachineInstr *
57623 X86TargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
57624                                  MachineBasicBlock::instr_iterator &MBBI,
57625                                  const TargetInstrInfo *TII) const {
57626   assert(MBBI->isCall() && MBBI->getCFIType() &&
57627          "Invalid call instruction for a KCFI check");
57628 
57629   MachineFunction &MF = *MBB.getParent();
57630   // If the call target is a memory operand, unfold it and use R11 for the
57631   // call, so KCFI_CHECK won't have to recompute the address.
57632   switch (MBBI->getOpcode()) {
57633   case X86::CALL64m:
57634   case X86::CALL64m_NT:
57635   case X86::TAILJMPm64:
57636   case X86::TAILJMPm64_REX: {
57637     MachineBasicBlock::instr_iterator OrigCall = MBBI;
57638     SmallVector<MachineInstr *, 2> NewMIs;
57639     if (!TII->unfoldMemoryOperand(MF, *OrigCall, X86::R11, /*UnfoldLoad=*/true,
57640                                   /*UnfoldStore=*/false, NewMIs))
57641       report_fatal_error("Failed to unfold memory operand for a KCFI check");
57642     for (auto *NewMI : NewMIs)
57643       MBBI = MBB.insert(OrigCall, NewMI);
57644     assert(MBBI->isCall() &&
57645            "Unexpected instruction after memory operand unfolding");
57646     if (OrigCall->shouldUpdateCallSiteInfo())
57647       MF.moveCallSiteInfo(&*OrigCall, &*MBBI);
57648     MBBI->setCFIType(MF, OrigCall->getCFIType());
57649     OrigCall->eraseFromParent();
57650     break;
57651   }
57652   default:
57653     break;
57654   }
57655 
57656   MachineOperand &Target = MBBI->getOperand(0);
57657   Register TargetReg;
57658   switch (MBBI->getOpcode()) {
57659   case X86::CALL64r:
57660   case X86::CALL64r_NT:
57661   case X86::TAILJMPr64:
57662   case X86::TAILJMPr64_REX:
57663     assert(Target.isReg() && "Unexpected target operand for an indirect call");
57664     Target.setIsRenamable(false);
57665     TargetReg = Target.getReg();
57666     break;
57667   case X86::CALL64pcrel32:
57668   case X86::TAILJMPd64:
57669     assert(Target.isSymbol() && "Unexpected target operand for a direct call");
57670     // X86TargetLowering::EmitLoweredIndirectThunk always uses r11 for
57671     // 64-bit indirect thunk calls.
57672     assert(StringRef(Target.getSymbolName()).ends_with("_r11") &&
57673            "Unexpected register for an indirect thunk call");
57674     TargetReg = X86::R11;
57675     break;
57676   default:
57677     llvm_unreachable("Unexpected CFI call opcode");
57678     break;
57679   }
57680 
57681   return BuildMI(MBB, MBBI, MIMetadata(*MBBI), TII->get(X86::KCFI_CHECK))
57682       .addReg(TargetReg)
57683       .addImm(MBBI->getCFIType())
57684       .getInstr();
57685 }
57686 
57687 /// Returns true if stack probing through a function call is requested.
57688 bool X86TargetLowering::hasStackProbeSymbol(const MachineFunction &MF) const {
57689   return !getStackProbeSymbolName(MF).empty();
57690 }
57691 
57692 /// Returns true if stack probing through inline assembly is requested.
57693 bool X86TargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
57694 
57695   // No inline stack probe for Windows, they have their own mechanism.
57696   if (Subtarget.isOSWindows() ||
57697       MF.getFunction().hasFnAttribute("no-stack-arg-probe"))
57698     return false;
57699 
57700   // If the function specifically requests inline stack probes, emit them.
57701   if (MF.getFunction().hasFnAttribute("probe-stack"))
57702     return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
57703            "inline-asm";
57704 
57705   return false;
57706 }
57707 
57708 /// Returns the name of the symbol used to emit stack probes or the empty
57709 /// string if not applicable.
57710 StringRef
57711 X86TargetLowering::getStackProbeSymbolName(const MachineFunction &MF) const {
57712   // Inline Stack probes disable stack probe call
57713   if (hasInlineStackProbe(MF))
57714     return "";
57715 
57716   // If the function specifically requests stack probes, emit them.
57717   if (MF.getFunction().hasFnAttribute("probe-stack"))
57718     return MF.getFunction().getFnAttribute("probe-stack").getValueAsString();
57719 
57720   // Generally, if we aren't on Windows, the platform ABI does not include
57721   // support for stack probes, so don't emit them.
57722   if (!Subtarget.isOSWindows() || Subtarget.isTargetMachO() ||
57723       MF.getFunction().hasFnAttribute("no-stack-arg-probe"))
57724     return "";
57725 
57726   // We need a stack probe to conform to the Windows ABI. Choose the right
57727   // symbol.
57728   if (Subtarget.is64Bit())
57729     return Subtarget.isTargetCygMing() ? "___chkstk_ms" : "__chkstk";
57730   return Subtarget.isTargetCygMing() ? "_alloca" : "_chkstk";
57731 }
57732 
57733 unsigned
57734 X86TargetLowering::getStackProbeSize(const MachineFunction &MF) const {
57735   // The default stack probe size is 4096 if the function has no stackprobesize
57736   // attribute.
57737   return MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size",
57738                                                         4096);
57739 }
57740 
57741 Align X86TargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
57742   if (ML && ML->isInnermost() &&
57743       ExperimentalPrefInnermostLoopAlignment.getNumOccurrences())
57744     return Align(1ULL << ExperimentalPrefInnermostLoopAlignment);
57745   return TargetLowering::getPrefLoopAlignment();
57746 }
57747