xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (revision e6bfd18d21b225af6a0ed67ceeaf1293b7b9eba5)
1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements routines for translating from LLVM IR into SelectionDAG IR.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "SelectionDAGBuilder.h"
14 #include "SDNodeDbgValue.h"
15 #include "llvm/ADT/APFloat.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/ADT/BitVector.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallPtrSet.h"
22 #include "llvm/ADT/SmallSet.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/Triple.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/Analysis/AliasAnalysis.h"
27 #include "llvm/Analysis/BranchProbabilityInfo.h"
28 #include "llvm/Analysis/ConstantFolding.h"
29 #include "llvm/Analysis/EHPersonalities.h"
30 #include "llvm/Analysis/MemoryLocation.h"
31 #include "llvm/Analysis/TargetLibraryInfo.h"
32 #include "llvm/Analysis/ValueTracking.h"
33 #include "llvm/CodeGen/Analysis.h"
34 #include "llvm/CodeGen/CodeGenCommonISel.h"
35 #include "llvm/CodeGen/FunctionLoweringInfo.h"
36 #include "llvm/CodeGen/GCMetadata.h"
37 #include "llvm/CodeGen/MachineBasicBlock.h"
38 #include "llvm/CodeGen/MachineFrameInfo.h"
39 #include "llvm/CodeGen/MachineFunction.h"
40 #include "llvm/CodeGen/MachineInstrBuilder.h"
41 #include "llvm/CodeGen/MachineInstrBundleIterator.h"
42 #include "llvm/CodeGen/MachineMemOperand.h"
43 #include "llvm/CodeGen/MachineModuleInfo.h"
44 #include "llvm/CodeGen/MachineOperand.h"
45 #include "llvm/CodeGen/MachineRegisterInfo.h"
46 #include "llvm/CodeGen/RuntimeLibcalls.h"
47 #include "llvm/CodeGen/SelectionDAG.h"
48 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
49 #include "llvm/CodeGen/StackMaps.h"
50 #include "llvm/CodeGen/SwiftErrorValueTracking.h"
51 #include "llvm/CodeGen/TargetFrameLowering.h"
52 #include "llvm/CodeGen/TargetInstrInfo.h"
53 #include "llvm/CodeGen/TargetOpcodes.h"
54 #include "llvm/CodeGen/TargetRegisterInfo.h"
55 #include "llvm/CodeGen/TargetSubtargetInfo.h"
56 #include "llvm/CodeGen/WinEHFuncInfo.h"
57 #include "llvm/IR/Argument.h"
58 #include "llvm/IR/Attributes.h"
59 #include "llvm/IR/BasicBlock.h"
60 #include "llvm/IR/CFG.h"
61 #include "llvm/IR/CallingConv.h"
62 #include "llvm/IR/Constant.h"
63 #include "llvm/IR/ConstantRange.h"
64 #include "llvm/IR/Constants.h"
65 #include "llvm/IR/DataLayout.h"
66 #include "llvm/IR/DebugInfoMetadata.h"
67 #include "llvm/IR/DerivedTypes.h"
68 #include "llvm/IR/DiagnosticInfo.h"
69 #include "llvm/IR/Function.h"
70 #include "llvm/IR/GetElementPtrTypeIterator.h"
71 #include "llvm/IR/InlineAsm.h"
72 #include "llvm/IR/InstrTypes.h"
73 #include "llvm/IR/Instructions.h"
74 #include "llvm/IR/IntrinsicInst.h"
75 #include "llvm/IR/Intrinsics.h"
76 #include "llvm/IR/IntrinsicsAArch64.h"
77 #include "llvm/IR/IntrinsicsWebAssembly.h"
78 #include "llvm/IR/LLVMContext.h"
79 #include "llvm/IR/Metadata.h"
80 #include "llvm/IR/Module.h"
81 #include "llvm/IR/Operator.h"
82 #include "llvm/IR/PatternMatch.h"
83 #include "llvm/IR/Statepoint.h"
84 #include "llvm/IR/Type.h"
85 #include "llvm/IR/User.h"
86 #include "llvm/IR/Value.h"
87 #include "llvm/MC/MCContext.h"
88 #include "llvm/Support/AtomicOrdering.h"
89 #include "llvm/Support/Casting.h"
90 #include "llvm/Support/CommandLine.h"
91 #include "llvm/Support/Compiler.h"
92 #include "llvm/Support/Debug.h"
93 #include "llvm/Support/MathExtras.h"
94 #include "llvm/Support/raw_ostream.h"
95 #include "llvm/Target/TargetIntrinsicInfo.h"
96 #include "llvm/Target/TargetMachine.h"
97 #include "llvm/Target/TargetOptions.h"
98 #include "llvm/Transforms/Utils/Local.h"
99 #include <cstddef>
100 #include <iterator>
101 #include <limits>
102 #include <tuple>
103 
104 using namespace llvm;
105 using namespace PatternMatch;
106 using namespace SwitchCG;
107 
108 #define DEBUG_TYPE "isel"
109 
110 /// LimitFloatPrecision - Generate low-precision inline sequences for
111 /// some float libcalls (6, 8 or 12 bits).
112 static unsigned LimitFloatPrecision;
113 
114 static cl::opt<bool>
115     InsertAssertAlign("insert-assert-align", cl::init(true),
116                       cl::desc("Insert the experimental `assertalign` node."),
117                       cl::ReallyHidden);
118 
119 static cl::opt<unsigned, true>
120     LimitFPPrecision("limit-float-precision",
121                      cl::desc("Generate low-precision inline sequences "
122                               "for some float libcalls"),
123                      cl::location(LimitFloatPrecision), cl::Hidden,
124                      cl::init(0));
125 
126 static cl::opt<unsigned> SwitchPeelThreshold(
127     "switch-peel-threshold", cl::Hidden, cl::init(66),
128     cl::desc("Set the case probability threshold for peeling the case from a "
129              "switch statement. A value greater than 100 will void this "
130              "optimization"));
131 
132 // Limit the width of DAG chains. This is important in general to prevent
133 // DAG-based analysis from blowing up. For example, alias analysis and
134 // load clustering may not complete in reasonable time. It is difficult to
135 // recognize and avoid this situation within each individual analysis, and
136 // future analyses are likely to have the same behavior. Limiting DAG width is
137 // the safe approach and will be especially important with global DAGs.
138 //
139 // MaxParallelChains default is arbitrarily high to avoid affecting
140 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st
141 // sequence over this should have been converted to llvm.memcpy by the
142 // frontend. It is easy to induce this behavior with .ll code such as:
143 // %buffer = alloca [4096 x i8]
144 // %data = load [4096 x i8]* %argPtr
145 // store [4096 x i8] %data, [4096 x i8]* %buffer
146 static const unsigned MaxParallelChains = 64;
147 
148 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
149                                       const SDValue *Parts, unsigned NumParts,
150                                       MVT PartVT, EVT ValueVT, const Value *V,
151                                       Optional<CallingConv::ID> CC);
152 
153 /// getCopyFromParts - Create a value that contains the specified legal parts
154 /// combined into the value they represent.  If the parts combine to a type
155 /// larger than ValueVT then AssertOp can be used to specify whether the extra
156 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
157 /// (ISD::AssertSext).
158 static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
159                                 const SDValue *Parts, unsigned NumParts,
160                                 MVT PartVT, EVT ValueVT, const Value *V,
161                                 Optional<CallingConv::ID> CC = None,
162                                 Optional<ISD::NodeType> AssertOp = None) {
163   // Let the target assemble the parts if it wants to
164   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
165   if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
166                                                    PartVT, ValueVT, CC))
167     return Val;
168 
169   if (ValueVT.isVector())
170     return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
171                                   CC);
172 
173   assert(NumParts > 0 && "No parts to assemble!");
174   SDValue Val = Parts[0];
175 
176   if (NumParts > 1) {
177     // Assemble the value from multiple parts.
178     if (ValueVT.isInteger()) {
179       unsigned PartBits = PartVT.getSizeInBits();
180       unsigned ValueBits = ValueVT.getSizeInBits();
181 
182       // Assemble the power of 2 part.
183       unsigned RoundParts =
184           (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts;
185       unsigned RoundBits = PartBits * RoundParts;
186       EVT RoundVT = RoundBits == ValueBits ?
187         ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
188       SDValue Lo, Hi;
189 
190       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
191 
192       if (RoundParts > 2) {
193         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
194                               PartVT, HalfVT, V);
195         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
196                               RoundParts / 2, PartVT, HalfVT, V);
197       } else {
198         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
199         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
200       }
201 
202       if (DAG.getDataLayout().isBigEndian())
203         std::swap(Lo, Hi);
204 
205       Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
206 
207       if (RoundParts < NumParts) {
208         // Assemble the trailing non-power-of-2 part.
209         unsigned OddParts = NumParts - RoundParts;
210         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
211         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
212                               OddVT, V, CC);
213 
214         // Combine the round and odd parts.
215         Lo = Val;
216         if (DAG.getDataLayout().isBigEndian())
217           std::swap(Lo, Hi);
218         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
219         Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
220         Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
221                          DAG.getConstant(Lo.getValueSizeInBits(), DL,
222                                          TLI.getShiftAmountTy(
223                                              TotalVT, DAG.getDataLayout())));
224         Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
225         Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
226       }
227     } else if (PartVT.isFloatingPoint()) {
228       // FP split into multiple FP parts (for ppcf128)
229       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
230              "Unexpected split");
231       SDValue Lo, Hi;
232       Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
233       Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
234       if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
235         std::swap(Lo, Hi);
236       Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
237     } else {
238       // FP split into integer parts (soft fp)
239       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
240              !PartVT.isVector() && "Unexpected split");
241       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
242       Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
243     }
244   }
245 
246   // There is now one part, held in Val.  Correct it to match ValueVT.
247   // PartEVT is the type of the register class that holds the value.
248   // ValueVT is the type of the inline asm operation.
249   EVT PartEVT = Val.getValueType();
250 
251   if (PartEVT == ValueVT)
252     return Val;
253 
254   if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
255       ValueVT.bitsLT(PartEVT)) {
256     // For an FP value in an integer part, we need to truncate to the right
257     // width first.
258     PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
259     Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
260   }
261 
262   // Handle types that have the same size.
263   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
264     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
265 
266   // Handle types with different sizes.
267   if (PartEVT.isInteger() && ValueVT.isInteger()) {
268     if (ValueVT.bitsLT(PartEVT)) {
269       // For a truncate, see if we have any information to
270       // indicate whether the truncated bits will always be
271       // zero or sign-extension.
272       if (AssertOp)
273         Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
274                           DAG.getValueType(ValueVT));
275       return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
276     }
277     return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
278   }
279 
280   if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
281     // FP_ROUND's are always exact here.
282     if (ValueVT.bitsLT(Val.getValueType()))
283       return DAG.getNode(
284           ISD::FP_ROUND, DL, ValueVT, Val,
285           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
286 
287     return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
288   }
289 
290   // Handle MMX to a narrower integer type by bitcasting MMX to integer and
291   // then truncating.
292   if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
293       ValueVT.bitsLT(PartEVT)) {
294     Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
295     return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
296   }
297 
298   report_fatal_error("Unknown mismatch in getCopyFromParts!");
299 }
300 
301 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
302                                               const Twine &ErrMsg) {
303   const Instruction *I = dyn_cast_or_null<Instruction>(V);
304   if (!V)
305     return Ctx.emitError(ErrMsg);
306 
307   const char *AsmError = ", possible invalid constraint for vector type";
308   if (const CallInst *CI = dyn_cast<CallInst>(I))
309     if (CI->isInlineAsm())
310       return Ctx.emitError(I, ErrMsg + AsmError);
311 
312   return Ctx.emitError(I, ErrMsg);
313 }
314 
315 /// getCopyFromPartsVector - Create a value that contains the specified legal
316 /// parts combined into the value they represent.  If the parts combine to a
317 /// type larger than ValueVT then AssertOp can be used to specify whether the
318 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from
319 /// ValueVT (ISD::AssertSext).
320 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
321                                       const SDValue *Parts, unsigned NumParts,
322                                       MVT PartVT, EVT ValueVT, const Value *V,
323                                       Optional<CallingConv::ID> CallConv) {
324   assert(ValueVT.isVector() && "Not a vector value");
325   assert(NumParts > 0 && "No parts to assemble!");
326   const bool IsABIRegCopy = CallConv.has_value();
327 
328   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
329   SDValue Val = Parts[0];
330 
331   // Handle a multi-element vector.
332   if (NumParts > 1) {
333     EVT IntermediateVT;
334     MVT RegisterVT;
335     unsigned NumIntermediates;
336     unsigned NumRegs;
337 
338     if (IsABIRegCopy) {
339       NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
340           *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
341           NumIntermediates, RegisterVT);
342     } else {
343       NumRegs =
344           TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
345                                      NumIntermediates, RegisterVT);
346     }
347 
348     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
349     NumParts = NumRegs; // Silence a compiler warning.
350     assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
351     assert(RegisterVT.getSizeInBits() ==
352            Parts[0].getSimpleValueType().getSizeInBits() &&
353            "Part type sizes don't match!");
354 
355     // Assemble the parts into intermediate operands.
356     SmallVector<SDValue, 8> Ops(NumIntermediates);
357     if (NumIntermediates == NumParts) {
358       // If the register was not expanded, truncate or copy the value,
359       // as appropriate.
360       for (unsigned i = 0; i != NumParts; ++i)
361         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
362                                   PartVT, IntermediateVT, V, CallConv);
363     } else if (NumParts > 0) {
364       // If the intermediate type was expanded, build the intermediate
365       // operands from the parts.
366       assert(NumParts % NumIntermediates == 0 &&
367              "Must expand into a divisible number of parts!");
368       unsigned Factor = NumParts / NumIntermediates;
369       for (unsigned i = 0; i != NumIntermediates; ++i)
370         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
371                                   PartVT, IntermediateVT, V, CallConv);
372     }
373 
374     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
375     // intermediate operands.
376     EVT BuiltVectorTy =
377         IntermediateVT.isVector()
378             ? EVT::getVectorVT(
379                   *DAG.getContext(), IntermediateVT.getScalarType(),
380                   IntermediateVT.getVectorElementCount() * NumParts)
381             : EVT::getVectorVT(*DAG.getContext(),
382                                IntermediateVT.getScalarType(),
383                                NumIntermediates);
384     Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
385                                                 : ISD::BUILD_VECTOR,
386                       DL, BuiltVectorTy, Ops);
387   }
388 
389   // There is now one part, held in Val.  Correct it to match ValueVT.
390   EVT PartEVT = Val.getValueType();
391 
392   if (PartEVT == ValueVT)
393     return Val;
394 
395   if (PartEVT.isVector()) {
396     // Vector/Vector bitcast.
397     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
398       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
399 
400     // If the element type of the source/dest vectors are the same, but the
401     // parts vector has more elements than the value vector, then we have a
402     // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
403     // elements we want.
404     if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
405       assert((PartEVT.getVectorElementCount().getKnownMinValue() >
406               ValueVT.getVectorElementCount().getKnownMinValue()) &&
407              (PartEVT.getVectorElementCount().isScalable() ==
408               ValueVT.getVectorElementCount().isScalable()) &&
409              "Cannot narrow, it would be a lossy transformation");
410       PartEVT =
411           EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),
412                            ValueVT.getVectorElementCount());
413       Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
414                         DAG.getVectorIdxConstant(0, DL));
415       if (PartEVT == ValueVT)
416         return Val;
417     }
418 
419     // Promoted vector extract
420     return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
421   }
422 
423   // Trivial bitcast if the types are the same size and the destination
424   // vector type is legal.
425   if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
426       TLI.isTypeLegal(ValueVT))
427     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
428 
429   if (ValueVT.getVectorNumElements() != 1) {
430      // Certain ABIs require that vectors are passed as integers. For vectors
431      // are the same size, this is an obvious bitcast.
432      if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
433        return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
434      } else if (ValueVT.bitsLT(PartEVT)) {
435        const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
436        EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
437        // Drop the extra bits.
438        Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
439        return DAG.getBitcast(ValueVT, Val);
440      }
441 
442      diagnosePossiblyInvalidConstraint(
443          *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
444      return DAG.getUNDEF(ValueVT);
445   }
446 
447   // Handle cases such as i8 -> <1 x i1>
448   EVT ValueSVT = ValueVT.getVectorElementType();
449   if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
450     if (ValueSVT.getSizeInBits() == PartEVT.getSizeInBits())
451       Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
452     else
453       Val = ValueVT.isFloatingPoint()
454                 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
455                 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
456   }
457 
458   return DAG.getBuildVector(ValueVT, DL, Val);
459 }
460 
461 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
462                                  SDValue Val, SDValue *Parts, unsigned NumParts,
463                                  MVT PartVT, const Value *V,
464                                  Optional<CallingConv::ID> CallConv);
465 
466 /// getCopyToParts - Create a series of nodes that contain the specified value
467 /// split into legal parts.  If the parts contain more bits than Val, then, for
468 /// integers, ExtendKind can be used to specify how to generate the extra bits.
469 static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
470                            SDValue *Parts, unsigned NumParts, MVT PartVT,
471                            const Value *V,
472                            Optional<CallingConv::ID> CallConv = None,
473                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
474   // Let the target split the parts if it wants to
475   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
476   if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
477                                       CallConv))
478     return;
479   EVT ValueVT = Val.getValueType();
480 
481   // Handle the vector case separately.
482   if (ValueVT.isVector())
483     return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
484                                 CallConv);
485 
486   unsigned PartBits = PartVT.getSizeInBits();
487   unsigned OrigNumParts = NumParts;
488   assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
489          "Copying to an illegal type!");
490 
491   if (NumParts == 0)
492     return;
493 
494   assert(!ValueVT.isVector() && "Vector case handled elsewhere");
495   EVT PartEVT = PartVT;
496   if (PartEVT == ValueVT) {
497     assert(NumParts == 1 && "No-op copy with multiple parts!");
498     Parts[0] = Val;
499     return;
500   }
501 
502   if (NumParts * PartBits > ValueVT.getSizeInBits()) {
503     // If the parts cover more bits than the value has, promote the value.
504     if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
505       assert(NumParts == 1 && "Do not know what to promote to!");
506       Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
507     } else {
508       if (ValueVT.isFloatingPoint()) {
509         // FP values need to be bitcast, then extended if they are being put
510         // into a larger container.
511         ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
512         Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
513       }
514       assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
515              ValueVT.isInteger() &&
516              "Unknown mismatch!");
517       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
518       Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
519       if (PartVT == MVT::x86mmx)
520         Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
521     }
522   } else if (PartBits == ValueVT.getSizeInBits()) {
523     // Different types of the same size.
524     assert(NumParts == 1 && PartEVT != ValueVT);
525     Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
526   } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
527     // If the parts cover less bits than value has, truncate the value.
528     assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
529            ValueVT.isInteger() &&
530            "Unknown mismatch!");
531     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
532     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
533     if (PartVT == MVT::x86mmx)
534       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
535   }
536 
537   // The value may have changed - recompute ValueVT.
538   ValueVT = Val.getValueType();
539   assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
540          "Failed to tile the value with PartVT!");
541 
542   if (NumParts == 1) {
543     if (PartEVT != ValueVT) {
544       diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
545                                         "scalar-to-vector conversion failed");
546       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
547     }
548 
549     Parts[0] = Val;
550     return;
551   }
552 
553   // Expand the value into multiple parts.
554   if (NumParts & (NumParts - 1)) {
555     // The number of parts is not a power of 2.  Split off and copy the tail.
556     assert(PartVT.isInteger() && ValueVT.isInteger() &&
557            "Do not know what to expand to!");
558     unsigned RoundParts = 1 << Log2_32(NumParts);
559     unsigned RoundBits = RoundParts * PartBits;
560     unsigned OddParts = NumParts - RoundParts;
561     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
562       DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
563 
564     getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
565                    CallConv);
566 
567     if (DAG.getDataLayout().isBigEndian())
568       // The odd parts were reversed by getCopyToParts - unreverse them.
569       std::reverse(Parts + RoundParts, Parts + NumParts);
570 
571     NumParts = RoundParts;
572     ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
573     Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
574   }
575 
576   // The number of parts is a power of 2.  Repeatedly bisect the value using
577   // EXTRACT_ELEMENT.
578   Parts[0] = DAG.getNode(ISD::BITCAST, DL,
579                          EVT::getIntegerVT(*DAG.getContext(),
580                                            ValueVT.getSizeInBits()),
581                          Val);
582 
583   for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
584     for (unsigned i = 0; i < NumParts; i += StepSize) {
585       unsigned ThisBits = StepSize * PartBits / 2;
586       EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
587       SDValue &Part0 = Parts[i];
588       SDValue &Part1 = Parts[i+StepSize/2];
589 
590       Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
591                           ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
592       Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
593                           ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
594 
595       if (ThisBits == PartBits && ThisVT != PartVT) {
596         Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
597         Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
598       }
599     }
600   }
601 
602   if (DAG.getDataLayout().isBigEndian())
603     std::reverse(Parts, Parts + OrigNumParts);
604 }
605 
606 static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
607                                      const SDLoc &DL, EVT PartVT) {
608   if (!PartVT.isVector())
609     return SDValue();
610 
611   EVT ValueVT = Val.getValueType();
612   ElementCount PartNumElts = PartVT.getVectorElementCount();
613   ElementCount ValueNumElts = ValueVT.getVectorElementCount();
614 
615   // We only support widening vectors with equivalent element types and
616   // fixed/scalable properties. If a target needs to widen a fixed-length type
617   // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
618   if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
619       PartNumElts.isScalable() != ValueNumElts.isScalable() ||
620       PartVT.getVectorElementType() != ValueVT.getVectorElementType())
621     return SDValue();
622 
623   // Widening a scalable vector to another scalable vector is done by inserting
624   // the vector into a larger undef one.
625   if (PartNumElts.isScalable())
626     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
627                        Val, DAG.getVectorIdxConstant(0, DL));
628 
629   EVT ElementVT = PartVT.getVectorElementType();
630   // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
631   // undef elements.
632   SmallVector<SDValue, 16> Ops;
633   DAG.ExtractVectorElements(Val, Ops);
634   SDValue EltUndef = DAG.getUNDEF(ElementVT);
635   Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
636 
637   // FIXME: Use CONCAT for 2x -> 4x.
638   return DAG.getBuildVector(PartVT, DL, Ops);
639 }
640 
641 /// getCopyToPartsVector - Create a series of nodes that contain the specified
642 /// value split into legal parts.
643 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
644                                  SDValue Val, SDValue *Parts, unsigned NumParts,
645                                  MVT PartVT, const Value *V,
646                                  Optional<CallingConv::ID> CallConv) {
647   EVT ValueVT = Val.getValueType();
648   assert(ValueVT.isVector() && "Not a vector");
649   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
650   const bool IsABIRegCopy = CallConv.has_value();
651 
652   if (NumParts == 1) {
653     EVT PartEVT = PartVT;
654     if (PartEVT == ValueVT) {
655       // Nothing to do.
656     } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
657       // Bitconvert vector->vector case.
658       Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
659     } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
660       Val = Widened;
661     } else if (PartVT.isVector() &&
662                PartEVT.getVectorElementType().bitsGE(
663                    ValueVT.getVectorElementType()) &&
664                PartEVT.getVectorElementCount() ==
665                    ValueVT.getVectorElementCount()) {
666 
667       // Promoted vector extract
668       Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
669     } else if (PartEVT.isVector() &&
670                PartEVT.getVectorElementType() !=
671                    ValueVT.getVectorElementType() &&
672                TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
673                    TargetLowering::TypeWidenVector) {
674       // Combination of widening and promotion.
675       EVT WidenVT =
676           EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(),
677                            PartVT.getVectorElementCount());
678       SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
679       Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
680     } else {
681       if (ValueVT.getVectorElementCount().isScalar()) {
682         Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
683                           DAG.getVectorIdxConstant(0, DL));
684       } else {
685         uint64_t ValueSize = ValueVT.getFixedSizeInBits();
686         assert(PartVT.getFixedSizeInBits() > ValueSize &&
687                "lossy conversion of vector to scalar type");
688         EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
689         Val = DAG.getBitcast(IntermediateType, Val);
690         Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
691       }
692     }
693 
694     assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
695     Parts[0] = Val;
696     return;
697   }
698 
699   // Handle a multi-element vector.
700   EVT IntermediateVT;
701   MVT RegisterVT;
702   unsigned NumIntermediates;
703   unsigned NumRegs;
704   if (IsABIRegCopy) {
705     NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
706         *DAG.getContext(), CallConv.value(), ValueVT, IntermediateVT,
707         NumIntermediates, RegisterVT);
708   } else {
709     NumRegs =
710         TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
711                                    NumIntermediates, RegisterVT);
712   }
713 
714   assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
715   NumParts = NumRegs; // Silence a compiler warning.
716   assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
717 
718   assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
719          "Mixing scalable and fixed vectors when copying in parts");
720 
721   Optional<ElementCount> DestEltCnt;
722 
723   if (IntermediateVT.isVector())
724     DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
725   else
726     DestEltCnt = ElementCount::getFixed(NumIntermediates);
727 
728   EVT BuiltVectorTy = EVT::getVectorVT(
729       *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
730 
731   if (ValueVT == BuiltVectorTy) {
732     // Nothing to do.
733   } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
734     // Bitconvert vector->vector case.
735     Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
736   } else {
737     if (BuiltVectorTy.getVectorElementType().bitsGT(
738             ValueVT.getVectorElementType())) {
739       // Integer promotion.
740       ValueVT = EVT::getVectorVT(*DAG.getContext(),
741                                  BuiltVectorTy.getVectorElementType(),
742                                  ValueVT.getVectorElementCount());
743       Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
744     }
745 
746     if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
747       Val = Widened;
748     }
749   }
750 
751   assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
752 
753   // Split the vector into intermediate operands.
754   SmallVector<SDValue, 8> Ops(NumIntermediates);
755   for (unsigned i = 0; i != NumIntermediates; ++i) {
756     if (IntermediateVT.isVector()) {
757       // This does something sensible for scalable vectors - see the
758       // definition of EXTRACT_SUBVECTOR for further details.
759       unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
760       Ops[i] =
761           DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
762                       DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
763     } else {
764       Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
765                            DAG.getVectorIdxConstant(i, DL));
766     }
767   }
768 
769   // Split the intermediate operands into legal parts.
770   if (NumParts == NumIntermediates) {
771     // If the register was not expanded, promote or copy the value,
772     // as appropriate.
773     for (unsigned i = 0; i != NumParts; ++i)
774       getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
775   } else if (NumParts > 0) {
776     // If the intermediate type was expanded, split each the value into
777     // legal parts.
778     assert(NumIntermediates != 0 && "division by zero");
779     assert(NumParts % NumIntermediates == 0 &&
780            "Must expand into a divisible number of parts!");
781     unsigned Factor = NumParts / NumIntermediates;
782     for (unsigned i = 0; i != NumIntermediates; ++i)
783       getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
784                      CallConv);
785   }
786 }
787 
788 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
789                            EVT valuevt, Optional<CallingConv::ID> CC)
790     : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
791       RegCount(1, regs.size()), CallConv(CC) {}
792 
793 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
794                            const DataLayout &DL, unsigned Reg, Type *Ty,
795                            Optional<CallingConv::ID> CC) {
796   ComputeValueVTs(TLI, DL, Ty, ValueVTs);
797 
798   CallConv = CC;
799 
800   for (EVT ValueVT : ValueVTs) {
801     unsigned NumRegs =
802         isABIMangled()
803             ? TLI.getNumRegistersForCallingConv(Context, CC.value(), ValueVT)
804             : TLI.getNumRegisters(Context, ValueVT);
805     MVT RegisterVT =
806         isABIMangled()
807             ? TLI.getRegisterTypeForCallingConv(Context, CC.value(), ValueVT)
808             : TLI.getRegisterType(Context, ValueVT);
809     for (unsigned i = 0; i != NumRegs; ++i)
810       Regs.push_back(Reg + i);
811     RegVTs.push_back(RegisterVT);
812     RegCount.push_back(NumRegs);
813     Reg += NumRegs;
814   }
815 }
816 
817 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
818                                       FunctionLoweringInfo &FuncInfo,
819                                       const SDLoc &dl, SDValue &Chain,
820                                       SDValue *Flag, const Value *V) const {
821   // A Value with type {} or [0 x %t] needs no registers.
822   if (ValueVTs.empty())
823     return SDValue();
824 
825   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
826 
827   // Assemble the legal parts into the final values.
828   SmallVector<SDValue, 4> Values(ValueVTs.size());
829   SmallVector<SDValue, 8> Parts;
830   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
831     // Copy the legal parts from the registers.
832     EVT ValueVT = ValueVTs[Value];
833     unsigned NumRegs = RegCount[Value];
834     MVT RegisterVT =
835         isABIMangled() ? TLI.getRegisterTypeForCallingConv(
836                              *DAG.getContext(), CallConv.value(), RegVTs[Value])
837                        : RegVTs[Value];
838 
839     Parts.resize(NumRegs);
840     for (unsigned i = 0; i != NumRegs; ++i) {
841       SDValue P;
842       if (!Flag) {
843         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
844       } else {
845         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
846         *Flag = P.getValue(2);
847       }
848 
849       Chain = P.getValue(1);
850       Parts[i] = P;
851 
852       // If the source register was virtual and if we know something about it,
853       // add an assert node.
854       if (!Register::isVirtualRegister(Regs[Part + i]) ||
855           !RegisterVT.isInteger())
856         continue;
857 
858       const FunctionLoweringInfo::LiveOutInfo *LOI =
859         FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
860       if (!LOI)
861         continue;
862 
863       unsigned RegSize = RegisterVT.getScalarSizeInBits();
864       unsigned NumSignBits = LOI->NumSignBits;
865       unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
866 
867       if (NumZeroBits == RegSize) {
868         // The current value is a zero.
869         // Explicitly express that as it would be easier for
870         // optimizations to kick in.
871         Parts[i] = DAG.getConstant(0, dl, RegisterVT);
872         continue;
873       }
874 
875       // FIXME: We capture more information than the dag can represent.  For
876       // now, just use the tightest assertzext/assertsext possible.
877       bool isSExt;
878       EVT FromVT(MVT::Other);
879       if (NumZeroBits) {
880         FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
881         isSExt = false;
882       } else if (NumSignBits > 1) {
883         FromVT =
884             EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
885         isSExt = true;
886       } else {
887         continue;
888       }
889       // Add an assertion node.
890       assert(FromVT != MVT::Other);
891       Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
892                              RegisterVT, P, DAG.getValueType(FromVT));
893     }
894 
895     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
896                                      RegisterVT, ValueVT, V, CallConv);
897     Part += NumRegs;
898     Parts.clear();
899   }
900 
901   return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
902 }
903 
904 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
905                                  const SDLoc &dl, SDValue &Chain, SDValue *Flag,
906                                  const Value *V,
907                                  ISD::NodeType PreferredExtendType) const {
908   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
909   ISD::NodeType ExtendKind = PreferredExtendType;
910 
911   // Get the list of the values's legal parts.
912   unsigned NumRegs = Regs.size();
913   SmallVector<SDValue, 8> Parts(NumRegs);
914   for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
915     unsigned NumParts = RegCount[Value];
916 
917     MVT RegisterVT =
918         isABIMangled() ? TLI.getRegisterTypeForCallingConv(
919                              *DAG.getContext(), CallConv.value(), RegVTs[Value])
920                        : RegVTs[Value];
921 
922     if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
923       ExtendKind = ISD::ZERO_EXTEND;
924 
925     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
926                    NumParts, RegisterVT, V, CallConv, ExtendKind);
927     Part += NumParts;
928   }
929 
930   // Copy the parts into the registers.
931   SmallVector<SDValue, 8> Chains(NumRegs);
932   for (unsigned i = 0; i != NumRegs; ++i) {
933     SDValue Part;
934     if (!Flag) {
935       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
936     } else {
937       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
938       *Flag = Part.getValue(1);
939     }
940 
941     Chains[i] = Part.getValue(0);
942   }
943 
944   if (NumRegs == 1 || Flag)
945     // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
946     // flagged to it. That is the CopyToReg nodes and the user are considered
947     // a single scheduling unit. If we create a TokenFactor and return it as
948     // chain, then the TokenFactor is both a predecessor (operand) of the
949     // user as well as a successor (the TF operands are flagged to the user).
950     // c1, f1 = CopyToReg
951     // c2, f2 = CopyToReg
952     // c3     = TokenFactor c1, c2
953     // ...
954     //        = op c3, ..., f2
955     Chain = Chains[NumRegs-1];
956   else
957     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
958 }
959 
960 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
961                                         unsigned MatchingIdx, const SDLoc &dl,
962                                         SelectionDAG &DAG,
963                                         std::vector<SDValue> &Ops) const {
964   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
965 
966   unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
967   if (HasMatching)
968     Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
969   else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
970     // Put the register class of the virtual registers in the flag word.  That
971     // way, later passes can recompute register class constraints for inline
972     // assembly as well as normal instructions.
973     // Don't do this for tied operands that can use the regclass information
974     // from the def.
975     const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
976     const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
977     Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
978   }
979 
980   SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
981   Ops.push_back(Res);
982 
983   if (Code == InlineAsm::Kind_Clobber) {
984     // Clobbers should always have a 1:1 mapping with registers, and may
985     // reference registers that have illegal (e.g. vector) types. Hence, we
986     // shouldn't try to apply any sort of splitting logic to them.
987     assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
988            "No 1:1 mapping from clobbers to regs?");
989     Register SP = TLI.getStackPointerRegisterToSaveRestore();
990     (void)SP;
991     for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
992       Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
993       assert(
994           (Regs[I] != SP ||
995            DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
996           "If we clobbered the stack pointer, MFI should know about it.");
997     }
998     return;
999   }
1000 
1001   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1002     MVT RegisterVT = RegVTs[Value];
1003     unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1004                                            RegisterVT);
1005     for (unsigned i = 0; i != NumRegs; ++i) {
1006       assert(Reg < Regs.size() && "Mismatch in # registers expected");
1007       unsigned TheReg = Regs[Reg++];
1008       Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1009     }
1010   }
1011 }
1012 
1013 SmallVector<std::pair<unsigned, TypeSize>, 4>
1014 RegsForValue::getRegsAndSizes() const {
1015   SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;
1016   unsigned I = 0;
1017   for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1018     unsigned RegCount = std::get<0>(CountAndVT);
1019     MVT RegisterVT = std::get<1>(CountAndVT);
1020     TypeSize RegisterSize = RegisterVT.getSizeInBits();
1021     for (unsigned E = I + RegCount; I != E; ++I)
1022       OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1023   }
1024   return OutVec;
1025 }
1026 
1027 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
1028                                const TargetLibraryInfo *li) {
1029   AA = aa;
1030   GFI = gfi;
1031   LibInfo = li;
1032   Context = DAG.getContext();
1033   LPadToCallSiteMap.clear();
1034   SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1035 }
1036 
1037 void SelectionDAGBuilder::clear() {
1038   NodeMap.clear();
1039   UnusedArgNodeMap.clear();
1040   PendingLoads.clear();
1041   PendingExports.clear();
1042   PendingConstrainedFP.clear();
1043   PendingConstrainedFPStrict.clear();
1044   CurInst = nullptr;
1045   HasTailCall = false;
1046   SDNodeOrder = LowestSDNodeOrder;
1047   StatepointLowering.clear();
1048 }
1049 
1050 void SelectionDAGBuilder::clearDanglingDebugInfo() {
1051   DanglingDebugInfoMap.clear();
1052 }
1053 
1054 // Update DAG root to include dependencies on Pending chains.
1055 SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1056   SDValue Root = DAG.getRoot();
1057 
1058   if (Pending.empty())
1059     return Root;
1060 
1061   // Add current root to PendingChains, unless we already indirectly
1062   // depend on it.
1063   if (Root.getOpcode() != ISD::EntryToken) {
1064     unsigned i = 0, e = Pending.size();
1065     for (; i != e; ++i) {
1066       assert(Pending[i].getNode()->getNumOperands() > 1);
1067       if (Pending[i].getNode()->getOperand(0) == Root)
1068         break;  // Don't add the root if we already indirectly depend on it.
1069     }
1070 
1071     if (i == e)
1072       Pending.push_back(Root);
1073   }
1074 
1075   if (Pending.size() == 1)
1076     Root = Pending[0];
1077   else
1078     Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1079 
1080   DAG.setRoot(Root);
1081   Pending.clear();
1082   return Root;
1083 }
1084 
1085 SDValue SelectionDAGBuilder::getMemoryRoot() {
1086   return updateRoot(PendingLoads);
1087 }
1088 
1089 SDValue SelectionDAGBuilder::getRoot() {
1090   // Chain up all pending constrained intrinsics together with all
1091   // pending loads, by simply appending them to PendingLoads and
1092   // then calling getMemoryRoot().
1093   PendingLoads.reserve(PendingLoads.size() +
1094                        PendingConstrainedFP.size() +
1095                        PendingConstrainedFPStrict.size());
1096   PendingLoads.append(PendingConstrainedFP.begin(),
1097                       PendingConstrainedFP.end());
1098   PendingLoads.append(PendingConstrainedFPStrict.begin(),
1099                       PendingConstrainedFPStrict.end());
1100   PendingConstrainedFP.clear();
1101   PendingConstrainedFPStrict.clear();
1102   return getMemoryRoot();
1103 }
1104 
1105 SDValue SelectionDAGBuilder::getControlRoot() {
1106   // We need to emit pending fpexcept.strict constrained intrinsics,
1107   // so append them to the PendingExports list.
1108   PendingExports.append(PendingConstrainedFPStrict.begin(),
1109                         PendingConstrainedFPStrict.end());
1110   PendingConstrainedFPStrict.clear();
1111   return updateRoot(PendingExports);
1112 }
1113 
1114 void SelectionDAGBuilder::visit(const Instruction &I) {
1115   // Set up outgoing PHI node register values before emitting the terminator.
1116   if (I.isTerminator()) {
1117     HandlePHINodesInSuccessorBlocks(I.getParent());
1118   }
1119 
1120   // Increase the SDNodeOrder if dealing with a non-debug instruction.
1121   if (!isa<DbgInfoIntrinsic>(I))
1122     ++SDNodeOrder;
1123 
1124   CurInst = &I;
1125 
1126   visit(I.getOpcode(), I);
1127 
1128   if (!I.isTerminator() && !HasTailCall &&
1129       !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1130     CopyToExportRegsIfNeeded(&I);
1131 
1132   CurInst = nullptr;
1133 }
1134 
1135 void SelectionDAGBuilder::visitPHI(const PHINode &) {
1136   llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1137 }
1138 
1139 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1140   // Note: this doesn't use InstVisitor, because it has to work with
1141   // ConstantExpr's in addition to instructions.
1142   switch (Opcode) {
1143   default: llvm_unreachable("Unknown instruction type encountered!");
1144     // Build the switch statement using the Instruction.def file.
1145 #define HANDLE_INST(NUM, OPCODE, CLASS) \
1146     case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1147 #include "llvm/IR/Instruction.def"
1148   }
1149 }
1150 
1151 void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,
1152                                                DebugLoc DL, unsigned Order) {
1153   // We treat variadic dbg_values differently at this stage.
1154   if (DI->hasArgList()) {
1155     // For variadic dbg_values we will now insert an undef.
1156     // FIXME: We can potentially recover these!
1157     SmallVector<SDDbgOperand, 2> Locs;
1158     for (const Value *V : DI->getValues()) {
1159       auto Undef = UndefValue::get(V->getType());
1160       Locs.push_back(SDDbgOperand::fromConst(Undef));
1161     }
1162     SDDbgValue *SDV = DAG.getDbgValueList(
1163         DI->getVariable(), DI->getExpression(), Locs, {},
1164         /*IsIndirect=*/false, DL, Order, /*IsVariadic=*/true);
1165     DAG.AddDbgValue(SDV, /*isParameter=*/false);
1166   } else {
1167     // TODO: Dangling debug info will eventually either be resolved or produce
1168     // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1169     // between the original dbg.value location and its resolved DBG_VALUE,
1170     // which we should ideally fill with an extra Undef DBG_VALUE.
1171     assert(DI->getNumVariableLocationOps() == 1 &&
1172            "DbgValueInst without an ArgList should have a single location "
1173            "operand.");
1174     DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, DL, Order);
1175   }
1176 }
1177 
1178 void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1179                                                 const DIExpression *Expr) {
1180   auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1181     const DbgValueInst *DI = DDI.getDI();
1182     DIVariable *DanglingVariable = DI->getVariable();
1183     DIExpression *DanglingExpr = DI->getExpression();
1184     if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1185       LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1186       return true;
1187     }
1188     return false;
1189   };
1190 
1191   for (auto &DDIMI : DanglingDebugInfoMap) {
1192     DanglingDebugInfoVector &DDIV = DDIMI.second;
1193 
1194     // If debug info is to be dropped, run it through final checks to see
1195     // whether it can be salvaged.
1196     for (auto &DDI : DDIV)
1197       if (isMatchingDbgValue(DDI))
1198         salvageUnresolvedDbgValue(DDI);
1199 
1200     erase_if(DDIV, isMatchingDbgValue);
1201   }
1202 }
1203 
1204 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1205 // generate the debug data structures now that we've seen its definition.
1206 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1207                                                    SDValue Val) {
1208   auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1209   if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1210     return;
1211 
1212   DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1213   for (auto &DDI : DDIV) {
1214     const DbgValueInst *DI = DDI.getDI();
1215     assert(!DI->hasArgList() && "Not implemented for variadic dbg_values");
1216     assert(DI && "Ill-formed DanglingDebugInfo");
1217     DebugLoc dl = DDI.getdl();
1218     unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1219     unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1220     DILocalVariable *Variable = DI->getVariable();
1221     DIExpression *Expr = DI->getExpression();
1222     assert(Variable->isValidLocationForIntrinsic(dl) &&
1223            "Expected inlined-at fields to agree");
1224     SDDbgValue *SDV;
1225     if (Val.getNode()) {
1226       // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1227       // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1228       // we couldn't resolve it directly when examining the DbgValue intrinsic
1229       // in the first place we should not be more successful here). Unless we
1230       // have some test case that prove this to be correct we should avoid
1231       // calling EmitFuncArgumentDbgValue here.
1232       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl,
1233                                     FuncArgumentDbgValueKind::Value, Val)) {
1234         LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1235                           << DbgSDNodeOrder << "] for:\n  " << *DI << "\n");
1236         LLVM_DEBUG(dbgs() << "  By mapping to:\n    "; Val.dump());
1237         // Increase the SDNodeOrder for the DbgValue here to make sure it is
1238         // inserted after the definition of Val when emitting the instructions
1239         // after ISel. An alternative could be to teach
1240         // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1241         LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1242                    << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1243                    << ValSDNodeOrder << "\n");
1244         SDV = getDbgValue(Val, Variable, Expr, dl,
1245                           std::max(DbgSDNodeOrder, ValSDNodeOrder));
1246         DAG.AddDbgValue(SDV, false);
1247       } else
1248         LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1249                           << "in EmitFuncArgumentDbgValue\n");
1250     } else {
1251       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1252       auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1253       auto SDV =
1254           DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1255       DAG.AddDbgValue(SDV, false);
1256     }
1257   }
1258   DDIV.clear();
1259 }
1260 
1261 void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
1262   // TODO: For the variadic implementation, instead of only checking the fail
1263   // state of `handleDebugValue`, we need know specifically which values were
1264   // invalid, so that we attempt to salvage only those values when processing
1265   // a DIArgList.
1266   assert(!DDI.getDI()->hasArgList() &&
1267          "Not implemented for variadic dbg_values");
1268   Value *V = DDI.getDI()->getValue(0);
1269   DILocalVariable *Var = DDI.getDI()->getVariable();
1270   DIExpression *Expr = DDI.getDI()->getExpression();
1271   DebugLoc DL = DDI.getdl();
1272   DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1273   unsigned SDOrder = DDI.getSDNodeOrder();
1274   // Currently we consider only dbg.value intrinsics -- we tell the salvager
1275   // that DW_OP_stack_value is desired.
1276   assert(isa<DbgValueInst>(DDI.getDI()));
1277   bool StackValue = true;
1278 
1279   // Can this Value can be encoded without any further work?
1280   if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder, /*IsVariadic=*/false))
1281     return;
1282 
1283   // Attempt to salvage back through as many instructions as possible. Bail if
1284   // a non-instruction is seen, such as a constant expression or global
1285   // variable. FIXME: Further work could recover those too.
1286   while (isa<Instruction>(V)) {
1287     Instruction &VAsInst = *cast<Instruction>(V);
1288     // Temporary "0", awaiting real implementation.
1289     SmallVector<uint64_t, 16> Ops;
1290     SmallVector<Value *, 4> AdditionalValues;
1291     V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1292                              AdditionalValues);
1293     // If we cannot salvage any further, and haven't yet found a suitable debug
1294     // expression, bail out.
1295     if (!V)
1296       break;
1297 
1298     // TODO: If AdditionalValues isn't empty, then the salvage can only be
1299     // represented with a DBG_VALUE_LIST, so we give up. When we have support
1300     // here for variadic dbg_values, remove that condition.
1301     if (!AdditionalValues.empty())
1302       break;
1303 
1304     // New value and expr now represent this debuginfo.
1305     Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1306 
1307     // Some kind of simplification occurred: check whether the operand of the
1308     // salvaged debug expression can be encoded in this DAG.
1309     if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder,
1310                          /*IsVariadic=*/false)) {
1311       LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n  "
1312                         << *DDI.getDI() << "\nBy stripping back to:\n  " << *V);
1313       return;
1314     }
1315   }
1316 
1317   // This was the final opportunity to salvage this debug information, and it
1318   // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1319   // any earlier variable location.
1320   auto Undef = UndefValue::get(DDI.getDI()->getValue(0)->getType());
1321   auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1322   DAG.AddDbgValue(SDV, false);
1323 
1324   LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n  " << *DDI.getDI()
1325                     << "\n");
1326   LLVM_DEBUG(dbgs() << "  Last seen at:\n    " << *DDI.getDI()->getOperand(0)
1327                     << "\n");
1328 }
1329 
1330 bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
1331                                            DILocalVariable *Var,
1332                                            DIExpression *Expr, DebugLoc dl,
1333                                            DebugLoc InstDL, unsigned Order,
1334                                            bool IsVariadic) {
1335   if (Values.empty())
1336     return true;
1337   SmallVector<SDDbgOperand> LocationOps;
1338   SmallVector<SDNode *> Dependencies;
1339   for (const Value *V : Values) {
1340     // Constant value.
1341     if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1342         isa<ConstantPointerNull>(V)) {
1343       LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1344       continue;
1345     }
1346 
1347     // If the Value is a frame index, we can create a FrameIndex debug value
1348     // without relying on the DAG at all.
1349     if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1350       auto SI = FuncInfo.StaticAllocaMap.find(AI);
1351       if (SI != FuncInfo.StaticAllocaMap.end()) {
1352         LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1353         continue;
1354       }
1355     }
1356 
1357     // Do not use getValue() in here; we don't want to generate code at
1358     // this point if it hasn't been done yet.
1359     SDValue N = NodeMap[V];
1360     if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1361       N = UnusedArgNodeMap[V];
1362     if (N.getNode()) {
1363       // Only emit func arg dbg value for non-variadic dbg.values for now.
1364       if (!IsVariadic &&
1365           EmitFuncArgumentDbgValue(V, Var, Expr, dl,
1366                                    FuncArgumentDbgValueKind::Value, N))
1367         return true;
1368       if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1369         // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1370         // describe stack slot locations.
1371         //
1372         // Consider "int x = 0; int *px = &x;". There are two kinds of
1373         // interesting debug values here after optimization:
1374         //
1375         //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
1376         //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1377         //
1378         // Both describe the direct values of their associated variables.
1379         Dependencies.push_back(N.getNode());
1380         LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1381         continue;
1382       }
1383       LocationOps.emplace_back(
1384           SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1385       continue;
1386     }
1387 
1388     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1389     // Special rules apply for the first dbg.values of parameter variables in a
1390     // function. Identify them by the fact they reference Argument Values, that
1391     // they're parameters, and they are parameters of the current function. We
1392     // need to let them dangle until they get an SDNode.
1393     bool IsParamOfFunc =
1394         isa<Argument>(V) && Var->isParameter() && !InstDL.getInlinedAt();
1395     if (IsParamOfFunc)
1396       return false;
1397 
1398     // The value is not used in this block yet (or it would have an SDNode).
1399     // We still want the value to appear for the user if possible -- if it has
1400     // an associated VReg, we can refer to that instead.
1401     auto VMI = FuncInfo.ValueMap.find(V);
1402     if (VMI != FuncInfo.ValueMap.end()) {
1403       unsigned Reg = VMI->second;
1404       // If this is a PHI node, it may be split up into several MI PHI nodes
1405       // (in FunctionLoweringInfo::set).
1406       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1407                        V->getType(), None);
1408       if (RFV.occupiesMultipleRegs()) {
1409         // FIXME: We could potentially support variadic dbg_values here.
1410         if (IsVariadic)
1411           return false;
1412         unsigned Offset = 0;
1413         unsigned BitsToDescribe = 0;
1414         if (auto VarSize = Var->getSizeInBits())
1415           BitsToDescribe = *VarSize;
1416         if (auto Fragment = Expr->getFragmentInfo())
1417           BitsToDescribe = Fragment->SizeInBits;
1418         for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1419           // Bail out if all bits are described already.
1420           if (Offset >= BitsToDescribe)
1421             break;
1422           // TODO: handle scalable vectors.
1423           unsigned RegisterSize = RegAndSize.second;
1424           unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1425                                       ? BitsToDescribe - Offset
1426                                       : RegisterSize;
1427           auto FragmentExpr = DIExpression::createFragmentExpression(
1428               Expr, Offset, FragmentSize);
1429           if (!FragmentExpr)
1430             continue;
1431           SDDbgValue *SDV = DAG.getVRegDbgValue(
1432               Var, *FragmentExpr, RegAndSize.first, false, dl, SDNodeOrder);
1433           DAG.AddDbgValue(SDV, false);
1434           Offset += RegisterSize;
1435         }
1436         return true;
1437       }
1438       // We can use simple vreg locations for variadic dbg_values as well.
1439       LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1440       continue;
1441     }
1442     // We failed to create a SDDbgOperand for V.
1443     return false;
1444   }
1445 
1446   // We have created a SDDbgOperand for each Value in Values.
1447   // Should use Order instead of SDNodeOrder?
1448   assert(!LocationOps.empty());
1449   SDDbgValue *SDV =
1450       DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1451                           /*IsIndirect=*/false, dl, SDNodeOrder, IsVariadic);
1452   DAG.AddDbgValue(SDV, /*isParameter=*/false);
1453   return true;
1454 }
1455 
1456 void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1457   // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1458   for (auto &Pair : DanglingDebugInfoMap)
1459     for (auto &DDI : Pair.second)
1460       salvageUnresolvedDbgValue(DDI);
1461   clearDanglingDebugInfo();
1462 }
1463 
1464 /// getCopyFromRegs - If there was virtual register allocated for the value V
1465 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1466 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1467   DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);
1468   SDValue Result;
1469 
1470   if (It != FuncInfo.ValueMap.end()) {
1471     Register InReg = It->second;
1472 
1473     RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1474                      DAG.getDataLayout(), InReg, Ty,
1475                      None); // This is not an ABI copy.
1476     SDValue Chain = DAG.getEntryNode();
1477     Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1478                                  V);
1479     resolveDanglingDebugInfo(V, Result);
1480   }
1481 
1482   return Result;
1483 }
1484 
1485 /// getValue - Return an SDValue for the given Value.
1486 SDValue SelectionDAGBuilder::getValue(const Value *V) {
1487   // If we already have an SDValue for this value, use it. It's important
1488   // to do this first, so that we don't create a CopyFromReg if we already
1489   // have a regular SDValue.
1490   SDValue &N = NodeMap[V];
1491   if (N.getNode()) return N;
1492 
1493   // If there's a virtual register allocated and initialized for this
1494   // value, use it.
1495   if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1496     return copyFromReg;
1497 
1498   // Otherwise create a new SDValue and remember it.
1499   SDValue Val = getValueImpl(V);
1500   NodeMap[V] = Val;
1501   resolveDanglingDebugInfo(V, Val);
1502   return Val;
1503 }
1504 
1505 /// getNonRegisterValue - Return an SDValue for the given Value, but
1506 /// don't look in FuncInfo.ValueMap for a virtual register.
1507 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1508   // If we already have an SDValue for this value, use it.
1509   SDValue &N = NodeMap[V];
1510   if (N.getNode()) {
1511     if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1512       // Remove the debug location from the node as the node is about to be used
1513       // in a location which may differ from the original debug location.  This
1514       // is relevant to Constant and ConstantFP nodes because they can appear
1515       // as constant expressions inside PHI nodes.
1516       N->setDebugLoc(DebugLoc());
1517     }
1518     return N;
1519   }
1520 
1521   // Otherwise create a new SDValue and remember it.
1522   SDValue Val = getValueImpl(V);
1523   NodeMap[V] = Val;
1524   resolveDanglingDebugInfo(V, Val);
1525   return Val;
1526 }
1527 
1528 /// getValueImpl - Helper function for getValue and getNonRegisterValue.
1529 /// Create an SDValue for the given value.
1530 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1531   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1532 
1533   if (const Constant *C = dyn_cast<Constant>(V)) {
1534     EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1535 
1536     if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1537       return DAG.getConstant(*CI, getCurSDLoc(), VT);
1538 
1539     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1540       return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1541 
1542     if (isa<ConstantPointerNull>(C)) {
1543       unsigned AS = V->getType()->getPointerAddressSpace();
1544       return DAG.getConstant(0, getCurSDLoc(),
1545                              TLI.getPointerTy(DAG.getDataLayout(), AS));
1546     }
1547 
1548     if (match(C, m_VScale(DAG.getDataLayout())))
1549       return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1550 
1551     if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1552       return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1553 
1554     if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1555       return DAG.getUNDEF(VT);
1556 
1557     if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1558       visit(CE->getOpcode(), *CE);
1559       SDValue N1 = NodeMap[V];
1560       assert(N1.getNode() && "visit didn't populate the NodeMap!");
1561       return N1;
1562     }
1563 
1564     if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1565       SmallVector<SDValue, 4> Constants;
1566       for (const Use &U : C->operands()) {
1567         SDNode *Val = getValue(U).getNode();
1568         // If the operand is an empty aggregate, there are no values.
1569         if (!Val) continue;
1570         // Add each leaf value from the operand to the Constants list
1571         // to form a flattened list of all the values.
1572         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1573           Constants.push_back(SDValue(Val, i));
1574       }
1575 
1576       return DAG.getMergeValues(Constants, getCurSDLoc());
1577     }
1578 
1579     if (const ConstantDataSequential *CDS =
1580           dyn_cast<ConstantDataSequential>(C)) {
1581       SmallVector<SDValue, 4> Ops;
1582       for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1583         SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1584         // Add each leaf value from the operand to the Constants list
1585         // to form a flattened list of all the values.
1586         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1587           Ops.push_back(SDValue(Val, i));
1588       }
1589 
1590       if (isa<ArrayType>(CDS->getType()))
1591         return DAG.getMergeValues(Ops, getCurSDLoc());
1592       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1593     }
1594 
1595     if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1596       assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1597              "Unknown struct or array constant!");
1598 
1599       SmallVector<EVT, 4> ValueVTs;
1600       ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1601       unsigned NumElts = ValueVTs.size();
1602       if (NumElts == 0)
1603         return SDValue(); // empty struct
1604       SmallVector<SDValue, 4> Constants(NumElts);
1605       for (unsigned i = 0; i != NumElts; ++i) {
1606         EVT EltVT = ValueVTs[i];
1607         if (isa<UndefValue>(C))
1608           Constants[i] = DAG.getUNDEF(EltVT);
1609         else if (EltVT.isFloatingPoint())
1610           Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1611         else
1612           Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1613       }
1614 
1615       return DAG.getMergeValues(Constants, getCurSDLoc());
1616     }
1617 
1618     if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1619       return DAG.getBlockAddress(BA, VT);
1620 
1621     if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1622       return getValue(Equiv->getGlobalValue());
1623 
1624     if (const auto *NC = dyn_cast<NoCFIValue>(C))
1625       return getValue(NC->getGlobalValue());
1626 
1627     VectorType *VecTy = cast<VectorType>(V->getType());
1628 
1629     // Now that we know the number and type of the elements, get that number of
1630     // elements into the Ops array based on what kind of constant it is.
1631     if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1632       SmallVector<SDValue, 16> Ops;
1633       unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1634       for (unsigned i = 0; i != NumElements; ++i)
1635         Ops.push_back(getValue(CV->getOperand(i)));
1636 
1637       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1638     }
1639 
1640     if (isa<ConstantAggregateZero>(C)) {
1641       EVT EltVT =
1642           TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1643 
1644       SDValue Op;
1645       if (EltVT.isFloatingPoint())
1646         Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1647       else
1648         Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1649 
1650       if (isa<ScalableVectorType>(VecTy))
1651         return NodeMap[V] = DAG.getSplatVector(VT, getCurSDLoc(), Op);
1652 
1653       SmallVector<SDValue, 16> Ops;
1654       Ops.assign(cast<FixedVectorType>(VecTy)->getNumElements(), Op);
1655       return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1656     }
1657 
1658     llvm_unreachable("Unknown vector constant");
1659   }
1660 
1661   // If this is a static alloca, generate it as the frameindex instead of
1662   // computation.
1663   if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1664     DenseMap<const AllocaInst*, int>::iterator SI =
1665       FuncInfo.StaticAllocaMap.find(AI);
1666     if (SI != FuncInfo.StaticAllocaMap.end())
1667       return DAG.getFrameIndex(SI->second,
1668                                TLI.getFrameIndexTy(DAG.getDataLayout()));
1669   }
1670 
1671   // If this is an instruction which fast-isel has deferred, select it now.
1672   if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1673     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1674 
1675     RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1676                      Inst->getType(), None);
1677     SDValue Chain = DAG.getEntryNode();
1678     return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1679   }
1680 
1681   if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1682     return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1683 
1684   if (const auto *BB = dyn_cast<BasicBlock>(V))
1685     return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1686 
1687   llvm_unreachable("Can't get register for value!");
1688 }
1689 
1690 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1691   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1692   bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1693   bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1694   bool IsSEH = isAsynchronousEHPersonality(Pers);
1695   MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1696   if (!IsSEH)
1697     CatchPadMBB->setIsEHScopeEntry();
1698   // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1699   if (IsMSVCCXX || IsCoreCLR)
1700     CatchPadMBB->setIsEHFuncletEntry();
1701 }
1702 
1703 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1704   // Update machine-CFG edge.
1705   MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1706   FuncInfo.MBB->addSuccessor(TargetMBB);
1707   TargetMBB->setIsEHCatchretTarget(true);
1708   DAG.getMachineFunction().setHasEHCatchret(true);
1709 
1710   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1711   bool IsSEH = isAsynchronousEHPersonality(Pers);
1712   if (IsSEH) {
1713     // If this is not a fall-through branch or optimizations are switched off,
1714     // emit the branch.
1715     if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1716         TM.getOptLevel() == CodeGenOpt::None)
1717       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1718                               getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1719     return;
1720   }
1721 
1722   // Figure out the funclet membership for the catchret's successor.
1723   // This will be used by the FuncletLayout pass to determine how to order the
1724   // BB's.
1725   // A 'catchret' returns to the outer scope's color.
1726   Value *ParentPad = I.getCatchSwitchParentPad();
1727   const BasicBlock *SuccessorColor;
1728   if (isa<ConstantTokenNone>(ParentPad))
1729     SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1730   else
1731     SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1732   assert(SuccessorColor && "No parent funclet for catchret!");
1733   MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1734   assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1735 
1736   // Create the terminator node.
1737   SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1738                             getControlRoot(), DAG.getBasicBlock(TargetMBB),
1739                             DAG.getBasicBlock(SuccessorColorMBB));
1740   DAG.setRoot(Ret);
1741 }
1742 
1743 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1744   // Don't emit any special code for the cleanuppad instruction. It just marks
1745   // the start of an EH scope/funclet.
1746   FuncInfo.MBB->setIsEHScopeEntry();
1747   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1748   if (Pers != EHPersonality::Wasm_CXX) {
1749     FuncInfo.MBB->setIsEHFuncletEntry();
1750     FuncInfo.MBB->setIsCleanupFuncletEntry();
1751   }
1752 }
1753 
1754 // In wasm EH, even though a catchpad may not catch an exception if a tag does
1755 // not match, it is OK to add only the first unwind destination catchpad to the
1756 // successors, because there will be at least one invoke instruction within the
1757 // catch scope that points to the next unwind destination, if one exists, so
1758 // CFGSort cannot mess up with BB sorting order.
1759 // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1760 // call within them, and catchpads only consisting of 'catch (...)' have a
1761 // '__cxa_end_catch' call within them, both of which generate invokes in case
1762 // the next unwind destination exists, i.e., the next unwind destination is not
1763 // the caller.)
1764 //
1765 // Having at most one EH pad successor is also simpler and helps later
1766 // transformations.
1767 //
1768 // For example,
1769 // current:
1770 //   invoke void @foo to ... unwind label %catch.dispatch
1771 // catch.dispatch:
1772 //   %0 = catchswitch within ... [label %catch.start] unwind label %next
1773 // catch.start:
1774 //   ...
1775 //   ... in this BB or some other child BB dominated by this BB there will be an
1776 //   invoke that points to 'next' BB as an unwind destination
1777 //
1778 // next: ; We don't need to add this to 'current' BB's successor
1779 //   ...
1780 static void findWasmUnwindDestinations(
1781     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1782     BranchProbability Prob,
1783     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1784         &UnwindDests) {
1785   while (EHPadBB) {
1786     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1787     if (isa<CleanupPadInst>(Pad)) {
1788       // Stop on cleanup pads.
1789       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1790       UnwindDests.back().first->setIsEHScopeEntry();
1791       break;
1792     } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1793       // Add the catchpad handlers to the possible destinations. We don't
1794       // continue to the unwind destination of the catchswitch for wasm.
1795       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1796         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1797         UnwindDests.back().first->setIsEHScopeEntry();
1798       }
1799       break;
1800     } else {
1801       continue;
1802     }
1803   }
1804 }
1805 
1806 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
1807 /// many places it could ultimately go. In the IR, we have a single unwind
1808 /// destination, but in the machine CFG, we enumerate all the possible blocks.
1809 /// This function skips over imaginary basic blocks that hold catchswitch
1810 /// instructions, and finds all the "real" machine
1811 /// basic block destinations. As those destinations may not be successors of
1812 /// EHPadBB, here we also calculate the edge probability to those destinations.
1813 /// The passed-in Prob is the edge probability to EHPadBB.
1814 static void findUnwindDestinations(
1815     FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1816     BranchProbability Prob,
1817     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1818         &UnwindDests) {
1819   EHPersonality Personality =
1820     classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1821   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1822   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1823   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1824   bool IsSEH = isAsynchronousEHPersonality(Personality);
1825 
1826   if (IsWasmCXX) {
1827     findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1828     assert(UnwindDests.size() <= 1 &&
1829            "There should be at most one unwind destination for wasm");
1830     return;
1831   }
1832 
1833   while (EHPadBB) {
1834     const Instruction *Pad = EHPadBB->getFirstNonPHI();
1835     BasicBlock *NewEHPadBB = nullptr;
1836     if (isa<LandingPadInst>(Pad)) {
1837       // Stop on landingpads. They are not funclets.
1838       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1839       break;
1840     } else if (isa<CleanupPadInst>(Pad)) {
1841       // Stop on cleanup pads. Cleanups are always funclet entries for all known
1842       // personalities.
1843       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1844       UnwindDests.back().first->setIsEHScopeEntry();
1845       UnwindDests.back().first->setIsEHFuncletEntry();
1846       break;
1847     } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1848       // Add the catchpad handlers to the possible destinations.
1849       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1850         UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1851         // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1852         if (IsMSVCCXX || IsCoreCLR)
1853           UnwindDests.back().first->setIsEHFuncletEntry();
1854         if (!IsSEH)
1855           UnwindDests.back().first->setIsEHScopeEntry();
1856       }
1857       NewEHPadBB = CatchSwitch->getUnwindDest();
1858     } else {
1859       continue;
1860     }
1861 
1862     BranchProbabilityInfo *BPI = FuncInfo.BPI;
1863     if (BPI && NewEHPadBB)
1864       Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1865     EHPadBB = NewEHPadBB;
1866   }
1867 }
1868 
1869 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1870   // Update successor info.
1871   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1872   auto UnwindDest = I.getUnwindDest();
1873   BranchProbabilityInfo *BPI = FuncInfo.BPI;
1874   BranchProbability UnwindDestProb =
1875       (BPI && UnwindDest)
1876           ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1877           : BranchProbability::getZero();
1878   findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1879   for (auto &UnwindDest : UnwindDests) {
1880     UnwindDest.first->setIsEHPad();
1881     addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1882   }
1883   FuncInfo.MBB->normalizeSuccProbs();
1884 
1885   // Create the terminator node.
1886   SDValue Ret =
1887       DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1888   DAG.setRoot(Ret);
1889 }
1890 
1891 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1892   report_fatal_error("visitCatchSwitch not yet implemented!");
1893 }
1894 
1895 void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1896   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1897   auto &DL = DAG.getDataLayout();
1898   SDValue Chain = getControlRoot();
1899   SmallVector<ISD::OutputArg, 8> Outs;
1900   SmallVector<SDValue, 8> OutVals;
1901 
1902   // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1903   // lower
1904   //
1905   //   %val = call <ty> @llvm.experimental.deoptimize()
1906   //   ret <ty> %val
1907   //
1908   // differently.
1909   if (I.getParent()->getTerminatingDeoptimizeCall()) {
1910     LowerDeoptimizingReturn();
1911     return;
1912   }
1913 
1914   if (!FuncInfo.CanLowerReturn) {
1915     unsigned DemoteReg = FuncInfo.DemoteRegister;
1916     const Function *F = I.getParent()->getParent();
1917 
1918     // Emit a store of the return value through the virtual register.
1919     // Leave Outs empty so that LowerReturn won't try to load return
1920     // registers the usual way.
1921     SmallVector<EVT, 1> PtrValueVTs;
1922     ComputeValueVTs(TLI, DL,
1923                     F->getReturnType()->getPointerTo(
1924                         DAG.getDataLayout().getAllocaAddrSpace()),
1925                     PtrValueVTs);
1926 
1927     SDValue RetPtr =
1928         DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
1929     SDValue RetOp = getValue(I.getOperand(0));
1930 
1931     SmallVector<EVT, 4> ValueVTs, MemVTs;
1932     SmallVector<uint64_t, 4> Offsets;
1933     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1934                     &Offsets);
1935     unsigned NumValues = ValueVTs.size();
1936 
1937     SmallVector<SDValue, 4> Chains(NumValues);
1938     Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
1939     for (unsigned i = 0; i != NumValues; ++i) {
1940       // An aggregate return value cannot wrap around the address space, so
1941       // offsets to its parts don't wrap either.
1942       SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
1943                                            TypeSize::Fixed(Offsets[i]));
1944 
1945       SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
1946       if (MemVTs[i] != ValueVTs[i])
1947         Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1948       Chains[i] = DAG.getStore(
1949           Chain, getCurSDLoc(), Val,
1950           // FIXME: better loc info would be nice.
1951           Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
1952           commonAlignment(BaseAlign, Offsets[i]));
1953     }
1954 
1955     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1956                         MVT::Other, Chains);
1957   } else if (I.getNumOperands() != 0) {
1958     SmallVector<EVT, 4> ValueVTs;
1959     ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1960     unsigned NumValues = ValueVTs.size();
1961     if (NumValues) {
1962       SDValue RetOp = getValue(I.getOperand(0));
1963 
1964       const Function *F = I.getParent()->getParent();
1965 
1966       bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1967           I.getOperand(0)->getType(), F->getCallingConv(),
1968           /*IsVarArg*/ false, DL);
1969 
1970       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1971       if (F->getAttributes().hasRetAttr(Attribute::SExt))
1972         ExtendKind = ISD::SIGN_EXTEND;
1973       else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
1974         ExtendKind = ISD::ZERO_EXTEND;
1975 
1976       LLVMContext &Context = F->getContext();
1977       bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
1978 
1979       for (unsigned j = 0; j != NumValues; ++j) {
1980         EVT VT = ValueVTs[j];
1981 
1982         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1983           VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1984 
1985         CallingConv::ID CC = F->getCallingConv();
1986 
1987         unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1988         MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1989         SmallVector<SDValue, 4> Parts(NumParts);
1990         getCopyToParts(DAG, getCurSDLoc(),
1991                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1992                        &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1993 
1994         // 'inreg' on function refers to return value
1995         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1996         if (RetInReg)
1997           Flags.setInReg();
1998 
1999         if (I.getOperand(0)->getType()->isPointerTy()) {
2000           Flags.setPointer();
2001           Flags.setPointerAddrSpace(
2002               cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2003         }
2004 
2005         if (NeedsRegBlock) {
2006           Flags.setInConsecutiveRegs();
2007           if (j == NumValues - 1)
2008             Flags.setInConsecutiveRegsLast();
2009         }
2010 
2011         // Propagate extension type if any
2012         if (ExtendKind == ISD::SIGN_EXTEND)
2013           Flags.setSExt();
2014         else if (ExtendKind == ISD::ZERO_EXTEND)
2015           Flags.setZExt();
2016 
2017         for (unsigned i = 0; i < NumParts; ++i) {
2018           Outs.push_back(ISD::OutputArg(Flags,
2019                                         Parts[i].getValueType().getSimpleVT(),
2020                                         VT, /*isfixed=*/true, 0, 0));
2021           OutVals.push_back(Parts[i]);
2022         }
2023       }
2024     }
2025   }
2026 
2027   // Push in swifterror virtual register as the last element of Outs. This makes
2028   // sure swifterror virtual register will be returned in the swifterror
2029   // physical register.
2030   const Function *F = I.getParent()->getParent();
2031   if (TLI.supportSwiftError() &&
2032       F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2033     assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2034     ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2035     Flags.setSwiftError();
2036     Outs.push_back(ISD::OutputArg(
2037         Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2038         /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2039     // Create SDNode for the swifterror virtual register.
2040     OutVals.push_back(
2041         DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2042                             &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2043                         EVT(TLI.getPointerTy(DL))));
2044   }
2045 
2046   bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2047   CallingConv::ID CallConv =
2048     DAG.getMachineFunction().getFunction().getCallingConv();
2049   Chain = DAG.getTargetLoweringInfo().LowerReturn(
2050       Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2051 
2052   // Verify that the target's LowerReturn behaved as expected.
2053   assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2054          "LowerReturn didn't return a valid chain!");
2055 
2056   // Update the DAG with the new chain value resulting from return lowering.
2057   DAG.setRoot(Chain);
2058 }
2059 
2060 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
2061 /// created for it, emit nodes to copy the value into the virtual
2062 /// registers.
2063 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
2064   // Skip empty types
2065   if (V->getType()->isEmptyTy())
2066     return;
2067 
2068   DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);
2069   if (VMI != FuncInfo.ValueMap.end()) {
2070     assert(!V->use_empty() && "Unused value assigned virtual registers!");
2071     CopyValueToVirtualRegister(V, VMI->second);
2072   }
2073 }
2074 
2075 /// ExportFromCurrentBlock - If this condition isn't known to be exported from
2076 /// the current basic block, add it to ValueMap now so that we'll get a
2077 /// CopyTo/FromReg.
2078 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
2079   // No need to export constants.
2080   if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2081 
2082   // Already exported?
2083   if (FuncInfo.isExportedInst(V)) return;
2084 
2085   unsigned Reg = FuncInfo.InitializeRegForValue(V);
2086   CopyValueToVirtualRegister(V, Reg);
2087 }
2088 
2089 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
2090                                                      const BasicBlock *FromBB) {
2091   // The operands of the setcc have to be in this block.  We don't know
2092   // how to export them from some other block.
2093   if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2094     // Can export from current BB.
2095     if (VI->getParent() == FromBB)
2096       return true;
2097 
2098     // Is already exported, noop.
2099     return FuncInfo.isExportedInst(V);
2100   }
2101 
2102   // If this is an argument, we can export it if the BB is the entry block or
2103   // if it is already exported.
2104   if (isa<Argument>(V)) {
2105     if (FromBB->isEntryBlock())
2106       return true;
2107 
2108     // Otherwise, can only export this if it is already exported.
2109     return FuncInfo.isExportedInst(V);
2110   }
2111 
2112   // Otherwise, constants can always be exported.
2113   return true;
2114 }
2115 
2116 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2117 BranchProbability
2118 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2119                                         const MachineBasicBlock *Dst) const {
2120   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2121   const BasicBlock *SrcBB = Src->getBasicBlock();
2122   const BasicBlock *DstBB = Dst->getBasicBlock();
2123   if (!BPI) {
2124     // If BPI is not available, set the default probability as 1 / N, where N is
2125     // the number of successors.
2126     auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2127     return BranchProbability(1, SuccSize);
2128   }
2129   return BPI->getEdgeProbability(SrcBB, DstBB);
2130 }
2131 
2132 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2133                                                MachineBasicBlock *Dst,
2134                                                BranchProbability Prob) {
2135   if (!FuncInfo.BPI)
2136     Src->addSuccessorWithoutProb(Dst);
2137   else {
2138     if (Prob.isUnknown())
2139       Prob = getEdgeProbability(Src, Dst);
2140     Src->addSuccessor(Dst, Prob);
2141   }
2142 }
2143 
2144 static bool InBlock(const Value *V, const BasicBlock *BB) {
2145   if (const Instruction *I = dyn_cast<Instruction>(V))
2146     return I->getParent() == BB;
2147   return true;
2148 }
2149 
2150 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2151 /// This function emits a branch and is used at the leaves of an OR or an
2152 /// AND operator tree.
2153 void
2154 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2155                                                   MachineBasicBlock *TBB,
2156                                                   MachineBasicBlock *FBB,
2157                                                   MachineBasicBlock *CurBB,
2158                                                   MachineBasicBlock *SwitchBB,
2159                                                   BranchProbability TProb,
2160                                                   BranchProbability FProb,
2161                                                   bool InvertCond) {
2162   const BasicBlock *BB = CurBB->getBasicBlock();
2163 
2164   // If the leaf of the tree is a comparison, merge the condition into
2165   // the caseblock.
2166   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2167     // The operands of the cmp have to be in this block.  We don't know
2168     // how to export them from some other block.  If this is the first block
2169     // of the sequence, no exporting is needed.
2170     if (CurBB == SwitchBB ||
2171         (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2172          isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2173       ISD::CondCode Condition;
2174       if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2175         ICmpInst::Predicate Pred =
2176             InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2177         Condition = getICmpCondCode(Pred);
2178       } else {
2179         const FCmpInst *FC = cast<FCmpInst>(Cond);
2180         FCmpInst::Predicate Pred =
2181             InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2182         Condition = getFCmpCondCode(Pred);
2183         if (TM.Options.NoNaNsFPMath)
2184           Condition = getFCmpCodeWithoutNaN(Condition);
2185       }
2186 
2187       CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2188                    TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2189       SL->SwitchCases.push_back(CB);
2190       return;
2191     }
2192   }
2193 
2194   // Create a CaseBlock record representing this branch.
2195   ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2196   CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2197                nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2198   SL->SwitchCases.push_back(CB);
2199 }
2200 
2201 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2202                                                MachineBasicBlock *TBB,
2203                                                MachineBasicBlock *FBB,
2204                                                MachineBasicBlock *CurBB,
2205                                                MachineBasicBlock *SwitchBB,
2206                                                Instruction::BinaryOps Opc,
2207                                                BranchProbability TProb,
2208                                                BranchProbability FProb,
2209                                                bool InvertCond) {
2210   // Skip over not part of the tree and remember to invert op and operands at
2211   // next level.
2212   Value *NotCond;
2213   if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2214       InBlock(NotCond, CurBB->getBasicBlock())) {
2215     FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2216                          !InvertCond);
2217     return;
2218   }
2219 
2220   const Instruction *BOp = dyn_cast<Instruction>(Cond);
2221   const Value *BOpOp0, *BOpOp1;
2222   // Compute the effective opcode for Cond, taking into account whether it needs
2223   // to be inverted, e.g.
2224   //   and (not (or A, B)), C
2225   // gets lowered as
2226   //   and (and (not A, not B), C)
2227   Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
2228   if (BOp) {
2229     BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2230                ? Instruction::And
2231                : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2232                       ? Instruction::Or
2233                       : (Instruction::BinaryOps)0);
2234     if (InvertCond) {
2235       if (BOpc == Instruction::And)
2236         BOpc = Instruction::Or;
2237       else if (BOpc == Instruction::Or)
2238         BOpc = Instruction::And;
2239     }
2240   }
2241 
2242   // If this node is not part of the or/and tree, emit it as a branch.
2243   // Note that all nodes in the tree should have same opcode.
2244   bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2245   if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2246       !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2247       !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2248     EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2249                                  TProb, FProb, InvertCond);
2250     return;
2251   }
2252 
2253   //  Create TmpBB after CurBB.
2254   MachineFunction::iterator BBI(CurBB);
2255   MachineFunction &MF = DAG.getMachineFunction();
2256   MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
2257   CurBB->getParent()->insert(++BBI, TmpBB);
2258 
2259   if (Opc == Instruction::Or) {
2260     // Codegen X | Y as:
2261     // BB1:
2262     //   jmp_if_X TBB
2263     //   jmp TmpBB
2264     // TmpBB:
2265     //   jmp_if_Y TBB
2266     //   jmp FBB
2267     //
2268 
2269     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2270     // The requirement is that
2271     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2272     //     = TrueProb for original BB.
2273     // Assuming the original probabilities are A and B, one choice is to set
2274     // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2275     // A/(1+B) and 2B/(1+B). This choice assumes that
2276     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2277     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2278     // TmpBB, but the math is more complicated.
2279 
2280     auto NewTrueProb = TProb / 2;
2281     auto NewFalseProb = TProb / 2 + FProb;
2282     // Emit the LHS condition.
2283     FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2284                          NewFalseProb, InvertCond);
2285 
2286     // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2287     SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2288     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2289     // Emit the RHS condition into TmpBB.
2290     FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2291                          Probs[1], InvertCond);
2292   } else {
2293     assert(Opc == Instruction::And && "Unknown merge op!");
2294     // Codegen X & Y as:
2295     // BB1:
2296     //   jmp_if_X TmpBB
2297     //   jmp FBB
2298     // TmpBB:
2299     //   jmp_if_Y TBB
2300     //   jmp FBB
2301     //
2302     //  This requires creation of TmpBB after CurBB.
2303 
2304     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2305     // The requirement is that
2306     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2307     //     = FalseProb for original BB.
2308     // Assuming the original probabilities are A and B, one choice is to set
2309     // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2310     // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2311     // TrueProb for BB1 * FalseProb for TmpBB.
2312 
2313     auto NewTrueProb = TProb + FProb / 2;
2314     auto NewFalseProb = FProb / 2;
2315     // Emit the LHS condition.
2316     FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2317                          NewFalseProb, InvertCond);
2318 
2319     // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2320     SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2321     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2322     // Emit the RHS condition into TmpBB.
2323     FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2324                          Probs[1], InvertCond);
2325   }
2326 }
2327 
2328 /// If the set of cases should be emitted as a series of branches, return true.
2329 /// If we should emit this as a bunch of and/or'd together conditions, return
2330 /// false.
2331 bool
2332 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2333   if (Cases.size() != 2) return true;
2334 
2335   // If this is two comparisons of the same values or'd or and'd together, they
2336   // will get folded into a single comparison, so don't emit two blocks.
2337   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2338        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2339       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2340        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2341     return false;
2342   }
2343 
2344   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2345   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2346   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2347       Cases[0].CC == Cases[1].CC &&
2348       isa<Constant>(Cases[0].CmpRHS) &&
2349       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2350     if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2351       return false;
2352     if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2353       return false;
2354   }
2355 
2356   return true;
2357 }
2358 
2359 void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2360   MachineBasicBlock *BrMBB = FuncInfo.MBB;
2361 
2362   // Update machine-CFG edges.
2363   MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2364 
2365   if (I.isUnconditional()) {
2366     // Update machine-CFG edges.
2367     BrMBB->addSuccessor(Succ0MBB);
2368 
2369     // If this is not a fall-through branch or optimizations are switched off,
2370     // emit the branch.
2371     if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2372       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2373                               MVT::Other, getControlRoot(),
2374                               DAG.getBasicBlock(Succ0MBB)));
2375 
2376     return;
2377   }
2378 
2379   // If this condition is one of the special cases we handle, do special stuff
2380   // now.
2381   const Value *CondVal = I.getCondition();
2382   MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2383 
2384   // If this is a series of conditions that are or'd or and'd together, emit
2385   // this as a sequence of branches instead of setcc's with and/or operations.
2386   // As long as jumps are not expensive (exceptions for multi-use logic ops,
2387   // unpredictable branches, and vector extracts because those jumps are likely
2388   // expensive for any target), this should improve performance.
2389   // For example, instead of something like:
2390   //     cmp A, B
2391   //     C = seteq
2392   //     cmp D, E
2393   //     F = setle
2394   //     or C, F
2395   //     jnz foo
2396   // Emit:
2397   //     cmp A, B
2398   //     je foo
2399   //     cmp D, E
2400   //     jle foo
2401   const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2402   if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2403       BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2404     Value *Vec;
2405     const Value *BOp0, *BOp1;
2406     Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
2407     if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2408       Opcode = Instruction::And;
2409     else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2410       Opcode = Instruction::Or;
2411 
2412     if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2413                     match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2414       FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2415                            getEdgeProbability(BrMBB, Succ0MBB),
2416                            getEdgeProbability(BrMBB, Succ1MBB),
2417                            /*InvertCond=*/false);
2418       // If the compares in later blocks need to use values not currently
2419       // exported from this block, export them now.  This block should always
2420       // be the first entry.
2421       assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2422 
2423       // Allow some cases to be rejected.
2424       if (ShouldEmitAsBranches(SL->SwitchCases)) {
2425         for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2426           ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2427           ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2428         }
2429 
2430         // Emit the branch for this block.
2431         visitSwitchCase(SL->SwitchCases[0], BrMBB);
2432         SL->SwitchCases.erase(SL->SwitchCases.begin());
2433         return;
2434       }
2435 
2436       // Okay, we decided not to do this, remove any inserted MBB's and clear
2437       // SwitchCases.
2438       for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2439         FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2440 
2441       SL->SwitchCases.clear();
2442     }
2443   }
2444 
2445   // Create a CaseBlock record representing this branch.
2446   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2447                nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2448 
2449   // Use visitSwitchCase to actually insert the fast branch sequence for this
2450   // cond branch.
2451   visitSwitchCase(CB, BrMBB);
2452 }
2453 
2454 /// visitSwitchCase - Emits the necessary code to represent a single node in
2455 /// the binary search tree resulting from lowering a switch instruction.
2456 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2457                                           MachineBasicBlock *SwitchBB) {
2458   SDValue Cond;
2459   SDValue CondLHS = getValue(CB.CmpLHS);
2460   SDLoc dl = CB.DL;
2461 
2462   if (CB.CC == ISD::SETTRUE) {
2463     // Branch or fall through to TrueBB.
2464     addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2465     SwitchBB->normalizeSuccProbs();
2466     if (CB.TrueBB != NextBlock(SwitchBB)) {
2467       DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2468                               DAG.getBasicBlock(CB.TrueBB)));
2469     }
2470     return;
2471   }
2472 
2473   auto &TLI = DAG.getTargetLoweringInfo();
2474   EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2475 
2476   // Build the setcc now.
2477   if (!CB.CmpMHS) {
2478     // Fold "(X == true)" to X and "(X == false)" to !X to
2479     // handle common cases produced by branch lowering.
2480     if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2481         CB.CC == ISD::SETEQ)
2482       Cond = CondLHS;
2483     else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2484              CB.CC == ISD::SETEQ) {
2485       SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2486       Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2487     } else {
2488       SDValue CondRHS = getValue(CB.CmpRHS);
2489 
2490       // If a pointer's DAG type is larger than its memory type then the DAG
2491       // values are zero-extended. This breaks signed comparisons so truncate
2492       // back to the underlying type before doing the compare.
2493       if (CondLHS.getValueType() != MemVT) {
2494         CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2495         CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2496       }
2497       Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2498     }
2499   } else {
2500     assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2501 
2502     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2503     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2504 
2505     SDValue CmpOp = getValue(CB.CmpMHS);
2506     EVT VT = CmpOp.getValueType();
2507 
2508     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2509       Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2510                           ISD::SETLE);
2511     } else {
2512       SDValue SUB = DAG.getNode(ISD::SUB, dl,
2513                                 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2514       Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2515                           DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2516     }
2517   }
2518 
2519   // Update successor info
2520   addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2521   // TrueBB and FalseBB are always different unless the incoming IR is
2522   // degenerate. This only happens when running llc on weird IR.
2523   if (CB.TrueBB != CB.FalseBB)
2524     addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2525   SwitchBB->normalizeSuccProbs();
2526 
2527   // If the lhs block is the next block, invert the condition so that we can
2528   // fall through to the lhs instead of the rhs block.
2529   if (CB.TrueBB == NextBlock(SwitchBB)) {
2530     std::swap(CB.TrueBB, CB.FalseBB);
2531     SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2532     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2533   }
2534 
2535   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2536                                MVT::Other, getControlRoot(), Cond,
2537                                DAG.getBasicBlock(CB.TrueBB));
2538 
2539   // Insert the false branch. Do this even if it's a fall through branch,
2540   // this makes it easier to do DAG optimizations which require inverting
2541   // the branch condition.
2542   BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2543                        DAG.getBasicBlock(CB.FalseBB));
2544 
2545   DAG.setRoot(BrCond);
2546 }
2547 
2548 /// visitJumpTable - Emit JumpTable node in the current MBB
2549 void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
2550   // Emit the code for the jump table
2551   assert(JT.Reg != -1U && "Should lower JT Header first!");
2552   EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2553   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2554                                      JT.Reg, PTy);
2555   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2556   SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2557                                     MVT::Other, Index.getValue(1),
2558                                     Table, Index);
2559   DAG.setRoot(BrJumpTable);
2560 }
2561 
2562 /// visitJumpTableHeader - This function emits necessary code to produce index
2563 /// in the JumpTable from switch case.
2564 void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
2565                                                JumpTableHeader &JTH,
2566                                                MachineBasicBlock *SwitchBB) {
2567   SDLoc dl = getCurSDLoc();
2568 
2569   // Subtract the lowest switch case value from the value being switched on.
2570   SDValue SwitchOp = getValue(JTH.SValue);
2571   EVT VT = SwitchOp.getValueType();
2572   SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2573                             DAG.getConstant(JTH.First, dl, VT));
2574 
2575   // The SDNode we just created, which holds the value being switched on minus
2576   // the smallest case value, needs to be copied to a virtual register so it
2577   // can be used as an index into the jump table in a subsequent basic block.
2578   // This value may be smaller or larger than the target's pointer type, and
2579   // therefore require extension or truncating.
2580   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2581   SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2582 
2583   unsigned JumpTableReg =
2584       FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2585   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2586                                     JumpTableReg, SwitchOp);
2587   JT.Reg = JumpTableReg;
2588 
2589   if (!JTH.FallthroughUnreachable) {
2590     // Emit the range check for the jump table, and branch to the default block
2591     // for the switch statement if the value being switched on exceeds the
2592     // largest case in the switch.
2593     SDValue CMP = DAG.getSetCC(
2594         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2595                                    Sub.getValueType()),
2596         Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2597 
2598     SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2599                                  MVT::Other, CopyTo, CMP,
2600                                  DAG.getBasicBlock(JT.Default));
2601 
2602     // Avoid emitting unnecessary branches to the next block.
2603     if (JT.MBB != NextBlock(SwitchBB))
2604       BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2605                            DAG.getBasicBlock(JT.MBB));
2606 
2607     DAG.setRoot(BrCond);
2608   } else {
2609     // Avoid emitting unnecessary branches to the next block.
2610     if (JT.MBB != NextBlock(SwitchBB))
2611       DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2612                               DAG.getBasicBlock(JT.MBB)));
2613     else
2614       DAG.setRoot(CopyTo);
2615   }
2616 }
2617 
2618 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2619 /// variable if there exists one.
2620 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2621                                  SDValue &Chain) {
2622   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2623   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2624   EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2625   MachineFunction &MF = DAG.getMachineFunction();
2626   Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2627   MachineSDNode *Node =
2628       DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2629   if (Global) {
2630     MachinePointerInfo MPInfo(Global);
2631     auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2632                  MachineMemOperand::MODereferenceable;
2633     MachineMemOperand *MemRef = MF.getMachineMemOperand(
2634         MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2635     DAG.setNodeMemRefs(Node, {MemRef});
2636   }
2637   if (PtrTy != PtrMemTy)
2638     return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2639   return SDValue(Node, 0);
2640 }
2641 
2642 /// Codegen a new tail for a stack protector check ParentMBB which has had its
2643 /// tail spliced into a stack protector check success bb.
2644 ///
2645 /// For a high level explanation of how this fits into the stack protector
2646 /// generation see the comment on the declaration of class
2647 /// StackProtectorDescriptor.
2648 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2649                                                   MachineBasicBlock *ParentBB) {
2650 
2651   // First create the loads to the guard/stack slot for the comparison.
2652   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2653   EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2654   EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2655 
2656   MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2657   int FI = MFI.getStackProtectorIndex();
2658 
2659   SDValue Guard;
2660   SDLoc dl = getCurSDLoc();
2661   SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2662   const Module &M = *ParentBB->getParent()->getFunction().getParent();
2663   Align Align =
2664       DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
2665 
2666   // Generate code to load the content of the guard slot.
2667   SDValue GuardVal = DAG.getLoad(
2668       PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2669       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2670       MachineMemOperand::MOVolatile);
2671 
2672   if (TLI.useStackGuardXorFP())
2673     GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2674 
2675   // Retrieve guard check function, nullptr if instrumentation is inlined.
2676   if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2677     // The target provides a guard check function to validate the guard value.
2678     // Generate a call to that function with the content of the guard slot as
2679     // argument.
2680     FunctionType *FnTy = GuardCheckFn->getFunctionType();
2681     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2682 
2683     TargetLowering::ArgListTy Args;
2684     TargetLowering::ArgListEntry Entry;
2685     Entry.Node = GuardVal;
2686     Entry.Ty = FnTy->getParamType(0);
2687     if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2688       Entry.IsInReg = true;
2689     Args.push_back(Entry);
2690 
2691     TargetLowering::CallLoweringInfo CLI(DAG);
2692     CLI.setDebugLoc(getCurSDLoc())
2693         .setChain(DAG.getEntryNode())
2694         .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2695                    getValue(GuardCheckFn), std::move(Args));
2696 
2697     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2698     DAG.setRoot(Result.second);
2699     return;
2700   }
2701 
2702   // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2703   // Otherwise, emit a volatile load to retrieve the stack guard value.
2704   SDValue Chain = DAG.getEntryNode();
2705   if (TLI.useLoadStackGuardNode()) {
2706     Guard = getLoadStackGuard(DAG, dl, Chain);
2707   } else {
2708     const Value *IRGuard = TLI.getSDagStackGuard(M);
2709     SDValue GuardPtr = getValue(IRGuard);
2710 
2711     Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2712                         MachinePointerInfo(IRGuard, 0), Align,
2713                         MachineMemOperand::MOVolatile);
2714   }
2715 
2716   // Perform the comparison via a getsetcc.
2717   SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2718                                                         *DAG.getContext(),
2719                                                         Guard.getValueType()),
2720                              Guard, GuardVal, ISD::SETNE);
2721 
2722   // If the guard/stackslot do not equal, branch to failure MBB.
2723   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2724                                MVT::Other, GuardVal.getOperand(0),
2725                                Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2726   // Otherwise branch to success MBB.
2727   SDValue Br = DAG.getNode(ISD::BR, dl,
2728                            MVT::Other, BrCond,
2729                            DAG.getBasicBlock(SPD.getSuccessMBB()));
2730 
2731   DAG.setRoot(Br);
2732 }
2733 
2734 /// Codegen the failure basic block for a stack protector check.
2735 ///
2736 /// A failure stack protector machine basic block consists simply of a call to
2737 /// __stack_chk_fail().
2738 ///
2739 /// For a high level explanation of how this fits into the stack protector
2740 /// generation see the comment on the declaration of class
2741 /// StackProtectorDescriptor.
2742 void
2743 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2744   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2745   TargetLowering::MakeLibCallOptions CallOptions;
2746   CallOptions.setDiscardResult(true);
2747   SDValue Chain =
2748       TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2749                       None, CallOptions, getCurSDLoc()).second;
2750   // On PS4/PS5, the "return address" must still be within the calling
2751   // function, even if it's at the very end, so emit an explicit TRAP here.
2752   // Passing 'true' for doesNotReturn above won't generate the trap for us.
2753   if (TM.getTargetTriple().isPS())
2754     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2755   // WebAssembly needs an unreachable instruction after a non-returning call,
2756   // because the function return type can be different from __stack_chk_fail's
2757   // return type (void).
2758   if (TM.getTargetTriple().isWasm())
2759     Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2760 
2761   DAG.setRoot(Chain);
2762 }
2763 
2764 /// visitBitTestHeader - This function emits necessary code to produce value
2765 /// suitable for "bit tests"
2766 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2767                                              MachineBasicBlock *SwitchBB) {
2768   SDLoc dl = getCurSDLoc();
2769 
2770   // Subtract the minimum value.
2771   SDValue SwitchOp = getValue(B.SValue);
2772   EVT VT = SwitchOp.getValueType();
2773   SDValue RangeSub =
2774       DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2775 
2776   // Determine the type of the test operands.
2777   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2778   bool UsePtrType = false;
2779   if (!TLI.isTypeLegal(VT)) {
2780     UsePtrType = true;
2781   } else {
2782     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2783       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2784         // Switch table case range are encoded into series of masks.
2785         // Just use pointer type, it's guaranteed to fit.
2786         UsePtrType = true;
2787         break;
2788       }
2789   }
2790   SDValue Sub = RangeSub;
2791   if (UsePtrType) {
2792     VT = TLI.getPointerTy(DAG.getDataLayout());
2793     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2794   }
2795 
2796   B.RegVT = VT.getSimpleVT();
2797   B.Reg = FuncInfo.CreateReg(B.RegVT);
2798   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2799 
2800   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2801 
2802   if (!B.FallthroughUnreachable)
2803     addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2804   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2805   SwitchBB->normalizeSuccProbs();
2806 
2807   SDValue Root = CopyTo;
2808   if (!B.FallthroughUnreachable) {
2809     // Conditional branch to the default block.
2810     SDValue RangeCmp = DAG.getSetCC(dl,
2811         TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2812                                RangeSub.getValueType()),
2813         RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2814         ISD::SETUGT);
2815 
2816     Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2817                        DAG.getBasicBlock(B.Default));
2818   }
2819 
2820   // Avoid emitting unnecessary branches to the next block.
2821   if (MBB != NextBlock(SwitchBB))
2822     Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2823 
2824   DAG.setRoot(Root);
2825 }
2826 
2827 /// visitBitTestCase - this function produces one "bit test"
2828 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2829                                            MachineBasicBlock* NextMBB,
2830                                            BranchProbability BranchProbToNext,
2831                                            unsigned Reg,
2832                                            BitTestCase &B,
2833                                            MachineBasicBlock *SwitchBB) {
2834   SDLoc dl = getCurSDLoc();
2835   MVT VT = BB.RegVT;
2836   SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2837   SDValue Cmp;
2838   unsigned PopCount = countPopulation(B.Mask);
2839   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2840   if (PopCount == 1) {
2841     // Testing for a single bit; just compare the shift count with what it
2842     // would need to be to shift a 1 bit in that position.
2843     Cmp = DAG.getSetCC(
2844         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2845         ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2846         ISD::SETEQ);
2847   } else if (PopCount == BB.Range) {
2848     // There is only one zero bit in the range, test for it directly.
2849     Cmp = DAG.getSetCC(
2850         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2851         ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2852         ISD::SETNE);
2853   } else {
2854     // Make desired shift
2855     SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2856                                     DAG.getConstant(1, dl, VT), ShiftOp);
2857 
2858     // Emit bit tests and jumps
2859     SDValue AndOp = DAG.getNode(ISD::AND, dl,
2860                                 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2861     Cmp = DAG.getSetCC(
2862         dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2863         AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2864   }
2865 
2866   // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2867   addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2868   // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2869   addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2870   // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2871   // one as they are relative probabilities (and thus work more like weights),
2872   // and hence we need to normalize them to let the sum of them become one.
2873   SwitchBB->normalizeSuccProbs();
2874 
2875   SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2876                               MVT::Other, getControlRoot(),
2877                               Cmp, DAG.getBasicBlock(B.TargetBB));
2878 
2879   // Avoid emitting unnecessary branches to the next block.
2880   if (NextMBB != NextBlock(SwitchBB))
2881     BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2882                         DAG.getBasicBlock(NextMBB));
2883 
2884   DAG.setRoot(BrAnd);
2885 }
2886 
2887 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2888   MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2889 
2890   // Retrieve successors. Look through artificial IR level blocks like
2891   // catchswitch for successors.
2892   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2893   const BasicBlock *EHPadBB = I.getSuccessor(1);
2894 
2895   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2896   // have to do anything here to lower funclet bundles.
2897   assert(!I.hasOperandBundlesOtherThan(
2898              {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
2899               LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
2900               LLVMContext::OB_cfguardtarget,
2901               LLVMContext::OB_clang_arc_attachedcall}) &&
2902          "Cannot lower invokes with arbitrary operand bundles yet!");
2903 
2904   const Value *Callee(I.getCalledOperand());
2905   const Function *Fn = dyn_cast<Function>(Callee);
2906   if (isa<InlineAsm>(Callee))
2907     visitInlineAsm(I, EHPadBB);
2908   else if (Fn && Fn->isIntrinsic()) {
2909     switch (Fn->getIntrinsicID()) {
2910     default:
2911       llvm_unreachable("Cannot invoke this intrinsic");
2912     case Intrinsic::donothing:
2913       // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2914     case Intrinsic::seh_try_begin:
2915     case Intrinsic::seh_scope_begin:
2916     case Intrinsic::seh_try_end:
2917     case Intrinsic::seh_scope_end:
2918       break;
2919     case Intrinsic::experimental_patchpoint_void:
2920     case Intrinsic::experimental_patchpoint_i64:
2921       visitPatchpoint(I, EHPadBB);
2922       break;
2923     case Intrinsic::experimental_gc_statepoint:
2924       LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
2925       break;
2926     case Intrinsic::wasm_rethrow: {
2927       // This is usually done in visitTargetIntrinsic, but this intrinsic is
2928       // special because it can be invoked, so we manually lower it to a DAG
2929       // node here.
2930       SmallVector<SDValue, 8> Ops;
2931       Ops.push_back(getRoot()); // inchain
2932       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2933       Ops.push_back(
2934           DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
2935                                 TLI.getPointerTy(DAG.getDataLayout())));
2936       SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2937       DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
2938       break;
2939     }
2940     }
2941   } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2942     // Currently we do not lower any intrinsic calls with deopt operand bundles.
2943     // Eventually we will support lowering the @llvm.experimental.deoptimize
2944     // intrinsic, and right now there are no plans to support other intrinsics
2945     // with deopt state.
2946     LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2947   } else {
2948     LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
2949   }
2950 
2951   // If the value of the invoke is used outside of its defining block, make it
2952   // available as a virtual register.
2953   // We already took care of the exported value for the statepoint instruction
2954   // during call to the LowerStatepoint.
2955   if (!isa<GCStatepointInst>(I)) {
2956     CopyToExportRegsIfNeeded(&I);
2957   }
2958 
2959   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2960   BranchProbabilityInfo *BPI = FuncInfo.BPI;
2961   BranchProbability EHPadBBProb =
2962       BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2963           : BranchProbability::getZero();
2964   findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2965 
2966   // Update successor info.
2967   addSuccessorWithProb(InvokeMBB, Return);
2968   for (auto &UnwindDest : UnwindDests) {
2969     UnwindDest.first->setIsEHPad();
2970     addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2971   }
2972   InvokeMBB->normalizeSuccProbs();
2973 
2974   // Drop into normal successor.
2975   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2976                           DAG.getBasicBlock(Return)));
2977 }
2978 
2979 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2980   MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2981 
2982   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2983   // have to do anything here to lower funclet bundles.
2984   assert(!I.hasOperandBundlesOtherThan(
2985              {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2986          "Cannot lower callbrs with arbitrary operand bundles yet!");
2987 
2988   assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
2989   visitInlineAsm(I);
2990   CopyToExportRegsIfNeeded(&I);
2991 
2992   // Retrieve successors.
2993   SmallPtrSet<BasicBlock *, 8> Dests;
2994   Dests.insert(I.getDefaultDest());
2995   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2996 
2997   // Update successor info.
2998   addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
2999   for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3000     BasicBlock *Dest = I.getIndirectDest(i);
3001     MachineBasicBlock *Target = FuncInfo.MBBMap[Dest];
3002     Target->setIsInlineAsmBrIndirectTarget();
3003     Target->setHasAddressTaken();
3004     // Don't add duplicate machine successors.
3005     if (Dests.insert(Dest).second)
3006       addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3007   }
3008   CallBrMBB->normalizeSuccProbs();
3009 
3010   // Drop into default successor.
3011   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3012                           MVT::Other, getControlRoot(),
3013                           DAG.getBasicBlock(Return)));
3014 }
3015 
3016 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3017   llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3018 }
3019 
3020 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3021   assert(FuncInfo.MBB->isEHPad() &&
3022          "Call to landingpad not in landing pad!");
3023 
3024   // If there aren't registers to copy the values into (e.g., during SjLj
3025   // exceptions), then don't bother to create these DAG nodes.
3026   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3027   const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3028   if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3029       TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3030     return;
3031 
3032   // If landingpad's return type is token type, we don't create DAG nodes
3033   // for its exception pointer and selector value. The extraction of exception
3034   // pointer or selector value from token type landingpads is not currently
3035   // supported.
3036   if (LP.getType()->isTokenTy())
3037     return;
3038 
3039   SmallVector<EVT, 2> ValueVTs;
3040   SDLoc dl = getCurSDLoc();
3041   ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3042   assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3043 
3044   // Get the two live-in registers as SDValues. The physregs have already been
3045   // copied into virtual registers.
3046   SDValue Ops[2];
3047   if (FuncInfo.ExceptionPointerVirtReg) {
3048     Ops[0] = DAG.getZExtOrTrunc(
3049         DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3050                            FuncInfo.ExceptionPointerVirtReg,
3051                            TLI.getPointerTy(DAG.getDataLayout())),
3052         dl, ValueVTs[0]);
3053   } else {
3054     Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3055   }
3056   Ops[1] = DAG.getZExtOrTrunc(
3057       DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3058                          FuncInfo.ExceptionSelectorVirtReg,
3059                          TLI.getPointerTy(DAG.getDataLayout())),
3060       dl, ValueVTs[1]);
3061 
3062   // Merge into one.
3063   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3064                             DAG.getVTList(ValueVTs), Ops);
3065   setValue(&LP, Res);
3066 }
3067 
3068 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
3069                                            MachineBasicBlock *Last) {
3070   // Update JTCases.
3071   for (JumpTableBlock &JTB : SL->JTCases)
3072     if (JTB.first.HeaderBB == First)
3073       JTB.first.HeaderBB = Last;
3074 
3075   // Update BitTestCases.
3076   for (BitTestBlock &BTB : SL->BitTestCases)
3077     if (BTB.Parent == First)
3078       BTB.Parent = Last;
3079 }
3080 
3081 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3082   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3083 
3084   // Update machine-CFG edges with unique successors.
3085   SmallSet<BasicBlock*, 32> Done;
3086   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3087     BasicBlock *BB = I.getSuccessor(i);
3088     bool Inserted = Done.insert(BB).second;
3089     if (!Inserted)
3090         continue;
3091 
3092     MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3093     addSuccessorWithProb(IndirectBrMBB, Succ);
3094   }
3095   IndirectBrMBB->normalizeSuccProbs();
3096 
3097   DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
3098                           MVT::Other, getControlRoot(),
3099                           getValue(I.getAddress())));
3100 }
3101 
3102 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3103   if (!DAG.getTarget().Options.TrapUnreachable)
3104     return;
3105 
3106   // We may be able to ignore unreachable behind a noreturn call.
3107   if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
3108     const BasicBlock &BB = *I.getParent();
3109     if (&I != &BB.front()) {
3110       BasicBlock::const_iterator PredI =
3111         std::prev(BasicBlock::const_iterator(&I));
3112       if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3113         if (Call->doesNotReturn())
3114           return;
3115       }
3116     }
3117   }
3118 
3119   DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3120 }
3121 
3122 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3123   SDNodeFlags Flags;
3124   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3125     Flags.copyFMF(*FPOp);
3126 
3127   SDValue Op = getValue(I.getOperand(0));
3128   SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3129                                     Op, Flags);
3130   setValue(&I, UnNodeValue);
3131 }
3132 
3133 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3134   SDNodeFlags Flags;
3135   if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3136     Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3137     Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3138   }
3139   if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3140     Flags.setExact(ExactOp->isExact());
3141   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3142     Flags.copyFMF(*FPOp);
3143 
3144   SDValue Op1 = getValue(I.getOperand(0));
3145   SDValue Op2 = getValue(I.getOperand(1));
3146   SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3147                                      Op1, Op2, Flags);
3148   setValue(&I, BinNodeValue);
3149 }
3150 
3151 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3152   SDValue Op1 = getValue(I.getOperand(0));
3153   SDValue Op2 = getValue(I.getOperand(1));
3154 
3155   EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3156       Op1.getValueType(), DAG.getDataLayout());
3157 
3158   // Coerce the shift amount to the right type if we can. This exposes the
3159   // truncate or zext to optimization early.
3160   if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3161     assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&
3162            "Unexpected shift type");
3163     Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3164   }
3165 
3166   bool nuw = false;
3167   bool nsw = false;
3168   bool exact = false;
3169 
3170   if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3171 
3172     if (const OverflowingBinaryOperator *OFBinOp =
3173             dyn_cast<const OverflowingBinaryOperator>(&I)) {
3174       nuw = OFBinOp->hasNoUnsignedWrap();
3175       nsw = OFBinOp->hasNoSignedWrap();
3176     }
3177     if (const PossiblyExactOperator *ExactOp =
3178             dyn_cast<const PossiblyExactOperator>(&I))
3179       exact = ExactOp->isExact();
3180   }
3181   SDNodeFlags Flags;
3182   Flags.setExact(exact);
3183   Flags.setNoSignedWrap(nsw);
3184   Flags.setNoUnsignedWrap(nuw);
3185   SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3186                             Flags);
3187   setValue(&I, Res);
3188 }
3189 
3190 void SelectionDAGBuilder::visitSDiv(const User &I) {
3191   SDValue Op1 = getValue(I.getOperand(0));
3192   SDValue Op2 = getValue(I.getOperand(1));
3193 
3194   SDNodeFlags Flags;
3195   Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3196                  cast<PossiblyExactOperator>(&I)->isExact());
3197   setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3198                            Op2, Flags));
3199 }
3200 
3201 void SelectionDAGBuilder::visitICmp(const User &I) {
3202   ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
3203   if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3204     predicate = IC->getPredicate();
3205   else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3206     predicate = ICmpInst::Predicate(IC->getPredicate());
3207   SDValue Op1 = getValue(I.getOperand(0));
3208   SDValue Op2 = getValue(I.getOperand(1));
3209   ISD::CondCode Opcode = getICmpCondCode(predicate);
3210 
3211   auto &TLI = DAG.getTargetLoweringInfo();
3212   EVT MemVT =
3213       TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3214 
3215   // If a pointer's DAG type is larger than its memory type then the DAG values
3216   // are zero-extended. This breaks signed comparisons so truncate back to the
3217   // underlying type before doing the compare.
3218   if (Op1.getValueType() != MemVT) {
3219     Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3220     Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3221   }
3222 
3223   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3224                                                         I.getType());
3225   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3226 }
3227 
3228 void SelectionDAGBuilder::visitFCmp(const User &I) {
3229   FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
3230   if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3231     predicate = FC->getPredicate();
3232   else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3233     predicate = FCmpInst::Predicate(FC->getPredicate());
3234   SDValue Op1 = getValue(I.getOperand(0));
3235   SDValue Op2 = getValue(I.getOperand(1));
3236 
3237   ISD::CondCode Condition = getFCmpCondCode(predicate);
3238   auto *FPMO = cast<FPMathOperator>(&I);
3239   if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3240     Condition = getFCmpCodeWithoutNaN(Condition);
3241 
3242   SDNodeFlags Flags;
3243   Flags.copyFMF(*FPMO);
3244   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3245 
3246   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3247                                                         I.getType());
3248   setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3249 }
3250 
3251 // Check if the condition of the select has one use or two users that are both
3252 // selects with the same condition.
3253 static bool hasOnlySelectUsers(const Value *Cond) {
3254   return llvm::all_of(Cond->users(), [](const Value *V) {
3255     return isa<SelectInst>(V);
3256   });
3257 }
3258 
3259 void SelectionDAGBuilder::visitSelect(const User &I) {
3260   SmallVector<EVT, 4> ValueVTs;
3261   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3262                   ValueVTs);
3263   unsigned NumValues = ValueVTs.size();
3264   if (NumValues == 0) return;
3265 
3266   SmallVector<SDValue, 4> Values(NumValues);
3267   SDValue Cond     = getValue(I.getOperand(0));
3268   SDValue LHSVal   = getValue(I.getOperand(1));
3269   SDValue RHSVal   = getValue(I.getOperand(2));
3270   SmallVector<SDValue, 1> BaseOps(1, Cond);
3271   ISD::NodeType OpCode =
3272       Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3273 
3274   bool IsUnaryAbs = false;
3275   bool Negate = false;
3276 
3277   SDNodeFlags Flags;
3278   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3279     Flags.copyFMF(*FPOp);
3280 
3281   // Min/max matching is only viable if all output VTs are the same.
3282   if (is_splat(ValueVTs)) {
3283     EVT VT = ValueVTs[0];
3284     LLVMContext &Ctx = *DAG.getContext();
3285     auto &TLI = DAG.getTargetLoweringInfo();
3286 
3287     // We care about the legality of the operation after it has been type
3288     // legalized.
3289     while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3290       VT = TLI.getTypeToTransformTo(Ctx, VT);
3291 
3292     // If the vselect is legal, assume we want to leave this as a vector setcc +
3293     // vselect. Otherwise, if this is going to be scalarized, we want to see if
3294     // min/max is legal on the scalar type.
3295     bool UseScalarMinMax = VT.isVector() &&
3296       !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
3297 
3298     Value *LHS, *RHS;
3299     auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3300     ISD::NodeType Opc = ISD::DELETED_NODE;
3301     switch (SPR.Flavor) {
3302     case SPF_UMAX:    Opc = ISD::UMAX; break;
3303     case SPF_UMIN:    Opc = ISD::UMIN; break;
3304     case SPF_SMAX:    Opc = ISD::SMAX; break;
3305     case SPF_SMIN:    Opc = ISD::SMIN; break;
3306     case SPF_FMINNUM:
3307       switch (SPR.NaNBehavior) {
3308       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3309       case SPNB_RETURNS_NAN:   Opc = ISD::FMINIMUM; break;
3310       case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3311       case SPNB_RETURNS_ANY: {
3312         if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
3313           Opc = ISD::FMINNUM;
3314         else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3315           Opc = ISD::FMINIMUM;
3316         else if (UseScalarMinMax)
3317           Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3318             ISD::FMINNUM : ISD::FMINIMUM;
3319         break;
3320       }
3321       }
3322       break;
3323     case SPF_FMAXNUM:
3324       switch (SPR.NaNBehavior) {
3325       case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3326       case SPNB_RETURNS_NAN:   Opc = ISD::FMAXIMUM; break;
3327       case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3328       case SPNB_RETURNS_ANY:
3329 
3330         if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3331           Opc = ISD::FMAXNUM;
3332         else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3333           Opc = ISD::FMAXIMUM;
3334         else if (UseScalarMinMax)
3335           Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3336             ISD::FMAXNUM : ISD::FMAXIMUM;
3337         break;
3338       }
3339       break;
3340     case SPF_NABS:
3341       Negate = true;
3342       LLVM_FALLTHROUGH;
3343     case SPF_ABS:
3344       IsUnaryAbs = true;
3345       Opc = ISD::ABS;
3346       break;
3347     default: break;
3348     }
3349 
3350     if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3351         (TLI.isOperationLegalOrCustom(Opc, VT) ||
3352          (UseScalarMinMax &&
3353           TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3354         // If the underlying comparison instruction is used by any other
3355         // instruction, the consumed instructions won't be destroyed, so it is
3356         // not profitable to convert to a min/max.
3357         hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3358       OpCode = Opc;
3359       LHSVal = getValue(LHS);
3360       RHSVal = getValue(RHS);
3361       BaseOps.clear();
3362     }
3363 
3364     if (IsUnaryAbs) {
3365       OpCode = Opc;
3366       LHSVal = getValue(LHS);
3367       BaseOps.clear();
3368     }
3369   }
3370 
3371   if (IsUnaryAbs) {
3372     for (unsigned i = 0; i != NumValues; ++i) {
3373       SDLoc dl = getCurSDLoc();
3374       EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3375       Values[i] =
3376           DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3377       if (Negate)
3378         Values[i] = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
3379                                 Values[i]);
3380     }
3381   } else {
3382     for (unsigned i = 0; i != NumValues; ++i) {
3383       SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3384       Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3385       Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3386       Values[i] = DAG.getNode(
3387           OpCode, getCurSDLoc(),
3388           LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3389     }
3390   }
3391 
3392   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3393                            DAG.getVTList(ValueVTs), Values));
3394 }
3395 
3396 void SelectionDAGBuilder::visitTrunc(const User &I) {
3397   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3398   SDValue N = getValue(I.getOperand(0));
3399   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3400                                                         I.getType());
3401   setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3402 }
3403 
3404 void SelectionDAGBuilder::visitZExt(const User &I) {
3405   // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3406   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3407   SDValue N = getValue(I.getOperand(0));
3408   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3409                                                         I.getType());
3410   setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3411 }
3412 
3413 void SelectionDAGBuilder::visitSExt(const User &I) {
3414   // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3415   // SExt also can't be a cast to bool for same reason. So, nothing much to do
3416   SDValue N = getValue(I.getOperand(0));
3417   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3418                                                         I.getType());
3419   setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3420 }
3421 
3422 void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3423   // FPTrunc is never a no-op cast, no need to check
3424   SDValue N = getValue(I.getOperand(0));
3425   SDLoc dl = getCurSDLoc();
3426   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3427   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3428   setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3429                            DAG.getTargetConstant(
3430                                0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3431 }
3432 
3433 void SelectionDAGBuilder::visitFPExt(const User &I) {
3434   // FPExt is never a no-op cast, no need to check
3435   SDValue N = getValue(I.getOperand(0));
3436   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3437                                                         I.getType());
3438   setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3439 }
3440 
3441 void SelectionDAGBuilder::visitFPToUI(const User &I) {
3442   // FPToUI is never a no-op cast, no need to check
3443   SDValue N = getValue(I.getOperand(0));
3444   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3445                                                         I.getType());
3446   setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3447 }
3448 
3449 void SelectionDAGBuilder::visitFPToSI(const User &I) {
3450   // FPToSI is never a no-op cast, no need to check
3451   SDValue N = getValue(I.getOperand(0));
3452   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3453                                                         I.getType());
3454   setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3455 }
3456 
3457 void SelectionDAGBuilder::visitUIToFP(const User &I) {
3458   // UIToFP is never a no-op cast, no need to check
3459   SDValue N = getValue(I.getOperand(0));
3460   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3461                                                         I.getType());
3462   setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3463 }
3464 
3465 void SelectionDAGBuilder::visitSIToFP(const User &I) {
3466   // SIToFP is never a no-op cast, no need to check
3467   SDValue N = getValue(I.getOperand(0));
3468   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3469                                                         I.getType());
3470   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3471 }
3472 
3473 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3474   // What to do depends on the size of the integer and the size of the pointer.
3475   // We can either truncate, zero extend, or no-op, accordingly.
3476   SDValue N = getValue(I.getOperand(0));
3477   auto &TLI = DAG.getTargetLoweringInfo();
3478   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3479                                                         I.getType());
3480   EVT PtrMemVT =
3481       TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3482   N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3483   N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3484   setValue(&I, N);
3485 }
3486 
3487 void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3488   // What to do depends on the size of the integer and the size of the pointer.
3489   // We can either truncate, zero extend, or no-op, accordingly.
3490   SDValue N = getValue(I.getOperand(0));
3491   auto &TLI = DAG.getTargetLoweringInfo();
3492   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3493   EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3494   N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3495   N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3496   setValue(&I, N);
3497 }
3498 
3499 void SelectionDAGBuilder::visitBitCast(const User &I) {
3500   SDValue N = getValue(I.getOperand(0));
3501   SDLoc dl = getCurSDLoc();
3502   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3503                                                         I.getType());
3504 
3505   // BitCast assures us that source and destination are the same size so this is
3506   // either a BITCAST or a no-op.
3507   if (DestVT != N.getValueType())
3508     setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3509                              DestVT, N)); // convert types.
3510   // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3511   // might fold any kind of constant expression to an integer constant and that
3512   // is not what we are looking for. Only recognize a bitcast of a genuine
3513   // constant integer as an opaque constant.
3514   else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3515     setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3516                                  /*isOpaque*/true));
3517   else
3518     setValue(&I, N);            // noop cast.
3519 }
3520 
3521 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3522   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3523   const Value *SV = I.getOperand(0);
3524   SDValue N = getValue(SV);
3525   EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3526 
3527   unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3528   unsigned DestAS = I.getType()->getPointerAddressSpace();
3529 
3530   if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3531     N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3532 
3533   setValue(&I, N);
3534 }
3535 
3536 void SelectionDAGBuilder::visitInsertElement(const User &I) {
3537   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3538   SDValue InVec = getValue(I.getOperand(0));
3539   SDValue InVal = getValue(I.getOperand(1));
3540   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3541                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3542   setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3543                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3544                            InVec, InVal, InIdx));
3545 }
3546 
3547 void SelectionDAGBuilder::visitExtractElement(const User &I) {
3548   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3549   SDValue InVec = getValue(I.getOperand(0));
3550   SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3551                                      TLI.getVectorIdxTy(DAG.getDataLayout()));
3552   setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3553                            TLI.getValueType(DAG.getDataLayout(), I.getType()),
3554                            InVec, InIdx));
3555 }
3556 
3557 void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3558   SDValue Src1 = getValue(I.getOperand(0));
3559   SDValue Src2 = getValue(I.getOperand(1));
3560   ArrayRef<int> Mask;
3561   if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3562     Mask = SVI->getShuffleMask();
3563   else
3564     Mask = cast<ConstantExpr>(I).getShuffleMask();
3565   SDLoc DL = getCurSDLoc();
3566   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3567   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3568   EVT SrcVT = Src1.getValueType();
3569 
3570   if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3571       VT.isScalableVector()) {
3572     // Canonical splat form of first element of first input vector.
3573     SDValue FirstElt =
3574         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
3575                     DAG.getVectorIdxConstant(0, DL));
3576     setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3577     return;
3578   }
3579 
3580   // For now, we only handle splats for scalable vectors.
3581   // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3582   // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3583   assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3584 
3585   unsigned SrcNumElts = SrcVT.getVectorNumElements();
3586   unsigned MaskNumElts = Mask.size();
3587 
3588   if (SrcNumElts == MaskNumElts) {
3589     setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3590     return;
3591   }
3592 
3593   // Normalize the shuffle vector since mask and vector length don't match.
3594   if (SrcNumElts < MaskNumElts) {
3595     // Mask is longer than the source vectors. We can use concatenate vector to
3596     // make the mask and vectors lengths match.
3597 
3598     if (MaskNumElts % SrcNumElts == 0) {
3599       // Mask length is a multiple of the source vector length.
3600       // Check if the shuffle is some kind of concatenation of the input
3601       // vectors.
3602       unsigned NumConcat = MaskNumElts / SrcNumElts;
3603       bool IsConcat = true;
3604       SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3605       for (unsigned i = 0; i != MaskNumElts; ++i) {
3606         int Idx = Mask[i];
3607         if (Idx < 0)
3608           continue;
3609         // Ensure the indices in each SrcVT sized piece are sequential and that
3610         // the same source is used for the whole piece.
3611         if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3612             (ConcatSrcs[i / SrcNumElts] >= 0 &&
3613              ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3614           IsConcat = false;
3615           break;
3616         }
3617         // Remember which source this index came from.
3618         ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3619       }
3620 
3621       // The shuffle is concatenating multiple vectors together. Just emit
3622       // a CONCAT_VECTORS operation.
3623       if (IsConcat) {
3624         SmallVector<SDValue, 8> ConcatOps;
3625         for (auto Src : ConcatSrcs) {
3626           if (Src < 0)
3627             ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3628           else if (Src == 0)
3629             ConcatOps.push_back(Src1);
3630           else
3631             ConcatOps.push_back(Src2);
3632         }
3633         setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3634         return;
3635       }
3636     }
3637 
3638     unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3639     unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3640     EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3641                                     PaddedMaskNumElts);
3642 
3643     // Pad both vectors with undefs to make them the same length as the mask.
3644     SDValue UndefVal = DAG.getUNDEF(SrcVT);
3645 
3646     SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3647     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3648     MOps1[0] = Src1;
3649     MOps2[0] = Src2;
3650 
3651     Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3652     Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3653 
3654     // Readjust mask for new input vector length.
3655     SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3656     for (unsigned i = 0; i != MaskNumElts; ++i) {
3657       int Idx = Mask[i];
3658       if (Idx >= (int)SrcNumElts)
3659         Idx -= SrcNumElts - PaddedMaskNumElts;
3660       MappedOps[i] = Idx;
3661     }
3662 
3663     SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3664 
3665     // If the concatenated vector was padded, extract a subvector with the
3666     // correct number of elements.
3667     if (MaskNumElts != PaddedMaskNumElts)
3668       Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3669                            DAG.getVectorIdxConstant(0, DL));
3670 
3671     setValue(&I, Result);
3672     return;
3673   }
3674 
3675   if (SrcNumElts > MaskNumElts) {
3676     // Analyze the access pattern of the vector to see if we can extract
3677     // two subvectors and do the shuffle.
3678     int StartIdx[2] = { -1, -1 };  // StartIdx to extract from
3679     bool CanExtract = true;
3680     for (int Idx : Mask) {
3681       unsigned Input = 0;
3682       if (Idx < 0)
3683         continue;
3684 
3685       if (Idx >= (int)SrcNumElts) {
3686         Input = 1;
3687         Idx -= SrcNumElts;
3688       }
3689 
3690       // If all the indices come from the same MaskNumElts sized portion of
3691       // the sources we can use extract. Also make sure the extract wouldn't
3692       // extract past the end of the source.
3693       int NewStartIdx = alignDown(Idx, MaskNumElts);
3694       if (NewStartIdx + MaskNumElts > SrcNumElts ||
3695           (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3696         CanExtract = false;
3697       // Make sure we always update StartIdx as we use it to track if all
3698       // elements are undef.
3699       StartIdx[Input] = NewStartIdx;
3700     }
3701 
3702     if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3703       setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3704       return;
3705     }
3706     if (CanExtract) {
3707       // Extract appropriate subvector and generate a vector shuffle
3708       for (unsigned Input = 0; Input < 2; ++Input) {
3709         SDValue &Src = Input == 0 ? Src1 : Src2;
3710         if (StartIdx[Input] < 0)
3711           Src = DAG.getUNDEF(VT);
3712         else {
3713           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3714                             DAG.getVectorIdxConstant(StartIdx[Input], DL));
3715         }
3716       }
3717 
3718       // Calculate new mask.
3719       SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3720       for (int &Idx : MappedOps) {
3721         if (Idx >= (int)SrcNumElts)
3722           Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3723         else if (Idx >= 0)
3724           Idx -= StartIdx[0];
3725       }
3726 
3727       setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3728       return;
3729     }
3730   }
3731 
3732   // We can't use either concat vectors or extract subvectors so fall back to
3733   // replacing the shuffle with extract and build vector.
3734   // to insert and build vector.
3735   EVT EltVT = VT.getVectorElementType();
3736   SmallVector<SDValue,8> Ops;
3737   for (int Idx : Mask) {
3738     SDValue Res;
3739 
3740     if (Idx < 0) {
3741       Res = DAG.getUNDEF(EltVT);
3742     } else {
3743       SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3744       if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3745 
3746       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3747                         DAG.getVectorIdxConstant(Idx, DL));
3748     }
3749 
3750     Ops.push_back(Res);
3751   }
3752 
3753   setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3754 }
3755 
3756 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3757   ArrayRef<unsigned> Indices = I.getIndices();
3758   const Value *Op0 = I.getOperand(0);
3759   const Value *Op1 = I.getOperand(1);
3760   Type *AggTy = I.getType();
3761   Type *ValTy = Op1->getType();
3762   bool IntoUndef = isa<UndefValue>(Op0);
3763   bool FromUndef = isa<UndefValue>(Op1);
3764 
3765   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3766 
3767   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3768   SmallVector<EVT, 4> AggValueVTs;
3769   ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3770   SmallVector<EVT, 4> ValValueVTs;
3771   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3772 
3773   unsigned NumAggValues = AggValueVTs.size();
3774   unsigned NumValValues = ValValueVTs.size();
3775   SmallVector<SDValue, 4> Values(NumAggValues);
3776 
3777   // Ignore an insertvalue that produces an empty object
3778   if (!NumAggValues) {
3779     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3780     return;
3781   }
3782 
3783   SDValue Agg = getValue(Op0);
3784   unsigned i = 0;
3785   // Copy the beginning value(s) from the original aggregate.
3786   for (; i != LinearIndex; ++i)
3787     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3788                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3789   // Copy values from the inserted value(s).
3790   if (NumValValues) {
3791     SDValue Val = getValue(Op1);
3792     for (; i != LinearIndex + NumValValues; ++i)
3793       Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3794                   SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3795   }
3796   // Copy remaining value(s) from the original aggregate.
3797   for (; i != NumAggValues; ++i)
3798     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3799                 SDValue(Agg.getNode(), Agg.getResNo() + i);
3800 
3801   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3802                            DAG.getVTList(AggValueVTs), Values));
3803 }
3804 
3805 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3806   ArrayRef<unsigned> Indices = I.getIndices();
3807   const Value *Op0 = I.getOperand(0);
3808   Type *AggTy = Op0->getType();
3809   Type *ValTy = I.getType();
3810   bool OutOfUndef = isa<UndefValue>(Op0);
3811 
3812   unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3813 
3814   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3815   SmallVector<EVT, 4> ValValueVTs;
3816   ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3817 
3818   unsigned NumValValues = ValValueVTs.size();
3819 
3820   // Ignore a extractvalue that produces an empty object
3821   if (!NumValValues) {
3822     setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3823     return;
3824   }
3825 
3826   SmallVector<SDValue, 4> Values(NumValValues);
3827 
3828   SDValue Agg = getValue(Op0);
3829   // Copy out the selected value(s).
3830   for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3831     Values[i - LinearIndex] =
3832       OutOfUndef ?
3833         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3834         SDValue(Agg.getNode(), Agg.getResNo() + i);
3835 
3836   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3837                            DAG.getVTList(ValValueVTs), Values));
3838 }
3839 
3840 void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3841   Value *Op0 = I.getOperand(0);
3842   // Note that the pointer operand may be a vector of pointers. Take the scalar
3843   // element which holds a pointer.
3844   unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3845   SDValue N = getValue(Op0);
3846   SDLoc dl = getCurSDLoc();
3847   auto &TLI = DAG.getTargetLoweringInfo();
3848 
3849   // Normalize Vector GEP - all scalar operands should be converted to the
3850   // splat vector.
3851   bool IsVectorGEP = I.getType()->isVectorTy();
3852   ElementCount VectorElementCount =
3853       IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3854                   : ElementCount::getFixed(0);
3855 
3856   if (IsVectorGEP && !N.getValueType().isVector()) {
3857     LLVMContext &Context = *DAG.getContext();
3858     EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3859     if (VectorElementCount.isScalable())
3860       N = DAG.getSplatVector(VT, dl, N);
3861     else
3862       N = DAG.getSplatBuildVector(VT, dl, N);
3863   }
3864 
3865   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3866        GTI != E; ++GTI) {
3867     const Value *Idx = GTI.getOperand();
3868     if (StructType *StTy = GTI.getStructTypeOrNull()) {
3869       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3870       if (Field) {
3871         // N = N + Offset
3872         uint64_t Offset =
3873             DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
3874 
3875         // In an inbounds GEP with an offset that is nonnegative even when
3876         // interpreted as signed, assume there is no unsigned overflow.
3877         SDNodeFlags Flags;
3878         if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3879           Flags.setNoUnsignedWrap(true);
3880 
3881         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3882                         DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3883       }
3884     } else {
3885       // IdxSize is the width of the arithmetic according to IR semantics.
3886       // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
3887       // (and fix up the result later).
3888       unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3889       MVT IdxTy = MVT::getIntegerVT(IdxSize);
3890       TypeSize ElementSize =
3891           DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
3892       // We intentionally mask away the high bits here; ElementSize may not
3893       // fit in IdxTy.
3894       APInt ElementMul(IdxSize, ElementSize.getKnownMinSize());
3895       bool ElementScalable = ElementSize.isScalable();
3896 
3897       // If this is a scalar constant or a splat vector of constants,
3898       // handle it quickly.
3899       const auto *C = dyn_cast<Constant>(Idx);
3900       if (C && isa<VectorType>(C->getType()))
3901         C = C->getSplatValue();
3902 
3903       const auto *CI = dyn_cast_or_null<ConstantInt>(C);
3904       if (CI && CI->isZero())
3905         continue;
3906       if (CI && !ElementScalable) {
3907         APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
3908         LLVMContext &Context = *DAG.getContext();
3909         SDValue OffsVal;
3910         if (IsVectorGEP)
3911           OffsVal = DAG.getConstant(
3912               Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
3913         else
3914           OffsVal = DAG.getConstant(Offs, dl, IdxTy);
3915 
3916         // In an inbounds GEP with an offset that is nonnegative even when
3917         // interpreted as signed, assume there is no unsigned overflow.
3918         SDNodeFlags Flags;
3919         if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3920           Flags.setNoUnsignedWrap(true);
3921 
3922         OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3923 
3924         N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3925         continue;
3926       }
3927 
3928       // N = N + Idx * ElementMul;
3929       SDValue IdxN = getValue(Idx);
3930 
3931       if (!IdxN.getValueType().isVector() && IsVectorGEP) {
3932         EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
3933                                   VectorElementCount);
3934         if (VectorElementCount.isScalable())
3935           IdxN = DAG.getSplatVector(VT, dl, IdxN);
3936         else
3937           IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3938       }
3939 
3940       // If the index is smaller or larger than intptr_t, truncate or extend
3941       // it.
3942       IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3943 
3944       if (ElementScalable) {
3945         EVT VScaleTy = N.getValueType().getScalarType();
3946         SDValue VScale = DAG.getNode(
3947             ISD::VSCALE, dl, VScaleTy,
3948             DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
3949         if (IsVectorGEP)
3950           VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
3951         IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
3952       } else {
3953         // If this is a multiply by a power of two, turn it into a shl
3954         // immediately.  This is a very common case.
3955         if (ElementMul != 1) {
3956           if (ElementMul.isPowerOf2()) {
3957             unsigned Amt = ElementMul.logBase2();
3958             IdxN = DAG.getNode(ISD::SHL, dl,
3959                                N.getValueType(), IdxN,
3960                                DAG.getConstant(Amt, dl, IdxN.getValueType()));
3961           } else {
3962             SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
3963                                             IdxN.getValueType());
3964             IdxN = DAG.getNode(ISD::MUL, dl,
3965                                N.getValueType(), IdxN, Scale);
3966           }
3967         }
3968       }
3969 
3970       N = DAG.getNode(ISD::ADD, dl,
3971                       N.getValueType(), N, IdxN);
3972     }
3973   }
3974 
3975   MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
3976   MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
3977   if (IsVectorGEP) {
3978     PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
3979     PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
3980   }
3981 
3982   if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
3983     N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
3984 
3985   setValue(&I, N);
3986 }
3987 
3988 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3989   // If this is a fixed sized alloca in the entry block of the function,
3990   // allocate it statically on the stack.
3991   if (FuncInfo.StaticAllocaMap.count(&I))
3992     return;   // getValue will auto-populate this.
3993 
3994   SDLoc dl = getCurSDLoc();
3995   Type *Ty = I.getAllocatedType();
3996   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3997   auto &DL = DAG.getDataLayout();
3998   TypeSize TySize = DL.getTypeAllocSize(Ty);
3999   MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4000 
4001   SDValue AllocSize = getValue(I.getArraySize());
4002 
4003   EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
4004   if (AllocSize.getValueType() != IntPtr)
4005     AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4006 
4007   if (TySize.isScalable())
4008     AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4009                             DAG.getVScale(dl, IntPtr,
4010                                           APInt(IntPtr.getScalarSizeInBits(),
4011                                                 TySize.getKnownMinValue())));
4012   else
4013     AllocSize =
4014         DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4015                     DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));
4016 
4017   // Handle alignment.  If the requested alignment is less than or equal to
4018   // the stack alignment, ignore it.  If the size is greater than or equal to
4019   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4020   Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4021   if (*Alignment <= StackAlign)
4022     Alignment = None;
4023 
4024   const uint64_t StackAlignMask = StackAlign.value() - 1U;
4025   // Round the size of the allocation up to the stack alignment size
4026   // by add SA-1 to the size. This doesn't overflow because we're computing
4027   // an address inside an alloca.
4028   SDNodeFlags Flags;
4029   Flags.setNoUnsignedWrap(true);
4030   AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4031                           DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4032 
4033   // Mask out the low bits for alignment purposes.
4034   AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4035                           DAG.getConstant(~StackAlignMask, dl, IntPtr));
4036 
4037   SDValue Ops[] = {
4038       getRoot(), AllocSize,
4039       DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4040   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4041   SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4042   setValue(&I, DSA);
4043   DAG.setRoot(DSA.getValue(1));
4044 
4045   assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4046 }
4047 
4048 void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4049   if (I.isAtomic())
4050     return visitAtomicLoad(I);
4051 
4052   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4053   const Value *SV = I.getOperand(0);
4054   if (TLI.supportSwiftError()) {
4055     // Swifterror values can come from either a function parameter with
4056     // swifterror attribute or an alloca with swifterror attribute.
4057     if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4058       if (Arg->hasSwiftErrorAttr())
4059         return visitLoadFromSwiftError(I);
4060     }
4061 
4062     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4063       if (Alloca->isSwiftError())
4064         return visitLoadFromSwiftError(I);
4065     }
4066   }
4067 
4068   SDValue Ptr = getValue(SV);
4069 
4070   Type *Ty = I.getType();
4071   Align Alignment = I.getAlign();
4072 
4073   AAMDNodes AAInfo = I.getAAMetadata();
4074   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4075 
4076   SmallVector<EVT, 4> ValueVTs, MemVTs;
4077   SmallVector<uint64_t, 4> Offsets;
4078   ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4079   unsigned NumValues = ValueVTs.size();
4080   if (NumValues == 0)
4081     return;
4082 
4083   bool isVolatile = I.isVolatile();
4084   MachineMemOperand::Flags MMOFlags =
4085       TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
4086 
4087   SDValue Root;
4088   bool ConstantMemory = false;
4089   if (isVolatile)
4090     // Serialize volatile loads with other side effects.
4091     Root = getRoot();
4092   else if (NumValues > MaxParallelChains)
4093     Root = getMemoryRoot();
4094   else if (AA &&
4095            AA->pointsToConstantMemory(MemoryLocation(
4096                SV,
4097                LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4098                AAInfo))) {
4099     // Do not serialize (non-volatile) loads of constant memory with anything.
4100     Root = DAG.getEntryNode();
4101     ConstantMemory = true;
4102     MMOFlags |= MachineMemOperand::MOInvariant;
4103 
4104     // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
4105     // but the previous usage implied it did. Probably should check
4106     // isDereferenceableAndAlignedPointer.
4107     MMOFlags |= MachineMemOperand::MODereferenceable;
4108   } else {
4109     // Do not serialize non-volatile loads against each other.
4110     Root = DAG.getRoot();
4111   }
4112 
4113   SDLoc dl = getCurSDLoc();
4114 
4115   if (isVolatile)
4116     Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4117 
4118   // An aggregate load cannot wrap around the address space, so offsets to its
4119   // parts don't wrap either.
4120   SDNodeFlags Flags;
4121   Flags.setNoUnsignedWrap(true);
4122 
4123   SmallVector<SDValue, 4> Values(NumValues);
4124   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4125   EVT PtrVT = Ptr.getValueType();
4126 
4127   unsigned ChainI = 0;
4128   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4129     // Serializing loads here may result in excessive register pressure, and
4130     // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4131     // could recover a bit by hoisting nodes upward in the chain by recognizing
4132     // they are side-effect free or do not alias. The optimizer should really
4133     // avoid this case by converting large object/array copies to llvm.memcpy
4134     // (MaxParallelChains should always remain as failsafe).
4135     if (ChainI == MaxParallelChains) {
4136       assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4137       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4138                                   makeArrayRef(Chains.data(), ChainI));
4139       Root = Chain;
4140       ChainI = 0;
4141     }
4142     SDValue A = DAG.getNode(ISD::ADD, dl,
4143                             PtrVT, Ptr,
4144                             DAG.getConstant(Offsets[i], dl, PtrVT),
4145                             Flags);
4146 
4147     SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4148                             MachinePointerInfo(SV, Offsets[i]), Alignment,
4149                             MMOFlags, AAInfo, Ranges);
4150     Chains[ChainI] = L.getValue(1);
4151 
4152     if (MemVTs[i] != ValueVTs[i])
4153       L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4154 
4155     Values[i] = L;
4156   }
4157 
4158   if (!ConstantMemory) {
4159     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4160                                 makeArrayRef(Chains.data(), ChainI));
4161     if (isVolatile)
4162       DAG.setRoot(Chain);
4163     else
4164       PendingLoads.push_back(Chain);
4165   }
4166 
4167   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4168                            DAG.getVTList(ValueVTs), Values));
4169 }
4170 
4171 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4172   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4173          "call visitStoreToSwiftError when backend supports swifterror");
4174 
4175   SmallVector<EVT, 4> ValueVTs;
4176   SmallVector<uint64_t, 4> Offsets;
4177   const Value *SrcV = I.getOperand(0);
4178   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4179                   SrcV->getType(), ValueVTs, &Offsets);
4180   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4181          "expect a single EVT for swifterror");
4182 
4183   SDValue Src = getValue(SrcV);
4184   // Create a virtual register, then update the virtual register.
4185   Register VReg =
4186       SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4187   // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4188   // Chain can be getRoot or getControlRoot.
4189   SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4190                                       SDValue(Src.getNode(), Src.getResNo()));
4191   DAG.setRoot(CopyNode);
4192 }
4193 
4194 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4195   assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4196          "call visitLoadFromSwiftError when backend supports swifterror");
4197 
4198   assert(!I.isVolatile() &&
4199          !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4200          !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4201          "Support volatile, non temporal, invariant for load_from_swift_error");
4202 
4203   const Value *SV = I.getOperand(0);
4204   Type *Ty = I.getType();
4205   assert(
4206       (!AA ||
4207        !AA->pointsToConstantMemory(MemoryLocation(
4208            SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4209            I.getAAMetadata()))) &&
4210       "load_from_swift_error should not be constant memory");
4211 
4212   SmallVector<EVT, 4> ValueVTs;
4213   SmallVector<uint64_t, 4> Offsets;
4214   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
4215                   ValueVTs, &Offsets);
4216   assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4217          "expect a single EVT for swifterror");
4218 
4219   // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4220   SDValue L = DAG.getCopyFromReg(
4221       getRoot(), getCurSDLoc(),
4222       SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4223 
4224   setValue(&I, L);
4225 }
4226 
4227 void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4228   if (I.isAtomic())
4229     return visitAtomicStore(I);
4230 
4231   const Value *SrcV = I.getOperand(0);
4232   const Value *PtrV = I.getOperand(1);
4233 
4234   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4235   if (TLI.supportSwiftError()) {
4236     // Swifterror values can come from either a function parameter with
4237     // swifterror attribute or an alloca with swifterror attribute.
4238     if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4239       if (Arg->hasSwiftErrorAttr())
4240         return visitStoreToSwiftError(I);
4241     }
4242 
4243     if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4244       if (Alloca->isSwiftError())
4245         return visitStoreToSwiftError(I);
4246     }
4247   }
4248 
4249   SmallVector<EVT, 4> ValueVTs, MemVTs;
4250   SmallVector<uint64_t, 4> Offsets;
4251   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4252                   SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4253   unsigned NumValues = ValueVTs.size();
4254   if (NumValues == 0)
4255     return;
4256 
4257   // Get the lowered operands. Note that we do this after
4258   // checking if NumResults is zero, because with zero results
4259   // the operands won't have values in the map.
4260   SDValue Src = getValue(SrcV);
4261   SDValue Ptr = getValue(PtrV);
4262 
4263   SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4264   SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4265   SDLoc dl = getCurSDLoc();
4266   Align Alignment = I.getAlign();
4267   AAMDNodes AAInfo = I.getAAMetadata();
4268 
4269   auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4270 
4271   // An aggregate load cannot wrap around the address space, so offsets to its
4272   // parts don't wrap either.
4273   SDNodeFlags Flags;
4274   Flags.setNoUnsignedWrap(true);
4275 
4276   unsigned ChainI = 0;
4277   for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4278     // See visitLoad comments.
4279     if (ChainI == MaxParallelChains) {
4280       SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4281                                   makeArrayRef(Chains.data(), ChainI));
4282       Root = Chain;
4283       ChainI = 0;
4284     }
4285     SDValue Add =
4286         DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);
4287     SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4288     if (MemVTs[i] != ValueVTs[i])
4289       Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4290     SDValue St =
4291         DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4292                      Alignment, MMOFlags, AAInfo);
4293     Chains[ChainI] = St;
4294   }
4295 
4296   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4297                                   makeArrayRef(Chains.data(), ChainI));
4298   DAG.setRoot(StoreNode);
4299 }
4300 
4301 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4302                                            bool IsCompressing) {
4303   SDLoc sdl = getCurSDLoc();
4304 
4305   auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4306                                MaybeAlign &Alignment) {
4307     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4308     Src0 = I.getArgOperand(0);
4309     Ptr = I.getArgOperand(1);
4310     Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
4311     Mask = I.getArgOperand(3);
4312   };
4313   auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4314                                     MaybeAlign &Alignment) {
4315     // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4316     Src0 = I.getArgOperand(0);
4317     Ptr = I.getArgOperand(1);
4318     Mask = I.getArgOperand(2);
4319     Alignment = None;
4320   };
4321 
4322   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4323   MaybeAlign Alignment;
4324   if (IsCompressing)
4325     getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4326   else
4327     getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4328 
4329   SDValue Ptr = getValue(PtrOperand);
4330   SDValue Src0 = getValue(Src0Operand);
4331   SDValue Mask = getValue(MaskOperand);
4332   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4333 
4334   EVT VT = Src0.getValueType();
4335   if (!Alignment)
4336     Alignment = DAG.getEVTAlign(VT);
4337 
4338   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4339       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
4340       MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata());
4341   SDValue StoreNode =
4342       DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4343                          ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4344   DAG.setRoot(StoreNode);
4345   setValue(&I, StoreNode);
4346 }
4347 
4348 // Get a uniform base for the Gather/Scatter intrinsic.
4349 // The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4350 // We try to represent it as a base pointer + vector of indices.
4351 // Usually, the vector of pointers comes from a 'getelementptr' instruction.
4352 // The first operand of the GEP may be a single pointer or a vector of pointers
4353 // Example:
4354 //   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4355 //  or
4356 //   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
4357 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4358 //
4359 // When the first GEP operand is a single pointer - it is the uniform base we
4360 // are looking for. If first operand of the GEP is a splat vector - we
4361 // extract the splat value and use it as a uniform base.
4362 // In all other cases the function returns 'false'.
4363 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4364                            ISD::MemIndexType &IndexType, SDValue &Scale,
4365                            SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4366                            uint64_t ElemSize) {
4367   SelectionDAG& DAG = SDB->DAG;
4368   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4369   const DataLayout &DL = DAG.getDataLayout();
4370 
4371   assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4372 
4373   // Handle splat constant pointer.
4374   if (auto *C = dyn_cast<Constant>(Ptr)) {
4375     C = C->getSplatValue();
4376     if (!C)
4377       return false;
4378 
4379     Base = SDB->getValue(C);
4380 
4381     ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4382     EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4383     Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4384     IndexType = ISD::SIGNED_SCALED;
4385     Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4386     return true;
4387   }
4388 
4389   const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4390   if (!GEP || GEP->getParent() != CurBB)
4391     return false;
4392 
4393   if (GEP->getNumOperands() != 2)
4394     return false;
4395 
4396   const Value *BasePtr = GEP->getPointerOperand();
4397   const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4398 
4399   // Make sure the base is scalar and the index is a vector.
4400   if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4401     return false;
4402 
4403   Base = SDB->getValue(BasePtr);
4404   Index = SDB->getValue(IndexVal);
4405   IndexType = ISD::SIGNED_SCALED;
4406 
4407   // MGATHER/MSCATTER are only required to support scaling by one or by the
4408   // element size. Other scales may be produced using target-specific DAG
4409   // combines.
4410   uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4411   if (ScaleVal != ElemSize && ScaleVal != 1)
4412     return false;
4413 
4414   Scale =
4415       DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4416   return true;
4417 }
4418 
4419 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4420   SDLoc sdl = getCurSDLoc();
4421 
4422   // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4423   const Value *Ptr = I.getArgOperand(1);
4424   SDValue Src0 = getValue(I.getArgOperand(0));
4425   SDValue Mask = getValue(I.getArgOperand(3));
4426   EVT VT = Src0.getValueType();
4427   Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4428                         ->getMaybeAlignValue()
4429                         .value_or(DAG.getEVTAlign(VT.getScalarType()));
4430   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4431 
4432   SDValue Base;
4433   SDValue Index;
4434   ISD::MemIndexType IndexType;
4435   SDValue Scale;
4436   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4437                                     I.getParent(), VT.getScalarStoreSize());
4438 
4439   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4440   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4441       MachinePointerInfo(AS), MachineMemOperand::MOStore,
4442       // TODO: Make MachineMemOperands aware of scalable
4443       // vectors.
4444       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
4445   if (!UniformBase) {
4446     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4447     Index = getValue(Ptr);
4448     IndexType = ISD::SIGNED_SCALED;
4449     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4450   }
4451 
4452   EVT IdxVT = Index.getValueType();
4453   EVT EltTy = IdxVT.getVectorElementType();
4454   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4455     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4456     Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4457   }
4458 
4459   SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4460   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4461                                          Ops, MMO, IndexType, false);
4462   DAG.setRoot(Scatter);
4463   setValue(&I, Scatter);
4464 }
4465 
4466 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4467   SDLoc sdl = getCurSDLoc();
4468 
4469   auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4470                               MaybeAlign &Alignment) {
4471     // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4472     Ptr = I.getArgOperand(0);
4473     Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
4474     Mask = I.getArgOperand(2);
4475     Src0 = I.getArgOperand(3);
4476   };
4477   auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4478                                  MaybeAlign &Alignment) {
4479     // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4480     Ptr = I.getArgOperand(0);
4481     Alignment = None;
4482     Mask = I.getArgOperand(1);
4483     Src0 = I.getArgOperand(2);
4484   };
4485 
4486   Value  *PtrOperand, *MaskOperand, *Src0Operand;
4487   MaybeAlign Alignment;
4488   if (IsExpanding)
4489     getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4490   else
4491     getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4492 
4493   SDValue Ptr = getValue(PtrOperand);
4494   SDValue Src0 = getValue(Src0Operand);
4495   SDValue Mask = getValue(MaskOperand);
4496   SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4497 
4498   EVT VT = Src0.getValueType();
4499   if (!Alignment)
4500     Alignment = DAG.getEVTAlign(VT);
4501 
4502   AAMDNodes AAInfo = I.getAAMetadata();
4503   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4504 
4505   // Do not serialize masked loads of constant memory with anything.
4506   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4507   bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4508 
4509   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4510 
4511   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4512       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
4513       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
4514 
4515   SDValue Load =
4516       DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4517                         ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4518   if (AddToChain)
4519     PendingLoads.push_back(Load.getValue(1));
4520   setValue(&I, Load);
4521 }
4522 
4523 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4524   SDLoc sdl = getCurSDLoc();
4525 
4526   // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4527   const Value *Ptr = I.getArgOperand(0);
4528   SDValue Src0 = getValue(I.getArgOperand(3));
4529   SDValue Mask = getValue(I.getArgOperand(2));
4530 
4531   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4532   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4533   Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4534                         ->getMaybeAlignValue()
4535                         .value_or(DAG.getEVTAlign(VT.getScalarType()));
4536 
4537   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4538 
4539   SDValue Root = DAG.getRoot();
4540   SDValue Base;
4541   SDValue Index;
4542   ISD::MemIndexType IndexType;
4543   SDValue Scale;
4544   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4545                                     I.getParent(), VT.getScalarStoreSize());
4546   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4547   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4548       MachinePointerInfo(AS), MachineMemOperand::MOLoad,
4549       // TODO: Make MachineMemOperands aware of scalable
4550       // vectors.
4551       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
4552 
4553   if (!UniformBase) {
4554     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4555     Index = getValue(Ptr);
4556     IndexType = ISD::SIGNED_SCALED;
4557     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4558   }
4559 
4560   EVT IdxVT = Index.getValueType();
4561   EVT EltTy = IdxVT.getVectorElementType();
4562   if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4563     EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4564     Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4565   }
4566 
4567   SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4568   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4569                                        Ops, MMO, IndexType, ISD::NON_EXTLOAD);
4570 
4571   PendingLoads.push_back(Gather.getValue(1));
4572   setValue(&I, Gather);
4573 }
4574 
4575 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4576   SDLoc dl = getCurSDLoc();
4577   AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4578   AtomicOrdering FailureOrdering = I.getFailureOrdering();
4579   SyncScope::ID SSID = I.getSyncScopeID();
4580 
4581   SDValue InChain = getRoot();
4582 
4583   MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4584   SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4585 
4586   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4587   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4588 
4589   MachineFunction &MF = DAG.getMachineFunction();
4590   MachineMemOperand *MMO = MF.getMachineMemOperand(
4591       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4592       DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
4593       FailureOrdering);
4594 
4595   SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
4596                                    dl, MemVT, VTs, InChain,
4597                                    getValue(I.getPointerOperand()),
4598                                    getValue(I.getCompareOperand()),
4599                                    getValue(I.getNewValOperand()), MMO);
4600 
4601   SDValue OutChain = L.getValue(2);
4602 
4603   setValue(&I, L);
4604   DAG.setRoot(OutChain);
4605 }
4606 
4607 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4608   SDLoc dl = getCurSDLoc();
4609   ISD::NodeType NT;
4610   switch (I.getOperation()) {
4611   default: llvm_unreachable("Unknown atomicrmw operation");
4612   case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4613   case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
4614   case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
4615   case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
4616   case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4617   case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
4618   case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
4619   case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
4620   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
4621   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4622   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4623   case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
4624   case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
4625   case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
4626   case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
4627   }
4628   AtomicOrdering Ordering = I.getOrdering();
4629   SyncScope::ID SSID = I.getSyncScopeID();
4630 
4631   SDValue InChain = getRoot();
4632 
4633   auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4634   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4635   auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
4636 
4637   MachineFunction &MF = DAG.getMachineFunction();
4638   MachineMemOperand *MMO = MF.getMachineMemOperand(
4639       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4640       DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
4641 
4642   SDValue L =
4643     DAG.getAtomic(NT, dl, MemVT, InChain,
4644                   getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4645                   MMO);
4646 
4647   SDValue OutChain = L.getValue(1);
4648 
4649   setValue(&I, L);
4650   DAG.setRoot(OutChain);
4651 }
4652 
4653 void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4654   SDLoc dl = getCurSDLoc();
4655   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4656   SDValue Ops[3];
4657   Ops[0] = getRoot();
4658   Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
4659                                  TLI.getFenceOperandTy(DAG.getDataLayout()));
4660   Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
4661                                  TLI.getFenceOperandTy(DAG.getDataLayout()));
4662   DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4663 }
4664 
4665 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4666   SDLoc dl = getCurSDLoc();
4667   AtomicOrdering Order = I.getOrdering();
4668   SyncScope::ID SSID = I.getSyncScopeID();
4669 
4670   SDValue InChain = getRoot();
4671 
4672   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4673   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4674   EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4675 
4676   if (!TLI.supportsUnalignedAtomics() &&
4677       I.getAlign().value() < MemVT.getSizeInBits() / 8)
4678     report_fatal_error("Cannot generate unaligned atomic load");
4679 
4680   auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout());
4681 
4682   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4683       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4684       I.getAlign(), AAMDNodes(), nullptr, SSID, Order);
4685 
4686   InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4687 
4688   SDValue Ptr = getValue(I.getPointerOperand());
4689 
4690   if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {
4691     // TODO: Once this is better exercised by tests, it should be merged with
4692     // the normal path for loads to prevent future divergence.
4693     SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);
4694     if (MemVT != VT)
4695       L = DAG.getPtrExtOrTrunc(L, dl, VT);
4696 
4697     setValue(&I, L);
4698     SDValue OutChain = L.getValue(1);
4699     if (!I.isUnordered())
4700       DAG.setRoot(OutChain);
4701     else
4702       PendingLoads.push_back(OutChain);
4703     return;
4704   }
4705 
4706   SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4707                             Ptr, MMO);
4708 
4709   SDValue OutChain = L.getValue(1);
4710   if (MemVT != VT)
4711     L = DAG.getPtrExtOrTrunc(L, dl, VT);
4712 
4713   setValue(&I, L);
4714   DAG.setRoot(OutChain);
4715 }
4716 
4717 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4718   SDLoc dl = getCurSDLoc();
4719 
4720   AtomicOrdering Ordering = I.getOrdering();
4721   SyncScope::ID SSID = I.getSyncScopeID();
4722 
4723   SDValue InChain = getRoot();
4724 
4725   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4726   EVT MemVT =
4727       TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4728 
4729   if (I.getAlign().value() < MemVT.getSizeInBits() / 8)
4730     report_fatal_error("Cannot generate unaligned atomic store");
4731 
4732   auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4733 
4734   MachineFunction &MF = DAG.getMachineFunction();
4735   MachineMemOperand *MMO = MF.getMachineMemOperand(
4736       MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
4737       I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
4738 
4739   SDValue Val = getValue(I.getValueOperand());
4740   if (Val.getValueType() != MemVT)
4741     Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
4742   SDValue Ptr = getValue(I.getPointerOperand());
4743 
4744   if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {
4745     // TODO: Once this is better exercised by tests, it should be merged with
4746     // the normal path for stores to prevent future divergence.
4747     SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);
4748     DAG.setRoot(S);
4749     return;
4750   }
4751   SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4752                                    Ptr, Val, MMO);
4753 
4754 
4755   DAG.setRoot(OutChain);
4756 }
4757 
4758 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4759 /// node.
4760 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4761                                                unsigned Intrinsic) {
4762   // Ignore the callsite's attributes. A specific call site may be marked with
4763   // readnone, but the lowering code will expect the chain based on the
4764   // definition.
4765   const Function *F = I.getCalledFunction();
4766   bool HasChain = !F->doesNotAccessMemory();
4767   bool OnlyLoad = HasChain && F->onlyReadsMemory();
4768 
4769   // Build the operand list.
4770   SmallVector<SDValue, 8> Ops;
4771   if (HasChain) {  // If this intrinsic has side-effects, chainify it.
4772     if (OnlyLoad) {
4773       // We don't need to serialize loads against other loads.
4774       Ops.push_back(DAG.getRoot());
4775     } else {
4776       Ops.push_back(getRoot());
4777     }
4778   }
4779 
4780   // Info is set by getTgtMemIntrinsic
4781   TargetLowering::IntrinsicInfo Info;
4782   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4783   bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4784                                                DAG.getMachineFunction(),
4785                                                Intrinsic);
4786 
4787   // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4788   if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4789       Info.opc == ISD::INTRINSIC_W_CHAIN)
4790     Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4791                                         TLI.getPointerTy(DAG.getDataLayout())));
4792 
4793   // Add all operands of the call to the operand list.
4794   for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
4795     const Value *Arg = I.getArgOperand(i);
4796     if (!I.paramHasAttr(i, Attribute::ImmArg)) {
4797       Ops.push_back(getValue(Arg));
4798       continue;
4799     }
4800 
4801     // Use TargetConstant instead of a regular constant for immarg.
4802     EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
4803     if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
4804       assert(CI->getBitWidth() <= 64 &&
4805              "large intrinsic immediates not handled");
4806       Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
4807     } else {
4808       Ops.push_back(
4809           DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
4810     }
4811   }
4812 
4813   SmallVector<EVT, 4> ValueVTs;
4814   ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4815 
4816   if (HasChain)
4817     ValueVTs.push_back(MVT::Other);
4818 
4819   SDVTList VTs = DAG.getVTList(ValueVTs);
4820 
4821   // Propagate fast-math-flags from IR to node(s).
4822   SDNodeFlags Flags;
4823   if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
4824     Flags.copyFMF(*FPMO);
4825   SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
4826 
4827   // Create the node.
4828   SDValue Result;
4829   if (IsTgtIntrinsic) {
4830     // This is target intrinsic that touches memory
4831     Result =
4832         DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
4833                                 MachinePointerInfo(Info.ptrVal, Info.offset),
4834                                 Info.align, Info.flags, Info.size,
4835                                 I.getAAMetadata());
4836   } else if (!HasChain) {
4837     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4838   } else if (!I.getType()->isVoidTy()) {
4839     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4840   } else {
4841     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4842   }
4843 
4844   if (HasChain) {
4845     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4846     if (OnlyLoad)
4847       PendingLoads.push_back(Chain);
4848     else
4849       DAG.setRoot(Chain);
4850   }
4851 
4852   if (!I.getType()->isVoidTy()) {
4853     if (!isa<VectorType>(I.getType()))
4854       Result = lowerRangeToAssertZExt(DAG, I, Result);
4855 
4856     MaybeAlign Alignment = I.getRetAlign();
4857     if (!Alignment)
4858       Alignment = F->getAttributes().getRetAlignment();
4859     // Insert `assertalign` node if there's an alignment.
4860     if (InsertAssertAlign && Alignment) {
4861       Result =
4862           DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
4863     }
4864 
4865     setValue(&I, Result);
4866   }
4867 }
4868 
4869 /// GetSignificand - Get the significand and build it into a floating-point
4870 /// number with exponent of 1:
4871 ///
4872 ///   Op = (Op & 0x007fffff) | 0x3f800000;
4873 ///
4874 /// where Op is the hexadecimal representation of floating point value.
4875 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4876   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4877                            DAG.getConstant(0x007fffff, dl, MVT::i32));
4878   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4879                            DAG.getConstant(0x3f800000, dl, MVT::i32));
4880   return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4881 }
4882 
4883 /// GetExponent - Get the exponent:
4884 ///
4885 ///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4886 ///
4887 /// where Op is the hexadecimal representation of floating point value.
4888 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4889                            const TargetLowering &TLI, const SDLoc &dl) {
4890   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4891                            DAG.getConstant(0x7f800000, dl, MVT::i32));
4892   SDValue t1 = DAG.getNode(
4893       ISD::SRL, dl, MVT::i32, t0,
4894       DAG.getConstant(23, dl,
4895                       TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
4896   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4897                            DAG.getConstant(127, dl, MVT::i32));
4898   return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4899 }
4900 
4901 /// getF32Constant - Get 32-bit floating point constant.
4902 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4903                               const SDLoc &dl) {
4904   return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4905                            MVT::f32);
4906 }
4907 
4908 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4909                                        SelectionDAG &DAG) {
4910   // TODO: What fast-math-flags should be set on the floating-point nodes?
4911 
4912   //   IntegerPartOfX = ((int32_t)(t0);
4913   SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4914 
4915   //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
4916   SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4917   SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4918 
4919   //   IntegerPartOfX <<= 23;
4920   IntegerPartOfX =
4921       DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4922                   DAG.getConstant(23, dl,
4923                                   DAG.getTargetLoweringInfo().getShiftAmountTy(
4924                                       MVT::i32, DAG.getDataLayout())));
4925 
4926   SDValue TwoToFractionalPartOfX;
4927   if (LimitFloatPrecision <= 6) {
4928     // For floating-point precision of 6:
4929     //
4930     //   TwoToFractionalPartOfX =
4931     //     0.997535578f +
4932     //       (0.735607626f + 0.252464424f * x) * x;
4933     //
4934     // error 0.0144103317, which is 6 bits
4935     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4936                              getF32Constant(DAG, 0x3e814304, dl));
4937     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4938                              getF32Constant(DAG, 0x3f3c50c8, dl));
4939     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4940     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4941                                          getF32Constant(DAG, 0x3f7f5e7e, dl));
4942   } else if (LimitFloatPrecision <= 12) {
4943     // For floating-point precision of 12:
4944     //
4945     //   TwoToFractionalPartOfX =
4946     //     0.999892986f +
4947     //       (0.696457318f +
4948     //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
4949     //
4950     // error 0.000107046256, which is 13 to 14 bits
4951     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4952                              getF32Constant(DAG, 0x3da235e3, dl));
4953     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4954                              getF32Constant(DAG, 0x3e65b8f3, dl));
4955     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4956     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4957                              getF32Constant(DAG, 0x3f324b07, dl));
4958     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4959     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4960                                          getF32Constant(DAG, 0x3f7ff8fd, dl));
4961   } else { // LimitFloatPrecision <= 18
4962     // For floating-point precision of 18:
4963     //
4964     //   TwoToFractionalPartOfX =
4965     //     0.999999982f +
4966     //       (0.693148872f +
4967     //         (0.240227044f +
4968     //           (0.554906021e-1f +
4969     //             (0.961591928e-2f +
4970     //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4971     // error 2.47208000*10^(-7), which is better than 18 bits
4972     SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4973                              getF32Constant(DAG, 0x3924b03e, dl));
4974     SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4975                              getF32Constant(DAG, 0x3ab24b87, dl));
4976     SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4977     SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4978                              getF32Constant(DAG, 0x3c1d8c17, dl));
4979     SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4980     SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4981                              getF32Constant(DAG, 0x3d634a1d, dl));
4982     SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4983     SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4984                              getF32Constant(DAG, 0x3e75fe14, dl));
4985     SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4986     SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4987                               getF32Constant(DAG, 0x3f317234, dl));
4988     SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4989     TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4990                                          getF32Constant(DAG, 0x3f800000, dl));
4991   }
4992 
4993   // Add the exponent into the result in integer domain.
4994   SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4995   return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4996                      DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4997 }
4998 
4999 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
5000 /// limited-precision mode.
5001 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5002                          const TargetLowering &TLI, SDNodeFlags Flags) {
5003   if (Op.getValueType() == MVT::f32 &&
5004       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5005 
5006     // Put the exponent in the right bit position for later addition to the
5007     // final result:
5008     //
5009     // t0 = Op * log2(e)
5010 
5011     // TODO: What fast-math-flags should be set here?
5012     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5013                              DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5014     return getLimitedPrecisionExp2(t0, dl, DAG);
5015   }
5016 
5017   // No special expansion.
5018   return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5019 }
5020 
5021 /// expandLog - Lower a log intrinsic. Handles the special sequences for
5022 /// limited-precision mode.
5023 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5024                          const TargetLowering &TLI, SDNodeFlags Flags) {
5025   // TODO: What fast-math-flags should be set on the floating-point nodes?
5026 
5027   if (Op.getValueType() == MVT::f32 &&
5028       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5029     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5030 
5031     // Scale the exponent by log(2).
5032     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5033     SDValue LogOfExponent =
5034         DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5035                     DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5036 
5037     // Get the significand and build it into a floating-point number with
5038     // exponent of 1.
5039     SDValue X = GetSignificand(DAG, Op1, dl);
5040 
5041     SDValue LogOfMantissa;
5042     if (LimitFloatPrecision <= 6) {
5043       // For floating-point precision of 6:
5044       //
5045       //   LogofMantissa =
5046       //     -1.1609546f +
5047       //       (1.4034025f - 0.23903021f * x) * x;
5048       //
5049       // error 0.0034276066, which is better than 8 bits
5050       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5051                                getF32Constant(DAG, 0xbe74c456, dl));
5052       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5053                                getF32Constant(DAG, 0x3fb3a2b1, dl));
5054       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5055       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5056                                   getF32Constant(DAG, 0x3f949a29, dl));
5057     } else if (LimitFloatPrecision <= 12) {
5058       // For floating-point precision of 12:
5059       //
5060       //   LogOfMantissa =
5061       //     -1.7417939f +
5062       //       (2.8212026f +
5063       //         (-1.4699568f +
5064       //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5065       //
5066       // error 0.000061011436, which is 14 bits
5067       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5068                                getF32Constant(DAG, 0xbd67b6d6, dl));
5069       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5070                                getF32Constant(DAG, 0x3ee4f4b8, dl));
5071       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5072       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5073                                getF32Constant(DAG, 0x3fbc278b, dl));
5074       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5075       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5076                                getF32Constant(DAG, 0x40348e95, dl));
5077       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5078       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5079                                   getF32Constant(DAG, 0x3fdef31a, dl));
5080     } else { // LimitFloatPrecision <= 18
5081       // For floating-point precision of 18:
5082       //
5083       //   LogOfMantissa =
5084       //     -2.1072184f +
5085       //       (4.2372794f +
5086       //         (-3.7029485f +
5087       //           (2.2781945f +
5088       //             (-0.87823314f +
5089       //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5090       //
5091       // error 0.0000023660568, which is better than 18 bits
5092       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5093                                getF32Constant(DAG, 0xbc91e5ac, dl));
5094       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5095                                getF32Constant(DAG, 0x3e4350aa, dl));
5096       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5097       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5098                                getF32Constant(DAG, 0x3f60d3e3, dl));
5099       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5100       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5101                                getF32Constant(DAG, 0x4011cdf0, dl));
5102       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5103       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5104                                getF32Constant(DAG, 0x406cfd1c, dl));
5105       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5106       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5107                                getF32Constant(DAG, 0x408797cb, dl));
5108       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5109       LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5110                                   getF32Constant(DAG, 0x4006dcab, dl));
5111     }
5112 
5113     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5114   }
5115 
5116   // No special expansion.
5117   return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5118 }
5119 
5120 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5121 /// limited-precision mode.
5122 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5123                           const TargetLowering &TLI, SDNodeFlags Flags) {
5124   // TODO: What fast-math-flags should be set on the floating-point nodes?
5125 
5126   if (Op.getValueType() == MVT::f32 &&
5127       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5128     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5129 
5130     // Get the exponent.
5131     SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5132 
5133     // Get the significand and build it into a floating-point number with
5134     // exponent of 1.
5135     SDValue X = GetSignificand(DAG, Op1, dl);
5136 
5137     // Different possible minimax approximations of significand in
5138     // floating-point for various degrees of accuracy over [1,2].
5139     SDValue Log2ofMantissa;
5140     if (LimitFloatPrecision <= 6) {
5141       // For floating-point precision of 6:
5142       //
5143       //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5144       //
5145       // error 0.0049451742, which is more than 7 bits
5146       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5147                                getF32Constant(DAG, 0xbeb08fe0, dl));
5148       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5149                                getF32Constant(DAG, 0x40019463, dl));
5150       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5151       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5152                                    getF32Constant(DAG, 0x3fd6633d, dl));
5153     } else if (LimitFloatPrecision <= 12) {
5154       // For floating-point precision of 12:
5155       //
5156       //   Log2ofMantissa =
5157       //     -2.51285454f +
5158       //       (4.07009056f +
5159       //         (-2.12067489f +
5160       //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5161       //
5162       // error 0.0000876136000, which is better than 13 bits
5163       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5164                                getF32Constant(DAG, 0xbda7262e, dl));
5165       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5166                                getF32Constant(DAG, 0x3f25280b, dl));
5167       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5168       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5169                                getF32Constant(DAG, 0x4007b923, dl));
5170       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5171       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5172                                getF32Constant(DAG, 0x40823e2f, dl));
5173       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5174       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5175                                    getF32Constant(DAG, 0x4020d29c, dl));
5176     } else { // LimitFloatPrecision <= 18
5177       // For floating-point precision of 18:
5178       //
5179       //   Log2ofMantissa =
5180       //     -3.0400495f +
5181       //       (6.1129976f +
5182       //         (-5.3420409f +
5183       //           (3.2865683f +
5184       //             (-1.2669343f +
5185       //               (0.27515199f -
5186       //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5187       //
5188       // error 0.0000018516, which is better than 18 bits
5189       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5190                                getF32Constant(DAG, 0xbcd2769e, dl));
5191       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5192                                getF32Constant(DAG, 0x3e8ce0b9, dl));
5193       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5194       SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5195                                getF32Constant(DAG, 0x3fa22ae7, dl));
5196       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5197       SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5198                                getF32Constant(DAG, 0x40525723, dl));
5199       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5200       SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5201                                getF32Constant(DAG, 0x40aaf200, dl));
5202       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5203       SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5204                                getF32Constant(DAG, 0x40c39dad, dl));
5205       SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5206       Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5207                                    getF32Constant(DAG, 0x4042902c, dl));
5208     }
5209 
5210     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5211   }
5212 
5213   // No special expansion.
5214   return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5215 }
5216 
5217 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5218 /// limited-precision mode.
5219 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5220                            const TargetLowering &TLI, SDNodeFlags Flags) {
5221   // TODO: What fast-math-flags should be set on the floating-point nodes?
5222 
5223   if (Op.getValueType() == MVT::f32 &&
5224       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5225     SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5226 
5227     // Scale the exponent by log10(2) [0.30102999f].
5228     SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5229     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5230                                         getF32Constant(DAG, 0x3e9a209a, dl));
5231 
5232     // Get the significand and build it into a floating-point number with
5233     // exponent of 1.
5234     SDValue X = GetSignificand(DAG, Op1, dl);
5235 
5236     SDValue Log10ofMantissa;
5237     if (LimitFloatPrecision <= 6) {
5238       // For floating-point precision of 6:
5239       //
5240       //   Log10ofMantissa =
5241       //     -0.50419619f +
5242       //       (0.60948995f - 0.10380950f * x) * x;
5243       //
5244       // error 0.0014886165, which is 6 bits
5245       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5246                                getF32Constant(DAG, 0xbdd49a13, dl));
5247       SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5248                                getF32Constant(DAG, 0x3f1c0789, dl));
5249       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5250       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5251                                     getF32Constant(DAG, 0x3f011300, dl));
5252     } else if (LimitFloatPrecision <= 12) {
5253       // For floating-point precision of 12:
5254       //
5255       //   Log10ofMantissa =
5256       //     -0.64831180f +
5257       //       (0.91751397f +
5258       //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5259       //
5260       // error 0.00019228036, which is better than 12 bits
5261       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5262                                getF32Constant(DAG, 0x3d431f31, dl));
5263       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5264                                getF32Constant(DAG, 0x3ea21fb2, dl));
5265       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5266       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5267                                getF32Constant(DAG, 0x3f6ae232, dl));
5268       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5269       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5270                                     getF32Constant(DAG, 0x3f25f7c3, dl));
5271     } else { // LimitFloatPrecision <= 18
5272       // For floating-point precision of 18:
5273       //
5274       //   Log10ofMantissa =
5275       //     -0.84299375f +
5276       //       (1.5327582f +
5277       //         (-1.0688956f +
5278       //           (0.49102474f +
5279       //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5280       //
5281       // error 0.0000037995730, which is better than 18 bits
5282       SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5283                                getF32Constant(DAG, 0x3c5d51ce, dl));
5284       SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5285                                getF32Constant(DAG, 0x3e00685a, dl));
5286       SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5287       SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5288                                getF32Constant(DAG, 0x3efb6798, dl));
5289       SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5290       SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5291                                getF32Constant(DAG, 0x3f88d192, dl));
5292       SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5293       SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5294                                getF32Constant(DAG, 0x3fc4316c, dl));
5295       SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5296       Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5297                                     getF32Constant(DAG, 0x3f57ce70, dl));
5298     }
5299 
5300     return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5301   }
5302 
5303   // No special expansion.
5304   return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5305 }
5306 
5307 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5308 /// limited-precision mode.
5309 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5310                           const TargetLowering &TLI, SDNodeFlags Flags) {
5311   if (Op.getValueType() == MVT::f32 &&
5312       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
5313     return getLimitedPrecisionExp2(Op, dl, DAG);
5314 
5315   // No special expansion.
5316   return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5317 }
5318 
5319 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
5320 /// limited-precision mode with x == 10.0f.
5321 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5322                          SelectionDAG &DAG, const TargetLowering &TLI,
5323                          SDNodeFlags Flags) {
5324   bool IsExp10 = false;
5325   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5326       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
5327     if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5328       APFloat Ten(10.0f);
5329       IsExp10 = LHSC->isExactlyValue(Ten);
5330     }
5331   }
5332 
5333   // TODO: What fast-math-flags should be set on the FMUL node?
5334   if (IsExp10) {
5335     // Put the exponent in the right bit position for later addition to the
5336     // final result:
5337     //
5338     //   #define LOG2OF10 3.3219281f
5339     //   t0 = Op * LOG2OF10;
5340     SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5341                              getF32Constant(DAG, 0x40549a78, dl));
5342     return getLimitedPrecisionExp2(t0, dl, DAG);
5343   }
5344 
5345   // No special expansion.
5346   return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5347 }
5348 
5349 /// ExpandPowI - Expand a llvm.powi intrinsic.
5350 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5351                           SelectionDAG &DAG) {
5352   // If RHS is a constant, we can expand this out to a multiplication tree if
5353   // it's beneficial on the target, otherwise we end up lowering to a call to
5354   // __powidf2 (for example).
5355   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5356     unsigned Val = RHSC->getSExtValue();
5357 
5358     // powi(x, 0) -> 1.0
5359     if (Val == 0)
5360       return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5361 
5362     if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
5363             Val, DAG.shouldOptForSize())) {
5364       // Get the exponent as a positive value.
5365       if ((int)Val < 0)
5366         Val = -Val;
5367       // We use the simple binary decomposition method to generate the multiply
5368       // sequence.  There are more optimal ways to do this (for example,
5369       // powi(x,15) generates one more multiply than it should), but this has
5370       // the benefit of being both really simple and much better than a libcall.
5371       SDValue Res; // Logically starts equal to 1.0
5372       SDValue CurSquare = LHS;
5373       // TODO: Intrinsics should have fast-math-flags that propagate to these
5374       // nodes.
5375       while (Val) {
5376         if (Val & 1) {
5377           if (Res.getNode())
5378             Res =
5379                 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5380           else
5381             Res = CurSquare; // 1.0*CurSquare.
5382         }
5383 
5384         CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5385                                 CurSquare, CurSquare);
5386         Val >>= 1;
5387       }
5388 
5389       // If the original was negative, invert the result, producing 1/(x*x*x).
5390       if (RHSC->getSExtValue() < 0)
5391         Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5392                           DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5393       return Res;
5394     }
5395   }
5396 
5397   // Otherwise, expand to a libcall.
5398   return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5399 }
5400 
5401 static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5402                             SDValue LHS, SDValue RHS, SDValue Scale,
5403                             SelectionDAG &DAG, const TargetLowering &TLI) {
5404   EVT VT = LHS.getValueType();
5405   bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5406   bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5407   LLVMContext &Ctx = *DAG.getContext();
5408 
5409   // If the type is legal but the operation isn't, this node might survive all
5410   // the way to operation legalization. If we end up there and we do not have
5411   // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5412   // node.
5413 
5414   // Coax the legalizer into expanding the node during type legalization instead
5415   // by bumping the size by one bit. This will force it to Promote, enabling the
5416   // early expansion and avoiding the need to expand later.
5417 
5418   // We don't have to do this if Scale is 0; that can always be expanded, unless
5419   // it's a saturating signed operation. Those can experience true integer
5420   // division overflow, a case which we must avoid.
5421 
5422   // FIXME: We wouldn't have to do this (or any of the early
5423   // expansion/promotion) if it was possible to expand a libcall of an
5424   // illegal type during operation legalization. But it's not, so things
5425   // get a bit hacky.
5426   unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();
5427   if ((ScaleInt > 0 || (Saturating && Signed)) &&
5428       (TLI.isTypeLegal(VT) ||
5429        (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5430     TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
5431         Opcode, VT, ScaleInt);
5432     if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5433       EVT PromVT;
5434       if (VT.isScalarInteger())
5435         PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5436       else if (VT.isVector()) {
5437         PromVT = VT.getVectorElementType();
5438         PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5439         PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5440       } else
5441         llvm_unreachable("Wrong VT for DIVFIX?");
5442       if (Signed) {
5443         LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);
5444         RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);
5445       } else {
5446         LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);
5447         RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);
5448       }
5449       EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5450       // For saturating operations, we need to shift up the LHS to get the
5451       // proper saturation width, and then shift down again afterwards.
5452       if (Saturating)
5453         LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5454                           DAG.getConstant(1, DL, ShiftTy));
5455       SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5456       if (Saturating)
5457         Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5458                           DAG.getConstant(1, DL, ShiftTy));
5459       return DAG.getZExtOrTrunc(Res, DL, VT);
5460     }
5461   }
5462 
5463   return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5464 }
5465 
5466 // getUnderlyingArgRegs - Find underlying registers used for a truncated,
5467 // bitcasted, or split argument. Returns a list of <Register, size in bits>
5468 static void
5469 getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5470                      const SDValue &N) {
5471   switch (N.getOpcode()) {
5472   case ISD::CopyFromReg: {
5473     SDValue Op = N.getOperand(1);
5474     Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5475                       Op.getValueType().getSizeInBits());
5476     return;
5477   }
5478   case ISD::BITCAST:
5479   case ISD::AssertZext:
5480   case ISD::AssertSext:
5481   case ISD::TRUNCATE:
5482     getUnderlyingArgRegs(Regs, N.getOperand(0));
5483     return;
5484   case ISD::BUILD_PAIR:
5485   case ISD::BUILD_VECTOR:
5486   case ISD::CONCAT_VECTORS:
5487     for (SDValue Op : N->op_values())
5488       getUnderlyingArgRegs(Regs, Op);
5489     return;
5490   default:
5491     return;
5492   }
5493 }
5494 
5495 /// If the DbgValueInst is a dbg_value of a function argument, create the
5496 /// corresponding DBG_VALUE machine instruction for it now.  At the end of
5497 /// instruction selection, they will be inserted to the entry BB.
5498 /// We don't currently support this for variadic dbg_values, as they shouldn't
5499 /// appear for function arguments or in the prologue.
5500 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5501     const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5502     DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5503   const Argument *Arg = dyn_cast<Argument>(V);
5504   if (!Arg)
5505     return false;
5506 
5507   MachineFunction &MF = DAG.getMachineFunction();
5508   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5509 
5510   // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5511   // we've been asked to pursue.
5512   auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5513                               bool Indirect) {
5514     if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5515       // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5516       // pointing at the VReg, which will be patched up later.
5517       auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5518       auto MIB = BuildMI(MF, DL, Inst);
5519       MIB.addReg(Reg);
5520       MIB.addImm(0);
5521       MIB.addMetadata(Variable);
5522       auto *NewDIExpr = FragExpr;
5523       // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5524       // the DIExpression.
5525       if (Indirect)
5526         NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5527       MIB.addMetadata(NewDIExpr);
5528       return MIB;
5529     } else {
5530       // Create a completely standard DBG_VALUE.
5531       auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5532       return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5533     }
5534   };
5535 
5536   if (Kind == FuncArgumentDbgValueKind::Value) {
5537     // ArgDbgValues are hoisted to the beginning of the entry block. So we
5538     // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5539     // the entry block.
5540     bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5541     if (!IsInEntryBlock)
5542       return false;
5543 
5544     // ArgDbgValues are hoisted to the beginning of the entry block.  So we
5545     // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5546     // variable that also is a param.
5547     //
5548     // Although, if we are at the top of the entry block already, we can still
5549     // emit using ArgDbgValue. This might catch some situations when the
5550     // dbg.value refers to an argument that isn't used in the entry block, so
5551     // any CopyToReg node would be optimized out and the only way to express
5552     // this DBG_VALUE is by using the physical reg (or FI) as done in this
5553     // method.  ArgDbgValues are hoisted to the beginning of the entry block. So
5554     // we should only emit as ArgDbgValue if the Variable is an argument to the
5555     // current function, and the dbg.value intrinsic is found in the entry
5556     // block.
5557     bool VariableIsFunctionInputArg = Variable->isParameter() &&
5558         !DL->getInlinedAt();
5559     bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5560     if (!IsInPrologue && !VariableIsFunctionInputArg)
5561       return false;
5562 
5563     // Here we assume that a function argument on IR level only can be used to
5564     // describe one input parameter on source level. If we for example have
5565     // source code like this
5566     //
5567     //    struct A { long x, y; };
5568     //    void foo(struct A a, long b) {
5569     //      ...
5570     //      b = a.x;
5571     //      ...
5572     //    }
5573     //
5574     // and IR like this
5575     //
5576     //  define void @foo(i32 %a1, i32 %a2, i32 %b)  {
5577     //  entry:
5578     //    call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
5579     //    call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
5580     //    call void @llvm.dbg.value(metadata i32 %b, "b",
5581     //    ...
5582     //    call void @llvm.dbg.value(metadata i32 %a1, "b"
5583     //    ...
5584     //
5585     // then the last dbg.value is describing a parameter "b" using a value that
5586     // is an argument. But since we already has used %a1 to describe a parameter
5587     // we should not handle that last dbg.value here (that would result in an
5588     // incorrect hoisting of the DBG_VALUE to the function entry).
5589     // Notice that we allow one dbg.value per IR level argument, to accommodate
5590     // for the situation with fragments above.
5591     if (VariableIsFunctionInputArg) {
5592       unsigned ArgNo = Arg->getArgNo();
5593       if (ArgNo >= FuncInfo.DescribedArgs.size())
5594         FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
5595       else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
5596         return false;
5597       FuncInfo.DescribedArgs.set(ArgNo);
5598     }
5599   }
5600 
5601   bool IsIndirect = false;
5602   Optional<MachineOperand> Op;
5603   // Some arguments' frame index is recorded during argument lowering.
5604   int FI = FuncInfo.getArgumentFrameIndex(Arg);
5605   if (FI != std::numeric_limits<int>::max())
5606     Op = MachineOperand::CreateFI(FI);
5607 
5608   SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;
5609   if (!Op && N.getNode()) {
5610     getUnderlyingArgRegs(ArgRegsAndSizes, N);
5611     Register Reg;
5612     if (ArgRegsAndSizes.size() == 1)
5613       Reg = ArgRegsAndSizes.front().first;
5614 
5615     if (Reg && Reg.isVirtual()) {
5616       MachineRegisterInfo &RegInfo = MF.getRegInfo();
5617       Register PR = RegInfo.getLiveInPhysReg(Reg);
5618       if (PR)
5619         Reg = PR;
5620     }
5621     if (Reg) {
5622       Op = MachineOperand::CreateReg(Reg, false);
5623       IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
5624     }
5625   }
5626 
5627   if (!Op && N.getNode()) {
5628     // Check if frame index is available.
5629     SDValue LCandidate = peekThroughBitcasts(N);
5630     if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
5631       if (FrameIndexSDNode *FINode =
5632           dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
5633         Op = MachineOperand::CreateFI(FINode->getIndex());
5634   }
5635 
5636   if (!Op) {
5637     // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
5638     auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
5639                                          SplitRegs) {
5640       unsigned Offset = 0;
5641       for (const auto &RegAndSize : SplitRegs) {
5642         // If the expression is already a fragment, the current register
5643         // offset+size might extend beyond the fragment. In this case, only
5644         // the register bits that are inside the fragment are relevant.
5645         int RegFragmentSizeInBits = RegAndSize.second;
5646         if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
5647           uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
5648           // The register is entirely outside the expression fragment,
5649           // so is irrelevant for debug info.
5650           if (Offset >= ExprFragmentSizeInBits)
5651             break;
5652           // The register is partially outside the expression fragment, only
5653           // the low bits within the fragment are relevant for debug info.
5654           if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
5655             RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
5656           }
5657         }
5658 
5659         auto FragmentExpr = DIExpression::createFragmentExpression(
5660             Expr, Offset, RegFragmentSizeInBits);
5661         Offset += RegAndSize.second;
5662         // If a valid fragment expression cannot be created, the variable's
5663         // correct value cannot be determined and so it is set as Undef.
5664         if (!FragmentExpr) {
5665           SDDbgValue *SDV = DAG.getConstantDbgValue(
5666               Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
5667           DAG.AddDbgValue(SDV, false);
5668           continue;
5669         }
5670         MachineInstr *NewMI =
5671             MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
5672                              Kind != FuncArgumentDbgValueKind::Value);
5673         FuncInfo.ArgDbgValues.push_back(NewMI);
5674       }
5675     };
5676 
5677     // Check if ValueMap has reg number.
5678     DenseMap<const Value *, Register>::const_iterator
5679       VMI = FuncInfo.ValueMap.find(V);
5680     if (VMI != FuncInfo.ValueMap.end()) {
5681       const auto &TLI = DAG.getTargetLoweringInfo();
5682       RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
5683                        V->getType(), None);
5684       if (RFV.occupiesMultipleRegs()) {
5685         splitMultiRegDbgValue(RFV.getRegsAndSizes());
5686         return true;
5687       }
5688 
5689       Op = MachineOperand::CreateReg(VMI->second, false);
5690       IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
5691     } else if (ArgRegsAndSizes.size() > 1) {
5692       // This was split due to the calling convention, and no virtual register
5693       // mapping exists for the value.
5694       splitMultiRegDbgValue(ArgRegsAndSizes);
5695       return true;
5696     }
5697   }
5698 
5699   if (!Op)
5700     return false;
5701 
5702   assert(Variable->isValidLocationForIntrinsic(DL) &&
5703          "Expected inlined-at fields to agree");
5704   MachineInstr *NewMI = nullptr;
5705 
5706   if (Op->isReg())
5707     NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
5708   else
5709     NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
5710                     Variable, Expr);
5711 
5712   // Otherwise, use ArgDbgValues.
5713   FuncInfo.ArgDbgValues.push_back(NewMI);
5714   return true;
5715 }
5716 
5717 /// Return the appropriate SDDbgValue based on N.
5718 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
5719                                              DILocalVariable *Variable,
5720                                              DIExpression *Expr,
5721                                              const DebugLoc &dl,
5722                                              unsigned DbgSDNodeOrder) {
5723   if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
5724     // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
5725     // stack slot locations.
5726     //
5727     // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
5728     // debug values here after optimization:
5729     //
5730     //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
5731     //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
5732     //
5733     // Both describe the direct values of their associated variables.
5734     return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
5735                                      /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5736   }
5737   return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
5738                          /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5739 }
5740 
5741 static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
5742   switch (Intrinsic) {
5743   case Intrinsic::smul_fix:
5744     return ISD::SMULFIX;
5745   case Intrinsic::umul_fix:
5746     return ISD::UMULFIX;
5747   case Intrinsic::smul_fix_sat:
5748     return ISD::SMULFIXSAT;
5749   case Intrinsic::umul_fix_sat:
5750     return ISD::UMULFIXSAT;
5751   case Intrinsic::sdiv_fix:
5752     return ISD::SDIVFIX;
5753   case Intrinsic::udiv_fix:
5754     return ISD::UDIVFIX;
5755   case Intrinsic::sdiv_fix_sat:
5756     return ISD::SDIVFIXSAT;
5757   case Intrinsic::udiv_fix_sat:
5758     return ISD::UDIVFIXSAT;
5759   default:
5760     llvm_unreachable("Unhandled fixed point intrinsic");
5761   }
5762 }
5763 
5764 void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
5765                                            const char *FunctionName) {
5766   assert(FunctionName && "FunctionName must not be nullptr");
5767   SDValue Callee = DAG.getExternalSymbol(
5768       FunctionName,
5769       DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
5770   LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
5771 }
5772 
5773 /// Given a @llvm.call.preallocated.setup, return the corresponding
5774 /// preallocated call.
5775 static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
5776   assert(cast<CallBase>(PreallocatedSetup)
5777                  ->getCalledFunction()
5778                  ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
5779          "expected call_preallocated_setup Value");
5780   for (const auto *U : PreallocatedSetup->users()) {
5781     auto *UseCall = cast<CallBase>(U);
5782     const Function *Fn = UseCall->getCalledFunction();
5783     if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
5784       return UseCall;
5785     }
5786   }
5787   llvm_unreachable("expected corresponding call to preallocated setup/arg");
5788 }
5789 
5790 /// Lower the call to the specified intrinsic function.
5791 void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
5792                                              unsigned Intrinsic) {
5793   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5794   SDLoc sdl = getCurSDLoc();
5795   DebugLoc dl = getCurDebugLoc();
5796   SDValue Res;
5797 
5798   SDNodeFlags Flags;
5799   if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
5800     Flags.copyFMF(*FPOp);
5801 
5802   switch (Intrinsic) {
5803   default:
5804     // By default, turn this into a target intrinsic node.
5805     visitTargetIntrinsic(I, Intrinsic);
5806     return;
5807   case Intrinsic::vscale: {
5808     match(&I, m_VScale(DAG.getDataLayout()));
5809     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5810     setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
5811     return;
5812   }
5813   case Intrinsic::vastart:  visitVAStart(I); return;
5814   case Intrinsic::vaend:    visitVAEnd(I); return;
5815   case Intrinsic::vacopy:   visitVACopy(I); return;
5816   case Intrinsic::returnaddress:
5817     setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5818                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
5819                              getValue(I.getArgOperand(0))));
5820     return;
5821   case Intrinsic::addressofreturnaddress:
5822     setValue(&I,
5823              DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5824                          TLI.getValueType(DAG.getDataLayout(), I.getType())));
5825     return;
5826   case Intrinsic::sponentry:
5827     setValue(&I,
5828              DAG.getNode(ISD::SPONENTRY, sdl,
5829                          TLI.getValueType(DAG.getDataLayout(), I.getType())));
5830     return;
5831   case Intrinsic::frameaddress:
5832     setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5833                              TLI.getFrameIndexTy(DAG.getDataLayout()),
5834                              getValue(I.getArgOperand(0))));
5835     return;
5836   case Intrinsic::read_volatile_register:
5837   case Intrinsic::read_register: {
5838     Value *Reg = I.getArgOperand(0);
5839     SDValue Chain = getRoot();
5840     SDValue RegName =
5841         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5842     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5843     Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5844       DAG.getVTList(VT, MVT::Other), Chain, RegName);
5845     setValue(&I, Res);
5846     DAG.setRoot(Res.getValue(1));
5847     return;
5848   }
5849   case Intrinsic::write_register: {
5850     Value *Reg = I.getArgOperand(0);
5851     Value *RegValue = I.getArgOperand(1);
5852     SDValue Chain = getRoot();
5853     SDValue RegName =
5854         DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5855     DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5856                             RegName, getValue(RegValue)));
5857     return;
5858   }
5859   case Intrinsic::memcpy: {
5860     const auto &MCI = cast<MemCpyInst>(I);
5861     SDValue Op1 = getValue(I.getArgOperand(0));
5862     SDValue Op2 = getValue(I.getArgOperand(1));
5863     SDValue Op3 = getValue(I.getArgOperand(2));
5864     // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5865     Align DstAlign = MCI.getDestAlign().valueOrOne();
5866     Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5867     Align Alignment = std::min(DstAlign, SrcAlign);
5868     bool isVol = MCI.isVolatile();
5869     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5870     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5871     // node.
5872     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5873     SDValue MC = DAG.getMemcpy(
5874         Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5875         /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
5876         MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
5877     updateDAGForMaybeTailCall(MC);
5878     return;
5879   }
5880   case Intrinsic::memcpy_inline: {
5881     const auto &MCI = cast<MemCpyInlineInst>(I);
5882     SDValue Dst = getValue(I.getArgOperand(0));
5883     SDValue Src = getValue(I.getArgOperand(1));
5884     SDValue Size = getValue(I.getArgOperand(2));
5885     assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
5886     // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
5887     Align DstAlign = MCI.getDestAlign().valueOrOne();
5888     Align SrcAlign = MCI.getSourceAlign().valueOrOne();
5889     Align Alignment = std::min(DstAlign, SrcAlign);
5890     bool isVol = MCI.isVolatile();
5891     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5892     // FIXME: Support passing different dest/src alignments to the memcpy DAG
5893     // node.
5894     SDValue MC = DAG.getMemcpy(
5895         getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
5896         /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
5897         MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
5898     updateDAGForMaybeTailCall(MC);
5899     return;
5900   }
5901   case Intrinsic::memset: {
5902     const auto &MSI = cast<MemSetInst>(I);
5903     SDValue Op1 = getValue(I.getArgOperand(0));
5904     SDValue Op2 = getValue(I.getArgOperand(1));
5905     SDValue Op3 = getValue(I.getArgOperand(2));
5906     // @llvm.memset defines 0 and 1 to both mean no alignment.
5907     Align Alignment = MSI.getDestAlign().valueOrOne();
5908     bool isVol = MSI.isVolatile();
5909     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5910     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5911     SDValue MS = DAG.getMemset(
5912         Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
5913         isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
5914     updateDAGForMaybeTailCall(MS);
5915     return;
5916   }
5917   case Intrinsic::memset_inline: {
5918     const auto &MSII = cast<MemSetInlineInst>(I);
5919     SDValue Dst = getValue(I.getArgOperand(0));
5920     SDValue Value = getValue(I.getArgOperand(1));
5921     SDValue Size = getValue(I.getArgOperand(2));
5922     assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
5923     // @llvm.memset defines 0 and 1 to both mean no alignment.
5924     Align DstAlign = MSII.getDestAlign().valueOrOne();
5925     bool isVol = MSII.isVolatile();
5926     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5927     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5928     SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
5929                                /* AlwaysInline */ true, isTC,
5930                                MachinePointerInfo(I.getArgOperand(0)),
5931                                I.getAAMetadata());
5932     updateDAGForMaybeTailCall(MC);
5933     return;
5934   }
5935   case Intrinsic::memmove: {
5936     const auto &MMI = cast<MemMoveInst>(I);
5937     SDValue Op1 = getValue(I.getArgOperand(0));
5938     SDValue Op2 = getValue(I.getArgOperand(1));
5939     SDValue Op3 = getValue(I.getArgOperand(2));
5940     // @llvm.memmove defines 0 and 1 to both mean no alignment.
5941     Align DstAlign = MMI.getDestAlign().valueOrOne();
5942     Align SrcAlign = MMI.getSourceAlign().valueOrOne();
5943     Align Alignment = std::min(DstAlign, SrcAlign);
5944     bool isVol = MMI.isVolatile();
5945     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5946     // FIXME: Support passing different dest/src alignments to the memmove DAG
5947     // node.
5948     SDValue Root = isVol ? getRoot() : getMemoryRoot();
5949     SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
5950                                 isTC, MachinePointerInfo(I.getArgOperand(0)),
5951                                 MachinePointerInfo(I.getArgOperand(1)),
5952                                 I.getAAMetadata(), AA);
5953     updateDAGForMaybeTailCall(MM);
5954     return;
5955   }
5956   case Intrinsic::memcpy_element_unordered_atomic: {
5957     const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
5958     SDValue Dst = getValue(MI.getRawDest());
5959     SDValue Src = getValue(MI.getRawSource());
5960     SDValue Length = getValue(MI.getLength());
5961 
5962     Type *LengthTy = MI.getLength()->getType();
5963     unsigned ElemSz = MI.getElementSizeInBytes();
5964     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5965     SDValue MC =
5966         DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
5967                             isTC, MachinePointerInfo(MI.getRawDest()),
5968                             MachinePointerInfo(MI.getRawSource()));
5969     updateDAGForMaybeTailCall(MC);
5970     return;
5971   }
5972   case Intrinsic::memmove_element_unordered_atomic: {
5973     auto &MI = cast<AtomicMemMoveInst>(I);
5974     SDValue Dst = getValue(MI.getRawDest());
5975     SDValue Src = getValue(MI.getRawSource());
5976     SDValue Length = getValue(MI.getLength());
5977 
5978     Type *LengthTy = MI.getLength()->getType();
5979     unsigned ElemSz = MI.getElementSizeInBytes();
5980     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5981     SDValue MC =
5982         DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
5983                              isTC, MachinePointerInfo(MI.getRawDest()),
5984                              MachinePointerInfo(MI.getRawSource()));
5985     updateDAGForMaybeTailCall(MC);
5986     return;
5987   }
5988   case Intrinsic::memset_element_unordered_atomic: {
5989     auto &MI = cast<AtomicMemSetInst>(I);
5990     SDValue Dst = getValue(MI.getRawDest());
5991     SDValue Val = getValue(MI.getValue());
5992     SDValue Length = getValue(MI.getLength());
5993 
5994     Type *LengthTy = MI.getLength()->getType();
5995     unsigned ElemSz = MI.getElementSizeInBytes();
5996     bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
5997     SDValue MC =
5998         DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
5999                             isTC, MachinePointerInfo(MI.getRawDest()));
6000     updateDAGForMaybeTailCall(MC);
6001     return;
6002   }
6003   case Intrinsic::call_preallocated_setup: {
6004     const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6005     SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6006     SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6007                               getRoot(), SrcValue);
6008     setValue(&I, Res);
6009     DAG.setRoot(Res);
6010     return;
6011   }
6012   case Intrinsic::call_preallocated_arg: {
6013     const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6014     SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6015     SDValue Ops[3];
6016     Ops[0] = getRoot();
6017     Ops[1] = SrcValue;
6018     Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6019                                    MVT::i32); // arg index
6020     SDValue Res = DAG.getNode(
6021         ISD::PREALLOCATED_ARG, sdl,
6022         DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6023     setValue(&I, Res);
6024     DAG.setRoot(Res.getValue(1));
6025     return;
6026   }
6027   case Intrinsic::dbg_addr:
6028   case Intrinsic::dbg_declare: {
6029     // Assume dbg.addr and dbg.declare can not currently use DIArgList, i.e.
6030     // they are non-variadic.
6031     const auto &DI = cast<DbgVariableIntrinsic>(I);
6032     assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6033     DILocalVariable *Variable = DI.getVariable();
6034     DIExpression *Expression = DI.getExpression();
6035     dropDanglingDebugInfo(Variable, Expression);
6036     assert(Variable && "Missing variable");
6037     LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DI
6038                       << "\n");
6039     // Check if address has undef value.
6040     const Value *Address = DI.getVariableLocationOp(0);
6041     if (!Address || isa<UndefValue>(Address) ||
6042         (Address->use_empty() && !isa<Argument>(Address))) {
6043       LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI
6044                         << " (bad/undef/unused-arg address)\n");
6045       return;
6046     }
6047 
6048     bool isParameter = Variable->isParameter() || isa<Argument>(Address);
6049 
6050     // Check if this variable can be described by a frame index, typically
6051     // either as a static alloca or a byval parameter.
6052     int FI = std::numeric_limits<int>::max();
6053     if (const auto *AI =
6054             dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
6055       if (AI->isStaticAlloca()) {
6056         auto I = FuncInfo.StaticAllocaMap.find(AI);
6057         if (I != FuncInfo.StaticAllocaMap.end())
6058           FI = I->second;
6059       }
6060     } else if (const auto *Arg = dyn_cast<Argument>(
6061                    Address->stripInBoundsConstantOffsets())) {
6062       FI = FuncInfo.getArgumentFrameIndex(Arg);
6063     }
6064 
6065     // llvm.dbg.addr is control dependent and always generates indirect
6066     // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in
6067     // the MachineFunction variable table.
6068     if (FI != std::numeric_limits<int>::max()) {
6069       if (Intrinsic == Intrinsic::dbg_addr) {
6070         SDDbgValue *SDV = DAG.getFrameIndexDbgValue(
6071             Variable, Expression, FI, getRoot().getNode(), /*IsIndirect*/ true,
6072             dl, SDNodeOrder);
6073         DAG.AddDbgValue(SDV, isParameter);
6074       } else {
6075         LLVM_DEBUG(dbgs() << "Skipping " << DI
6076                           << " (variable info stashed in MF side table)\n");
6077       }
6078       return;
6079     }
6080 
6081     SDValue &N = NodeMap[Address];
6082     if (!N.getNode() && isa<Argument>(Address))
6083       // Check unused arguments map.
6084       N = UnusedArgNodeMap[Address];
6085     SDDbgValue *SDV;
6086     if (N.getNode()) {
6087       if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
6088         Address = BCI->getOperand(0);
6089       // Parameters are handled specially.
6090       auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
6091       if (isParameter && FINode) {
6092         // Byval parameter. We have a frame index at this point.
6093         SDV =
6094             DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
6095                                       /*IsIndirect*/ true, dl, SDNodeOrder);
6096       } else if (isa<Argument>(Address)) {
6097         // Address is an argument, so try to emit its dbg value using
6098         // virtual register info from the FuncInfo.ValueMap.
6099         EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
6100                                  FuncArgumentDbgValueKind::Declare, N);
6101         return;
6102       } else {
6103         SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
6104                               true, dl, SDNodeOrder);
6105       }
6106       DAG.AddDbgValue(SDV, isParameter);
6107     } else {
6108       // If Address is an argument then try to emit its dbg value using
6109       // virtual register info from the FuncInfo.ValueMap.
6110       if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
6111                                     FuncArgumentDbgValueKind::Declare, N)) {
6112         LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI
6113                           << " (could not emit func-arg dbg_value)\n");
6114       }
6115     }
6116     return;
6117   }
6118   case Intrinsic::dbg_label: {
6119     const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6120     DILabel *Label = DI.getLabel();
6121     assert(Label && "Missing label");
6122 
6123     SDDbgLabel *SDV;
6124     SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6125     DAG.AddDbgLabel(SDV);
6126     return;
6127   }
6128   case Intrinsic::dbg_value: {
6129     const DbgValueInst &DI = cast<DbgValueInst>(I);
6130     assert(DI.getVariable() && "Missing variable");
6131 
6132     DILocalVariable *Variable = DI.getVariable();
6133     DIExpression *Expression = DI.getExpression();
6134     dropDanglingDebugInfo(Variable, Expression);
6135     SmallVector<Value *, 4> Values(DI.getValues());
6136     if (Values.empty())
6137       return;
6138 
6139     if (llvm::is_contained(Values, nullptr))
6140       return;
6141 
6142     bool IsVariadic = DI.hasArgList();
6143     if (!handleDebugValue(Values, Variable, Expression, dl, DI.getDebugLoc(),
6144                           SDNodeOrder, IsVariadic))
6145       addDanglingDebugInfo(&DI, dl, SDNodeOrder);
6146     return;
6147   }
6148 
6149   case Intrinsic::eh_typeid_for: {
6150     // Find the type id for the given typeinfo.
6151     GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6152     unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6153     Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6154     setValue(&I, Res);
6155     return;
6156   }
6157 
6158   case Intrinsic::eh_return_i32:
6159   case Intrinsic::eh_return_i64:
6160     DAG.getMachineFunction().setCallsEHReturn(true);
6161     DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
6162                             MVT::Other,
6163                             getControlRoot(),
6164                             getValue(I.getArgOperand(0)),
6165                             getValue(I.getArgOperand(1))));
6166     return;
6167   case Intrinsic::eh_unwind_init:
6168     DAG.getMachineFunction().setCallsUnwindInit(true);
6169     return;
6170   case Intrinsic::eh_dwarf_cfa:
6171     setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
6172                              TLI.getPointerTy(DAG.getDataLayout()),
6173                              getValue(I.getArgOperand(0))));
6174     return;
6175   case Intrinsic::eh_sjlj_callsite: {
6176     MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6177     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6178     assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
6179 
6180     MMI.setCurrentCallSite(CI->getZExtValue());
6181     return;
6182   }
6183   case Intrinsic::eh_sjlj_functioncontext: {
6184     // Get and store the index of the function context.
6185     MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6186     AllocaInst *FnCtx =
6187       cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6188     int FI = FuncInfo.StaticAllocaMap[FnCtx];
6189     MFI.setFunctionContextIndex(FI);
6190     return;
6191   }
6192   case Intrinsic::eh_sjlj_setjmp: {
6193     SDValue Ops[2];
6194     Ops[0] = getRoot();
6195     Ops[1] = getValue(I.getArgOperand(0));
6196     SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
6197                              DAG.getVTList(MVT::i32, MVT::Other), Ops);
6198     setValue(&I, Op.getValue(0));
6199     DAG.setRoot(Op.getValue(1));
6200     return;
6201   }
6202   case Intrinsic::eh_sjlj_longjmp:
6203     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6204                             getRoot(), getValue(I.getArgOperand(0))));
6205     return;
6206   case Intrinsic::eh_sjlj_setup_dispatch:
6207     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
6208                             getRoot()));
6209     return;
6210   case Intrinsic::masked_gather:
6211     visitMaskedGather(I);
6212     return;
6213   case Intrinsic::masked_load:
6214     visitMaskedLoad(I);
6215     return;
6216   case Intrinsic::masked_scatter:
6217     visitMaskedScatter(I);
6218     return;
6219   case Intrinsic::masked_store:
6220     visitMaskedStore(I);
6221     return;
6222   case Intrinsic::masked_expandload:
6223     visitMaskedLoad(I, true /* IsExpanding */);
6224     return;
6225   case Intrinsic::masked_compressstore:
6226     visitMaskedStore(I, true /* IsCompressing */);
6227     return;
6228   case Intrinsic::powi:
6229     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6230                             getValue(I.getArgOperand(1)), DAG));
6231     return;
6232   case Intrinsic::log:
6233     setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6234     return;
6235   case Intrinsic::log2:
6236     setValue(&I,
6237              expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6238     return;
6239   case Intrinsic::log10:
6240     setValue(&I,
6241              expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6242     return;
6243   case Intrinsic::exp:
6244     setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6245     return;
6246   case Intrinsic::exp2:
6247     setValue(&I,
6248              expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6249     return;
6250   case Intrinsic::pow:
6251     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6252                            getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6253     return;
6254   case Intrinsic::sqrt:
6255   case Intrinsic::fabs:
6256   case Intrinsic::sin:
6257   case Intrinsic::cos:
6258   case Intrinsic::floor:
6259   case Intrinsic::ceil:
6260   case Intrinsic::trunc:
6261   case Intrinsic::rint:
6262   case Intrinsic::nearbyint:
6263   case Intrinsic::round:
6264   case Intrinsic::roundeven:
6265   case Intrinsic::canonicalize: {
6266     unsigned Opcode;
6267     switch (Intrinsic) {
6268     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6269     case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
6270     case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
6271     case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
6272     case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
6273     case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
6274     case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
6275     case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
6276     case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
6277     case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6278     case Intrinsic::round:     Opcode = ISD::FROUND;     break;
6279     case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6280     case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6281     }
6282 
6283     setValue(&I, DAG.getNode(Opcode, sdl,
6284                              getValue(I.getArgOperand(0)).getValueType(),
6285                              getValue(I.getArgOperand(0)), Flags));
6286     return;
6287   }
6288   case Intrinsic::lround:
6289   case Intrinsic::llround:
6290   case Intrinsic::lrint:
6291   case Intrinsic::llrint: {
6292     unsigned Opcode;
6293     switch (Intrinsic) {
6294     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6295     case Intrinsic::lround:  Opcode = ISD::LROUND;  break;
6296     case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6297     case Intrinsic::lrint:   Opcode = ISD::LRINT;   break;
6298     case Intrinsic::llrint:  Opcode = ISD::LLRINT;  break;
6299     }
6300 
6301     EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6302     setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6303                              getValue(I.getArgOperand(0))));
6304     return;
6305   }
6306   case Intrinsic::minnum:
6307     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
6308                              getValue(I.getArgOperand(0)).getValueType(),
6309                              getValue(I.getArgOperand(0)),
6310                              getValue(I.getArgOperand(1)), Flags));
6311     return;
6312   case Intrinsic::maxnum:
6313     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
6314                              getValue(I.getArgOperand(0)).getValueType(),
6315                              getValue(I.getArgOperand(0)),
6316                              getValue(I.getArgOperand(1)), Flags));
6317     return;
6318   case Intrinsic::minimum:
6319     setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
6320                              getValue(I.getArgOperand(0)).getValueType(),
6321                              getValue(I.getArgOperand(0)),
6322                              getValue(I.getArgOperand(1)), Flags));
6323     return;
6324   case Intrinsic::maximum:
6325     setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
6326                              getValue(I.getArgOperand(0)).getValueType(),
6327                              getValue(I.getArgOperand(0)),
6328                              getValue(I.getArgOperand(1)), Flags));
6329     return;
6330   case Intrinsic::copysign:
6331     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
6332                              getValue(I.getArgOperand(0)).getValueType(),
6333                              getValue(I.getArgOperand(0)),
6334                              getValue(I.getArgOperand(1)), Flags));
6335     return;
6336   case Intrinsic::arithmetic_fence: {
6337     setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
6338                              getValue(I.getArgOperand(0)).getValueType(),
6339                              getValue(I.getArgOperand(0)), Flags));
6340     return;
6341   }
6342   case Intrinsic::fma:
6343     setValue(&I, DAG.getNode(
6344                      ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6345                      getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6346                      getValue(I.getArgOperand(2)), Flags));
6347     return;
6348 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
6349   case Intrinsic::INTRINSIC:
6350 #include "llvm/IR/ConstrainedOps.def"
6351     visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6352     return;
6353 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6354 #include "llvm/IR/VPIntrinsics.def"
6355     visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6356     return;
6357   case Intrinsic::fptrunc_round: {
6358     // Get the last argument, the metadata and convert it to an integer in the
6359     // call
6360     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6361     Optional<RoundingMode> RoundMode =
6362         convertStrToRoundingMode(cast<MDString>(MD)->getString());
6363 
6364     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6365 
6366     // Propagate fast-math-flags from IR to node(s).
6367     SDNodeFlags Flags;
6368     Flags.copyFMF(*cast<FPMathOperator>(&I));
6369     SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6370 
6371     SDValue Result;
6372     Result = DAG.getNode(
6373         ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6374         DAG.getTargetConstant((int)*RoundMode, sdl,
6375                               TLI.getPointerTy(DAG.getDataLayout())));
6376     setValue(&I, Result);
6377 
6378     return;
6379   }
6380   case Intrinsic::fmuladd: {
6381     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6382     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
6383         TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
6384       setValue(&I, DAG.getNode(ISD::FMA, sdl,
6385                                getValue(I.getArgOperand(0)).getValueType(),
6386                                getValue(I.getArgOperand(0)),
6387                                getValue(I.getArgOperand(1)),
6388                                getValue(I.getArgOperand(2)), Flags));
6389     } else {
6390       // TODO: Intrinsic calls should have fast-math-flags.
6391       SDValue Mul = DAG.getNode(
6392           ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6393           getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6394       SDValue Add = DAG.getNode(ISD::FADD, sdl,
6395                                 getValue(I.getArgOperand(0)).getValueType(),
6396                                 Mul, getValue(I.getArgOperand(2)), Flags);
6397       setValue(&I, Add);
6398     }
6399     return;
6400   }
6401   case Intrinsic::convert_to_fp16:
6402     setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6403                              DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6404                                          getValue(I.getArgOperand(0)),
6405                                          DAG.getTargetConstant(0, sdl,
6406                                                                MVT::i32))));
6407     return;
6408   case Intrinsic::convert_from_fp16:
6409     setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
6410                              TLI.getValueType(DAG.getDataLayout(), I.getType()),
6411                              DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6412                                          getValue(I.getArgOperand(0)))));
6413     return;
6414   case Intrinsic::fptosi_sat: {
6415     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6416     setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
6417                              getValue(I.getArgOperand(0)),
6418                              DAG.getValueType(VT.getScalarType())));
6419     return;
6420   }
6421   case Intrinsic::fptoui_sat: {
6422     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6423     setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
6424                              getValue(I.getArgOperand(0)),
6425                              DAG.getValueType(VT.getScalarType())));
6426     return;
6427   }
6428   case Intrinsic::set_rounding:
6429     Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
6430                       {getRoot(), getValue(I.getArgOperand(0))});
6431     setValue(&I, Res);
6432     DAG.setRoot(Res.getValue(0));
6433     return;
6434   case Intrinsic::is_fpclass: {
6435     const DataLayout DLayout = DAG.getDataLayout();
6436     EVT DestVT = TLI.getValueType(DLayout, I.getType());
6437     EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
6438     unsigned Test = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6439     MachineFunction &MF = DAG.getMachineFunction();
6440     const Function &F = MF.getFunction();
6441     SDValue Op = getValue(I.getArgOperand(0));
6442     SDNodeFlags Flags;
6443     Flags.setNoFPExcept(
6444         !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
6445     // If ISD::IS_FPCLASS should be expanded, do it right now, because the
6446     // expansion can use illegal types. Making expansion early allows
6447     // legalizing these types prior to selection.
6448     if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
6449       SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
6450       setValue(&I, Result);
6451       return;
6452     }
6453 
6454     SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
6455     SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
6456     setValue(&I, V);
6457     return;
6458   }
6459   case Intrinsic::pcmarker: {
6460     SDValue Tmp = getValue(I.getArgOperand(0));
6461     DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
6462     return;
6463   }
6464   case Intrinsic::readcyclecounter: {
6465     SDValue Op = getRoot();
6466     Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
6467                       DAG.getVTList(MVT::i64, MVT::Other), Op);
6468     setValue(&I, Res);
6469     DAG.setRoot(Res.getValue(1));
6470     return;
6471   }
6472   case Intrinsic::bitreverse:
6473     setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
6474                              getValue(I.getArgOperand(0)).getValueType(),
6475                              getValue(I.getArgOperand(0))));
6476     return;
6477   case Intrinsic::bswap:
6478     setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
6479                              getValue(I.getArgOperand(0)).getValueType(),
6480                              getValue(I.getArgOperand(0))));
6481     return;
6482   case Intrinsic::cttz: {
6483     SDValue Arg = getValue(I.getArgOperand(0));
6484     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6485     EVT Ty = Arg.getValueType();
6486     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
6487                              sdl, Ty, Arg));
6488     return;
6489   }
6490   case Intrinsic::ctlz: {
6491     SDValue Arg = getValue(I.getArgOperand(0));
6492     ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6493     EVT Ty = Arg.getValueType();
6494     setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
6495                              sdl, Ty, Arg));
6496     return;
6497   }
6498   case Intrinsic::ctpop: {
6499     SDValue Arg = getValue(I.getArgOperand(0));
6500     EVT Ty = Arg.getValueType();
6501     setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
6502     return;
6503   }
6504   case Intrinsic::fshl:
6505   case Intrinsic::fshr: {
6506     bool IsFSHL = Intrinsic == Intrinsic::fshl;
6507     SDValue X = getValue(I.getArgOperand(0));
6508     SDValue Y = getValue(I.getArgOperand(1));
6509     SDValue Z = getValue(I.getArgOperand(2));
6510     EVT VT = X.getValueType();
6511 
6512     if (X == Y) {
6513       auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
6514       setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
6515     } else {
6516       auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
6517       setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
6518     }
6519     return;
6520   }
6521   case Intrinsic::sadd_sat: {
6522     SDValue Op1 = getValue(I.getArgOperand(0));
6523     SDValue Op2 = getValue(I.getArgOperand(1));
6524     setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
6525     return;
6526   }
6527   case Intrinsic::uadd_sat: {
6528     SDValue Op1 = getValue(I.getArgOperand(0));
6529     SDValue Op2 = getValue(I.getArgOperand(1));
6530     setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
6531     return;
6532   }
6533   case Intrinsic::ssub_sat: {
6534     SDValue Op1 = getValue(I.getArgOperand(0));
6535     SDValue Op2 = getValue(I.getArgOperand(1));
6536     setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
6537     return;
6538   }
6539   case Intrinsic::usub_sat: {
6540     SDValue Op1 = getValue(I.getArgOperand(0));
6541     SDValue Op2 = getValue(I.getArgOperand(1));
6542     setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
6543     return;
6544   }
6545   case Intrinsic::sshl_sat: {
6546     SDValue Op1 = getValue(I.getArgOperand(0));
6547     SDValue Op2 = getValue(I.getArgOperand(1));
6548     setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
6549     return;
6550   }
6551   case Intrinsic::ushl_sat: {
6552     SDValue Op1 = getValue(I.getArgOperand(0));
6553     SDValue Op2 = getValue(I.getArgOperand(1));
6554     setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
6555     return;
6556   }
6557   case Intrinsic::smul_fix:
6558   case Intrinsic::umul_fix:
6559   case Intrinsic::smul_fix_sat:
6560   case Intrinsic::umul_fix_sat: {
6561     SDValue Op1 = getValue(I.getArgOperand(0));
6562     SDValue Op2 = getValue(I.getArgOperand(1));
6563     SDValue Op3 = getValue(I.getArgOperand(2));
6564     setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
6565                              Op1.getValueType(), Op1, Op2, Op3));
6566     return;
6567   }
6568   case Intrinsic::sdiv_fix:
6569   case Intrinsic::udiv_fix:
6570   case Intrinsic::sdiv_fix_sat:
6571   case Intrinsic::udiv_fix_sat: {
6572     SDValue Op1 = getValue(I.getArgOperand(0));
6573     SDValue Op2 = getValue(I.getArgOperand(1));
6574     SDValue Op3 = getValue(I.getArgOperand(2));
6575     setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
6576                               Op1, Op2, Op3, DAG, TLI));
6577     return;
6578   }
6579   case Intrinsic::smax: {
6580     SDValue Op1 = getValue(I.getArgOperand(0));
6581     SDValue Op2 = getValue(I.getArgOperand(1));
6582     setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
6583     return;
6584   }
6585   case Intrinsic::smin: {
6586     SDValue Op1 = getValue(I.getArgOperand(0));
6587     SDValue Op2 = getValue(I.getArgOperand(1));
6588     setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
6589     return;
6590   }
6591   case Intrinsic::umax: {
6592     SDValue Op1 = getValue(I.getArgOperand(0));
6593     SDValue Op2 = getValue(I.getArgOperand(1));
6594     setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
6595     return;
6596   }
6597   case Intrinsic::umin: {
6598     SDValue Op1 = getValue(I.getArgOperand(0));
6599     SDValue Op2 = getValue(I.getArgOperand(1));
6600     setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
6601     return;
6602   }
6603   case Intrinsic::abs: {
6604     // TODO: Preserve "int min is poison" arg in SDAG?
6605     SDValue Op1 = getValue(I.getArgOperand(0));
6606     setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
6607     return;
6608   }
6609   case Intrinsic::stacksave: {
6610     SDValue Op = getRoot();
6611     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6612     Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
6613     setValue(&I, Res);
6614     DAG.setRoot(Res.getValue(1));
6615     return;
6616   }
6617   case Intrinsic::stackrestore:
6618     Res = getValue(I.getArgOperand(0));
6619     DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
6620     return;
6621   case Intrinsic::get_dynamic_area_offset: {
6622     SDValue Op = getRoot();
6623     EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
6624     EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
6625     // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
6626     // target.
6627     if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
6628       report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
6629                          " intrinsic!");
6630     Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
6631                       Op);
6632     DAG.setRoot(Op);
6633     setValue(&I, Res);
6634     return;
6635   }
6636   case Intrinsic::stackguard: {
6637     MachineFunction &MF = DAG.getMachineFunction();
6638     const Module &M = *MF.getFunction().getParent();
6639     SDValue Chain = getRoot();
6640     if (TLI.useLoadStackGuardNode()) {
6641       Res = getLoadStackGuard(DAG, sdl, Chain);
6642     } else {
6643       EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
6644       const Value *Global = TLI.getSDagStackGuard(M);
6645       Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
6646       Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
6647                         MachinePointerInfo(Global, 0), Align,
6648                         MachineMemOperand::MOVolatile);
6649     }
6650     if (TLI.useStackGuardXorFP())
6651       Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
6652     DAG.setRoot(Chain);
6653     setValue(&I, Res);
6654     return;
6655   }
6656   case Intrinsic::stackprotector: {
6657     // Emit code into the DAG to store the stack guard onto the stack.
6658     MachineFunction &MF = DAG.getMachineFunction();
6659     MachineFrameInfo &MFI = MF.getFrameInfo();
6660     SDValue Src, Chain = getRoot();
6661 
6662     if (TLI.useLoadStackGuardNode())
6663       Src = getLoadStackGuard(DAG, sdl, Chain);
6664     else
6665       Src = getValue(I.getArgOperand(0));   // The guard's value.
6666 
6667     AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
6668 
6669     int FI = FuncInfo.StaticAllocaMap[Slot];
6670     MFI.setStackProtectorIndex(FI);
6671     EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
6672 
6673     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
6674 
6675     // Store the stack protector onto the stack.
6676     Res = DAG.getStore(
6677         Chain, sdl, Src, FIN,
6678         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
6679         MaybeAlign(), MachineMemOperand::MOVolatile);
6680     setValue(&I, Res);
6681     DAG.setRoot(Res);
6682     return;
6683   }
6684   case Intrinsic::objectsize:
6685     llvm_unreachable("llvm.objectsize.* should have been lowered already");
6686 
6687   case Intrinsic::is_constant:
6688     llvm_unreachable("llvm.is.constant.* should have been lowered already");
6689 
6690   case Intrinsic::annotation:
6691   case Intrinsic::ptr_annotation:
6692   case Intrinsic::launder_invariant_group:
6693   case Intrinsic::strip_invariant_group:
6694     // Drop the intrinsic, but forward the value
6695     setValue(&I, getValue(I.getOperand(0)));
6696     return;
6697 
6698   case Intrinsic::assume:
6699   case Intrinsic::experimental_noalias_scope_decl:
6700   case Intrinsic::var_annotation:
6701   case Intrinsic::sideeffect:
6702     // Discard annotate attributes, noalias scope declarations, assumptions, and
6703     // artificial side-effects.
6704     return;
6705 
6706   case Intrinsic::codeview_annotation: {
6707     // Emit a label associated with this metadata.
6708     MachineFunction &MF = DAG.getMachineFunction();
6709     MCSymbol *Label =
6710         MF.getMMI().getContext().createTempSymbol("annotation", true);
6711     Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
6712     MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
6713     Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
6714     DAG.setRoot(Res);
6715     return;
6716   }
6717 
6718   case Intrinsic::init_trampoline: {
6719     const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
6720 
6721     SDValue Ops[6];
6722     Ops[0] = getRoot();
6723     Ops[1] = getValue(I.getArgOperand(0));
6724     Ops[2] = getValue(I.getArgOperand(1));
6725     Ops[3] = getValue(I.getArgOperand(2));
6726     Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
6727     Ops[5] = DAG.getSrcValue(F);
6728 
6729     Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
6730 
6731     DAG.setRoot(Res);
6732     return;
6733   }
6734   case Intrinsic::adjust_trampoline:
6735     setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
6736                              TLI.getPointerTy(DAG.getDataLayout()),
6737                              getValue(I.getArgOperand(0))));
6738     return;
6739   case Intrinsic::gcroot: {
6740     assert(DAG.getMachineFunction().getFunction().hasGC() &&
6741            "only valid in functions with gc specified, enforced by Verifier");
6742     assert(GFI && "implied by previous");
6743     const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
6744     const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
6745 
6746     FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
6747     GFI->addStackRoot(FI->getIndex(), TypeMap);
6748     return;
6749   }
6750   case Intrinsic::gcread:
6751   case Intrinsic::gcwrite:
6752     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
6753   case Intrinsic::flt_rounds:
6754     Res = DAG.getNode(ISD::FLT_ROUNDS_, sdl, {MVT::i32, MVT::Other}, getRoot());
6755     setValue(&I, Res);
6756     DAG.setRoot(Res.getValue(1));
6757     return;
6758 
6759   case Intrinsic::expect:
6760     // Just replace __builtin_expect(exp, c) with EXP.
6761     setValue(&I, getValue(I.getArgOperand(0)));
6762     return;
6763 
6764   case Intrinsic::ubsantrap:
6765   case Intrinsic::debugtrap:
6766   case Intrinsic::trap: {
6767     StringRef TrapFuncName =
6768         I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
6769     if (TrapFuncName.empty()) {
6770       switch (Intrinsic) {
6771       case Intrinsic::trap:
6772         DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
6773         break;
6774       case Intrinsic::debugtrap:
6775         DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
6776         break;
6777       case Intrinsic::ubsantrap:
6778         DAG.setRoot(DAG.getNode(
6779             ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
6780             DAG.getTargetConstant(
6781                 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
6782                 MVT::i32)));
6783         break;
6784       default: llvm_unreachable("unknown trap intrinsic");
6785       }
6786       return;
6787     }
6788     TargetLowering::ArgListTy Args;
6789     if (Intrinsic == Intrinsic::ubsantrap) {
6790       Args.push_back(TargetLoweringBase::ArgListEntry());
6791       Args[0].Val = I.getArgOperand(0);
6792       Args[0].Node = getValue(Args[0].Val);
6793       Args[0].Ty = Args[0].Val->getType();
6794     }
6795 
6796     TargetLowering::CallLoweringInfo CLI(DAG);
6797     CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
6798         CallingConv::C, I.getType(),
6799         DAG.getExternalSymbol(TrapFuncName.data(),
6800                               TLI.getPointerTy(DAG.getDataLayout())),
6801         std::move(Args));
6802 
6803     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6804     DAG.setRoot(Result.second);
6805     return;
6806   }
6807 
6808   case Intrinsic::uadd_with_overflow:
6809   case Intrinsic::sadd_with_overflow:
6810   case Intrinsic::usub_with_overflow:
6811   case Intrinsic::ssub_with_overflow:
6812   case Intrinsic::umul_with_overflow:
6813   case Intrinsic::smul_with_overflow: {
6814     ISD::NodeType Op;
6815     switch (Intrinsic) {
6816     default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
6817     case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
6818     case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
6819     case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
6820     case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
6821     case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
6822     case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
6823     }
6824     SDValue Op1 = getValue(I.getArgOperand(0));
6825     SDValue Op2 = getValue(I.getArgOperand(1));
6826 
6827     EVT ResultVT = Op1.getValueType();
6828     EVT OverflowVT = MVT::i1;
6829     if (ResultVT.isVector())
6830       OverflowVT = EVT::getVectorVT(
6831           *Context, OverflowVT, ResultVT.getVectorElementCount());
6832 
6833     SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
6834     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
6835     return;
6836   }
6837   case Intrinsic::prefetch: {
6838     SDValue Ops[5];
6839     unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6840     auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
6841     Ops[0] = DAG.getRoot();
6842     Ops[1] = getValue(I.getArgOperand(0));
6843     Ops[2] = getValue(I.getArgOperand(1));
6844     Ops[3] = getValue(I.getArgOperand(2));
6845     Ops[4] = getValue(I.getArgOperand(3));
6846     SDValue Result = DAG.getMemIntrinsicNode(
6847         ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
6848         EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
6849         /* align */ None, Flags);
6850 
6851     // Chain the prefetch in parallell with any pending loads, to stay out of
6852     // the way of later optimizations.
6853     PendingLoads.push_back(Result);
6854     Result = getRoot();
6855     DAG.setRoot(Result);
6856     return;
6857   }
6858   case Intrinsic::lifetime_start:
6859   case Intrinsic::lifetime_end: {
6860     bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
6861     // Stack coloring is not enabled in O0, discard region information.
6862     if (TM.getOptLevel() == CodeGenOpt::None)
6863       return;
6864 
6865     const int64_t ObjectSize =
6866         cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
6867     Value *const ObjectPtr = I.getArgOperand(1);
6868     SmallVector<const Value *, 4> Allocas;
6869     getUnderlyingObjects(ObjectPtr, Allocas);
6870 
6871     for (const Value *Alloca : Allocas) {
6872       const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
6873 
6874       // Could not find an Alloca.
6875       if (!LifetimeObject)
6876         continue;
6877 
6878       // First check that the Alloca is static, otherwise it won't have a
6879       // valid frame index.
6880       auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
6881       if (SI == FuncInfo.StaticAllocaMap.end())
6882         return;
6883 
6884       const int FrameIndex = SI->second;
6885       int64_t Offset;
6886       if (GetPointerBaseWithConstantOffset(
6887               ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
6888         Offset = -1; // Cannot determine offset from alloca to lifetime object.
6889       Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
6890                                 Offset);
6891       DAG.setRoot(Res);
6892     }
6893     return;
6894   }
6895   case Intrinsic::pseudoprobe: {
6896     auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
6897     auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
6898     auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
6899     Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
6900     DAG.setRoot(Res);
6901     return;
6902   }
6903   case Intrinsic::invariant_start:
6904     // Discard region information.
6905     setValue(&I,
6906              DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
6907     return;
6908   case Intrinsic::invariant_end:
6909     // Discard region information.
6910     return;
6911   case Intrinsic::clear_cache:
6912     /// FunctionName may be null.
6913     if (const char *FunctionName = TLI.getClearCacheBuiltinName())
6914       lowerCallToExternalSymbol(I, FunctionName);
6915     return;
6916   case Intrinsic::donothing:
6917   case Intrinsic::seh_try_begin:
6918   case Intrinsic::seh_scope_begin:
6919   case Intrinsic::seh_try_end:
6920   case Intrinsic::seh_scope_end:
6921     // ignore
6922     return;
6923   case Intrinsic::experimental_stackmap:
6924     visitStackmap(I);
6925     return;
6926   case Intrinsic::experimental_patchpoint_void:
6927   case Intrinsic::experimental_patchpoint_i64:
6928     visitPatchpoint(I);
6929     return;
6930   case Intrinsic::experimental_gc_statepoint:
6931     LowerStatepoint(cast<GCStatepointInst>(I));
6932     return;
6933   case Intrinsic::experimental_gc_result:
6934     visitGCResult(cast<GCResultInst>(I));
6935     return;
6936   case Intrinsic::experimental_gc_relocate:
6937     visitGCRelocate(cast<GCRelocateInst>(I));
6938     return;
6939   case Intrinsic::instrprof_cover:
6940     llvm_unreachable("instrprof failed to lower a cover");
6941   case Intrinsic::instrprof_increment:
6942     llvm_unreachable("instrprof failed to lower an increment");
6943   case Intrinsic::instrprof_value_profile:
6944     llvm_unreachable("instrprof failed to lower a value profiling call");
6945   case Intrinsic::localescape: {
6946     MachineFunction &MF = DAG.getMachineFunction();
6947     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6948 
6949     // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
6950     // is the same on all targets.
6951     for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
6952       Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
6953       if (isa<ConstantPointerNull>(Arg))
6954         continue; // Skip null pointers. They represent a hole in index space.
6955       AllocaInst *Slot = cast<AllocaInst>(Arg);
6956       assert(FuncInfo.StaticAllocaMap.count(Slot) &&
6957              "can only escape static allocas");
6958       int FI = FuncInfo.StaticAllocaMap[Slot];
6959       MCSymbol *FrameAllocSym =
6960           MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6961               GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
6962       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
6963               TII->get(TargetOpcode::LOCAL_ESCAPE))
6964           .addSym(FrameAllocSym)
6965           .addFrameIndex(FI);
6966     }
6967 
6968     return;
6969   }
6970 
6971   case Intrinsic::localrecover: {
6972     // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
6973     MachineFunction &MF = DAG.getMachineFunction();
6974 
6975     // Get the symbol that defines the frame offset.
6976     auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
6977     auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
6978     unsigned IdxVal =
6979         unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
6980     MCSymbol *FrameAllocSym =
6981         MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6982             GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
6983 
6984     Value *FP = I.getArgOperand(1);
6985     SDValue FPVal = getValue(FP);
6986     EVT PtrVT = FPVal.getValueType();
6987 
6988     // Create a MCSymbol for the label to avoid any target lowering
6989     // that would make this PC relative.
6990     SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
6991     SDValue OffsetVal =
6992         DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
6993 
6994     // Add the offset to the FP.
6995     SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
6996     setValue(&I, Add);
6997 
6998     return;
6999   }
7000 
7001   case Intrinsic::eh_exceptionpointer:
7002   case Intrinsic::eh_exceptioncode: {
7003     // Get the exception pointer vreg, copy from it, and resize it to fit.
7004     const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7005     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7006     const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7007     unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7008     SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7009     if (Intrinsic == Intrinsic::eh_exceptioncode)
7010       N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7011     setValue(&I, N);
7012     return;
7013   }
7014   case Intrinsic::xray_customevent: {
7015     // Here we want to make sure that the intrinsic behaves as if it has a
7016     // specific calling convention, and only for x86_64.
7017     // FIXME: Support other platforms later.
7018     const auto &Triple = DAG.getTarget().getTargetTriple();
7019     if (Triple.getArch() != Triple::x86_64)
7020       return;
7021 
7022     SmallVector<SDValue, 8> Ops;
7023 
7024     // We want to say that we always want the arguments in registers.
7025     SDValue LogEntryVal = getValue(I.getArgOperand(0));
7026     SDValue StrSizeVal = getValue(I.getArgOperand(1));
7027     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7028     SDValue Chain = getRoot();
7029     Ops.push_back(LogEntryVal);
7030     Ops.push_back(StrSizeVal);
7031     Ops.push_back(Chain);
7032 
7033     // We need to enforce the calling convention for the callsite, so that
7034     // argument ordering is enforced correctly, and that register allocation can
7035     // see that some registers may be assumed clobbered and have to preserve
7036     // them across calls to the intrinsic.
7037     MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7038                                            sdl, NodeTys, Ops);
7039     SDValue patchableNode = SDValue(MN, 0);
7040     DAG.setRoot(patchableNode);
7041     setValue(&I, patchableNode);
7042     return;
7043   }
7044   case Intrinsic::xray_typedevent: {
7045     // Here we want to make sure that the intrinsic behaves as if it has a
7046     // specific calling convention, and only for x86_64.
7047     // FIXME: Support other platforms later.
7048     const auto &Triple = DAG.getTarget().getTargetTriple();
7049     if (Triple.getArch() != Triple::x86_64)
7050       return;
7051 
7052     SmallVector<SDValue, 8> Ops;
7053 
7054     // We want to say that we always want the arguments in registers.
7055     // It's unclear to me how manipulating the selection DAG here forces callers
7056     // to provide arguments in registers instead of on the stack.
7057     SDValue LogTypeId = getValue(I.getArgOperand(0));
7058     SDValue LogEntryVal = getValue(I.getArgOperand(1));
7059     SDValue StrSizeVal = getValue(I.getArgOperand(2));
7060     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7061     SDValue Chain = getRoot();
7062     Ops.push_back(LogTypeId);
7063     Ops.push_back(LogEntryVal);
7064     Ops.push_back(StrSizeVal);
7065     Ops.push_back(Chain);
7066 
7067     // We need to enforce the calling convention for the callsite, so that
7068     // argument ordering is enforced correctly, and that register allocation can
7069     // see that some registers may be assumed clobbered and have to preserve
7070     // them across calls to the intrinsic.
7071     MachineSDNode *MN = DAG.getMachineNode(
7072         TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7073     SDValue patchableNode = SDValue(MN, 0);
7074     DAG.setRoot(patchableNode);
7075     setValue(&I, patchableNode);
7076     return;
7077   }
7078   case Intrinsic::experimental_deoptimize:
7079     LowerDeoptimizeCall(&I);
7080     return;
7081   case Intrinsic::experimental_stepvector:
7082     visitStepVector(I);
7083     return;
7084   case Intrinsic::vector_reduce_fadd:
7085   case Intrinsic::vector_reduce_fmul:
7086   case Intrinsic::vector_reduce_add:
7087   case Intrinsic::vector_reduce_mul:
7088   case Intrinsic::vector_reduce_and:
7089   case Intrinsic::vector_reduce_or:
7090   case Intrinsic::vector_reduce_xor:
7091   case Intrinsic::vector_reduce_smax:
7092   case Intrinsic::vector_reduce_smin:
7093   case Intrinsic::vector_reduce_umax:
7094   case Intrinsic::vector_reduce_umin:
7095   case Intrinsic::vector_reduce_fmax:
7096   case Intrinsic::vector_reduce_fmin:
7097     visitVectorReduce(I, Intrinsic);
7098     return;
7099 
7100   case Intrinsic::icall_branch_funnel: {
7101     SmallVector<SDValue, 16> Ops;
7102     Ops.push_back(getValue(I.getArgOperand(0)));
7103 
7104     int64_t Offset;
7105     auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7106         I.getArgOperand(1), Offset, DAG.getDataLayout()));
7107     if (!Base)
7108       report_fatal_error(
7109           "llvm.icall.branch.funnel operand must be a GlobalValue");
7110     Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7111 
7112     struct BranchFunnelTarget {
7113       int64_t Offset;
7114       SDValue Target;
7115     };
7116     SmallVector<BranchFunnelTarget, 8> Targets;
7117 
7118     for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7119       auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7120           I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7121       if (ElemBase != Base)
7122         report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7123                            "to the same GlobalValue");
7124 
7125       SDValue Val = getValue(I.getArgOperand(Op + 1));
7126       auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7127       if (!GA)
7128         report_fatal_error(
7129             "llvm.icall.branch.funnel operand must be a GlobalValue");
7130       Targets.push_back({Offset, DAG.getTargetGlobalAddress(
7131                                      GA->getGlobal(), sdl, Val.getValueType(),
7132                                      GA->getOffset())});
7133     }
7134     llvm::sort(Targets,
7135                [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7136                  return T1.Offset < T2.Offset;
7137                });
7138 
7139     for (auto &T : Targets) {
7140       Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7141       Ops.push_back(T.Target);
7142     }
7143 
7144     Ops.push_back(DAG.getRoot()); // Chain
7145     SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7146                                  MVT::Other, Ops),
7147               0);
7148     DAG.setRoot(N);
7149     setValue(&I, N);
7150     HasTailCall = true;
7151     return;
7152   }
7153 
7154   case Intrinsic::wasm_landingpad_index:
7155     // Information this intrinsic contained has been transferred to
7156     // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7157     // delete it now.
7158     return;
7159 
7160   case Intrinsic::aarch64_settag:
7161   case Intrinsic::aarch64_settag_zero: {
7162     const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7163     bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7164     SDValue Val = TSI.EmitTargetCodeForSetTag(
7165         DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7166         getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7167         ZeroMemory);
7168     DAG.setRoot(Val);
7169     setValue(&I, Val);
7170     return;
7171   }
7172   case Intrinsic::ptrmask: {
7173     SDValue Ptr = getValue(I.getOperand(0));
7174     SDValue Const = getValue(I.getOperand(1));
7175 
7176     EVT PtrVT = Ptr.getValueType();
7177     setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr,
7178                              DAG.getZExtOrTrunc(Const, sdl, PtrVT)));
7179     return;
7180   }
7181   case Intrinsic::get_active_lane_mask: {
7182     EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7183     SDValue Index = getValue(I.getOperand(0));
7184     EVT ElementVT = Index.getValueType();
7185 
7186     if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7187       visitTargetIntrinsic(I, Intrinsic);
7188       return;
7189     }
7190 
7191     SDValue TripCount = getValue(I.getOperand(1));
7192     auto VecTy = CCVT.changeVectorElementType(ElementVT);
7193 
7194     SDValue VectorIndex, VectorTripCount;
7195     if (VecTy.isScalableVector()) {
7196       VectorIndex = DAG.getSplatVector(VecTy, sdl, Index);
7197       VectorTripCount = DAG.getSplatVector(VecTy, sdl, TripCount);
7198     } else {
7199       VectorIndex = DAG.getSplatBuildVector(VecTy, sdl, Index);
7200       VectorTripCount = DAG.getSplatBuildVector(VecTy, sdl, TripCount);
7201     }
7202     SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7203     SDValue VectorInduction = DAG.getNode(
7204         ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7205     SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7206                                  VectorTripCount, ISD::CondCode::SETULT);
7207     setValue(&I, SetCC);
7208     return;
7209   }
7210   case Intrinsic::vector_insert: {
7211     SDValue Vec = getValue(I.getOperand(0));
7212     SDValue SubVec = getValue(I.getOperand(1));
7213     SDValue Index = getValue(I.getOperand(2));
7214 
7215     // The intrinsic's index type is i64, but the SDNode requires an index type
7216     // suitable for the target. Convert the index as required.
7217     MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7218     if (Index.getValueType() != VectorIdxTy)
7219       Index = DAG.getVectorIdxConstant(
7220           cast<ConstantSDNode>(Index)->getZExtValue(), sdl);
7221 
7222     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7223     setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
7224                              Index));
7225     return;
7226   }
7227   case Intrinsic::vector_extract: {
7228     SDValue Vec = getValue(I.getOperand(0));
7229     SDValue Index = getValue(I.getOperand(1));
7230     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7231 
7232     // The intrinsic's index type is i64, but the SDNode requires an index type
7233     // suitable for the target. Convert the index as required.
7234     MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7235     if (Index.getValueType() != VectorIdxTy)
7236       Index = DAG.getVectorIdxConstant(
7237           cast<ConstantSDNode>(Index)->getZExtValue(), sdl);
7238 
7239     setValue(&I,
7240              DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
7241     return;
7242   }
7243   case Intrinsic::experimental_vector_reverse:
7244     visitVectorReverse(I);
7245     return;
7246   case Intrinsic::experimental_vector_splice:
7247     visitVectorSplice(I);
7248     return;
7249   }
7250 }
7251 
7252 void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
7253     const ConstrainedFPIntrinsic &FPI) {
7254   SDLoc sdl = getCurSDLoc();
7255 
7256   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7257   SmallVector<EVT, 4> ValueVTs;
7258   ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs);
7259   ValueVTs.push_back(MVT::Other); // Out chain
7260 
7261   // We do not need to serialize constrained FP intrinsics against
7262   // each other or against (nonvolatile) loads, so they can be
7263   // chained like loads.
7264   SDValue Chain = DAG.getRoot();
7265   SmallVector<SDValue, 4> Opers;
7266   Opers.push_back(Chain);
7267   if (FPI.isUnaryOp()) {
7268     Opers.push_back(getValue(FPI.getArgOperand(0)));
7269   } else if (FPI.isTernaryOp()) {
7270     Opers.push_back(getValue(FPI.getArgOperand(0)));
7271     Opers.push_back(getValue(FPI.getArgOperand(1)));
7272     Opers.push_back(getValue(FPI.getArgOperand(2)));
7273   } else {
7274     Opers.push_back(getValue(FPI.getArgOperand(0)));
7275     Opers.push_back(getValue(FPI.getArgOperand(1)));
7276   }
7277 
7278   auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
7279     assert(Result.getNode()->getNumValues() == 2);
7280 
7281     // Push node to the appropriate list so that future instructions can be
7282     // chained up correctly.
7283     SDValue OutChain = Result.getValue(1);
7284     switch (EB) {
7285     case fp::ExceptionBehavior::ebIgnore:
7286       // The only reason why ebIgnore nodes still need to be chained is that
7287       // they might depend on the current rounding mode, and therefore must
7288       // not be moved across instruction that may change that mode.
7289       LLVM_FALLTHROUGH;
7290     case fp::ExceptionBehavior::ebMayTrap:
7291       // These must not be moved across calls or instructions that may change
7292       // floating-point exception masks.
7293       PendingConstrainedFP.push_back(OutChain);
7294       break;
7295     case fp::ExceptionBehavior::ebStrict:
7296       // These must not be moved across calls or instructions that may change
7297       // floating-point exception masks or read floating-point exception flags.
7298       // In addition, they cannot be optimized out even if unused.
7299       PendingConstrainedFPStrict.push_back(OutChain);
7300       break;
7301     }
7302   };
7303 
7304   SDVTList VTs = DAG.getVTList(ValueVTs);
7305   fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
7306 
7307   SDNodeFlags Flags;
7308   if (EB == fp::ExceptionBehavior::ebIgnore)
7309     Flags.setNoFPExcept(true);
7310 
7311   if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
7312     Flags.copyFMF(*FPOp);
7313 
7314   unsigned Opcode;
7315   switch (FPI.getIntrinsicID()) {
7316   default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
7317 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
7318   case Intrinsic::INTRINSIC:                                                   \
7319     Opcode = ISD::STRICT_##DAGN;                                               \
7320     break;
7321 #include "llvm/IR/ConstrainedOps.def"
7322   case Intrinsic::experimental_constrained_fmuladd: {
7323     Opcode = ISD::STRICT_FMA;
7324     // Break fmuladd into fmul and fadd.
7325     if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
7326         !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(),
7327                                         ValueVTs[0])) {
7328       Opers.pop_back();
7329       SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
7330       pushOutChain(Mul, EB);
7331       Opcode = ISD::STRICT_FADD;
7332       Opers.clear();
7333       Opers.push_back(Mul.getValue(1));
7334       Opers.push_back(Mul.getValue(0));
7335       Opers.push_back(getValue(FPI.getArgOperand(2)));
7336     }
7337     break;
7338   }
7339   }
7340 
7341   // A few strict DAG nodes carry additional operands that are not
7342   // set up by the default code above.
7343   switch (Opcode) {
7344   default: break;
7345   case ISD::STRICT_FP_ROUND:
7346     Opers.push_back(
7347         DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
7348     break;
7349   case ISD::STRICT_FSETCC:
7350   case ISD::STRICT_FSETCCS: {
7351     auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
7352     ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
7353     if (TM.Options.NoNaNsFPMath)
7354       Condition = getFCmpCodeWithoutNaN(Condition);
7355     Opers.push_back(DAG.getCondCode(Condition));
7356     break;
7357   }
7358   }
7359 
7360   SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
7361   pushOutChain(Result, EB);
7362 
7363   SDValue FPResult = Result.getValue(0);
7364   setValue(&FPI, FPResult);
7365 }
7366 
7367 static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
7368   Optional<unsigned> ResOPC;
7369   switch (VPIntrin.getIntrinsicID()) {
7370 #define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)                                    \
7371   case Intrinsic::VPID:                                                        \
7372     ResOPC = ISD::VPSD;                                                        \
7373     break;
7374 #include "llvm/IR/VPIntrinsics.def"
7375   }
7376 
7377   if (!ResOPC)
7378     llvm_unreachable(
7379         "Inconsistency: no SDNode available for this VPIntrinsic!");
7380 
7381   if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
7382       *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
7383     if (VPIntrin.getFastMathFlags().allowReassoc())
7384       return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
7385                                                 : ISD::VP_REDUCE_FMUL;
7386   }
7387 
7388   return *ResOPC;
7389 }
7390 
7391 void SelectionDAGBuilder::visitVPLoadGather(const VPIntrinsic &VPIntrin, EVT VT,
7392                                             SmallVector<SDValue, 7> &OpValues,
7393                                             bool IsGather) {
7394   SDLoc DL = getCurSDLoc();
7395   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7396   Value *PtrOperand = VPIntrin.getArgOperand(0);
7397   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7398   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7399   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7400   SDValue LD;
7401   bool AddToChain = true;
7402   if (!IsGather) {
7403     // Do not serialize variable-length loads of constant memory with
7404     // anything.
7405     if (!Alignment)
7406       Alignment = DAG.getEVTAlign(VT);
7407     MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
7408     AddToChain = !AA || !AA->pointsToConstantMemory(ML);
7409     SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
7410     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7411         MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
7412         MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7413     LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
7414                        MMO, false /*IsExpanding */);
7415   } else {
7416     if (!Alignment)
7417       Alignment = DAG.getEVTAlign(VT.getScalarType());
7418     unsigned AS =
7419         PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
7420     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7421         MachinePointerInfo(AS), MachineMemOperand::MOLoad,
7422         MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7423     SDValue Base, Index, Scale;
7424     ISD::MemIndexType IndexType;
7425     bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
7426                                       this, VPIntrin.getParent(),
7427                                       VT.getScalarStoreSize());
7428     if (!UniformBase) {
7429       Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
7430       Index = getValue(PtrOperand);
7431       IndexType = ISD::SIGNED_SCALED;
7432       Scale =
7433           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
7434     }
7435     EVT IdxVT = Index.getValueType();
7436     EVT EltTy = IdxVT.getVectorElementType();
7437     if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
7438       EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
7439       Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
7440     }
7441     LD = DAG.getGatherVP(
7442         DAG.getVTList(VT, MVT::Other), VT, DL,
7443         {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
7444         IndexType);
7445   }
7446   if (AddToChain)
7447     PendingLoads.push_back(LD.getValue(1));
7448   setValue(&VPIntrin, LD);
7449 }
7450 
7451 void SelectionDAGBuilder::visitVPStoreScatter(const VPIntrinsic &VPIntrin,
7452                                               SmallVector<SDValue, 7> &OpValues,
7453                                               bool IsScatter) {
7454   SDLoc DL = getCurSDLoc();
7455   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7456   Value *PtrOperand = VPIntrin.getArgOperand(1);
7457   EVT VT = OpValues[0].getValueType();
7458   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7459   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7460   SDValue ST;
7461   if (!IsScatter) {
7462     if (!Alignment)
7463       Alignment = DAG.getEVTAlign(VT);
7464     SDValue Ptr = OpValues[1];
7465     SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
7466     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7467         MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
7468         MemoryLocation::UnknownSize, *Alignment, AAInfo);
7469     ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
7470                         OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
7471                         /* IsTruncating */ false, /*IsCompressing*/ false);
7472   } else {
7473     if (!Alignment)
7474       Alignment = DAG.getEVTAlign(VT.getScalarType());
7475     unsigned AS =
7476         PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
7477     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7478         MachinePointerInfo(AS), MachineMemOperand::MOStore,
7479         MemoryLocation::UnknownSize, *Alignment, AAInfo);
7480     SDValue Base, Index, Scale;
7481     ISD::MemIndexType IndexType;
7482     bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
7483                                       this, VPIntrin.getParent(),
7484                                       VT.getScalarStoreSize());
7485     if (!UniformBase) {
7486       Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
7487       Index = getValue(PtrOperand);
7488       IndexType = ISD::SIGNED_SCALED;
7489       Scale =
7490           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
7491     }
7492     EVT IdxVT = Index.getValueType();
7493     EVT EltTy = IdxVT.getVectorElementType();
7494     if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
7495       EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
7496       Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
7497     }
7498     ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
7499                           {getMemoryRoot(), OpValues[0], Base, Index, Scale,
7500                            OpValues[2], OpValues[3]},
7501                           MMO, IndexType);
7502   }
7503   DAG.setRoot(ST);
7504   setValue(&VPIntrin, ST);
7505 }
7506 
7507 void SelectionDAGBuilder::visitVPStridedLoad(
7508     const VPIntrinsic &VPIntrin, EVT VT, SmallVectorImpl<SDValue> &OpValues) {
7509   SDLoc DL = getCurSDLoc();
7510   Value *PtrOperand = VPIntrin.getArgOperand(0);
7511   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7512   if (!Alignment)
7513     Alignment = DAG.getEVTAlign(VT.getScalarType());
7514   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7515   const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
7516   MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
7517   bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
7518   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
7519   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7520       MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
7521       MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
7522 
7523   SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
7524                                     OpValues[2], OpValues[3], MMO,
7525                                     false /*IsExpanding*/);
7526 
7527   if (AddToChain)
7528     PendingLoads.push_back(LD.getValue(1));
7529   setValue(&VPIntrin, LD);
7530 }
7531 
7532 void SelectionDAGBuilder::visitVPStridedStore(
7533     const VPIntrinsic &VPIntrin, SmallVectorImpl<SDValue> &OpValues) {
7534   SDLoc DL = getCurSDLoc();
7535   Value *PtrOperand = VPIntrin.getArgOperand(1);
7536   EVT VT = OpValues[0].getValueType();
7537   MaybeAlign Alignment = VPIntrin.getPointerAlignment();
7538   if (!Alignment)
7539     Alignment = DAG.getEVTAlign(VT.getScalarType());
7540   AAMDNodes AAInfo = VPIntrin.getAAMetadata();
7541   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7542       MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
7543       MemoryLocation::UnknownSize, *Alignment, AAInfo);
7544 
7545   SDValue ST = DAG.getStridedStoreVP(
7546       getMemoryRoot(), DL, OpValues[0], OpValues[1],
7547       DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
7548       OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
7549       /*IsCompressing*/ false);
7550 
7551   DAG.setRoot(ST);
7552   setValue(&VPIntrin, ST);
7553 }
7554 
7555 void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
7556   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7557   SDLoc DL = getCurSDLoc();
7558 
7559   ISD::CondCode Condition;
7560   CmpInst::Predicate CondCode = VPIntrin.getPredicate();
7561   bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
7562   if (IsFP) {
7563     // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
7564     // flags, but calls that don't return floating-point types can't be
7565     // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
7566     Condition = getFCmpCondCode(CondCode);
7567     if (TM.Options.NoNaNsFPMath)
7568       Condition = getFCmpCodeWithoutNaN(Condition);
7569   } else {
7570     Condition = getICmpCondCode(CondCode);
7571   }
7572 
7573   SDValue Op1 = getValue(VPIntrin.getOperand(0));
7574   SDValue Op2 = getValue(VPIntrin.getOperand(1));
7575   // #2 is the condition code
7576   SDValue MaskOp = getValue(VPIntrin.getOperand(3));
7577   SDValue EVL = getValue(VPIntrin.getOperand(4));
7578   MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
7579   assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
7580          "Unexpected target EVL type");
7581   EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
7582 
7583   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7584                                                         VPIntrin.getType());
7585   setValue(&VPIntrin,
7586            DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
7587 }
7588 
7589 void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
7590     const VPIntrinsic &VPIntrin) {
7591   SDLoc DL = getCurSDLoc();
7592   unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
7593 
7594   auto IID = VPIntrin.getIntrinsicID();
7595 
7596   if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
7597     return visitVPCmp(*CmpI);
7598 
7599   SmallVector<EVT, 4> ValueVTs;
7600   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7601   ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
7602   SDVTList VTs = DAG.getVTList(ValueVTs);
7603 
7604   auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
7605 
7606   MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
7607   assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
7608          "Unexpected target EVL type");
7609 
7610   // Request operands.
7611   SmallVector<SDValue, 7> OpValues;
7612   for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
7613     auto Op = getValue(VPIntrin.getArgOperand(I));
7614     if (I == EVLParamPos)
7615       Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
7616     OpValues.push_back(Op);
7617   }
7618 
7619   switch (Opcode) {
7620   default: {
7621     SDNodeFlags SDFlags;
7622     if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
7623       SDFlags.copyFMF(*FPMO);
7624     SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
7625     setValue(&VPIntrin, Result);
7626     break;
7627   }
7628   case ISD::VP_LOAD:
7629   case ISD::VP_GATHER:
7630     visitVPLoadGather(VPIntrin, ValueVTs[0], OpValues,
7631                       Opcode == ISD::VP_GATHER);
7632     break;
7633   case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7634     visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
7635     break;
7636   case ISD::VP_STORE:
7637   case ISD::VP_SCATTER:
7638     visitVPStoreScatter(VPIntrin, OpValues, Opcode == ISD::VP_SCATTER);
7639     break;
7640   case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7641     visitVPStridedStore(VPIntrin, OpValues);
7642     break;
7643   }
7644 }
7645 
7646 SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
7647                                           const BasicBlock *EHPadBB,
7648                                           MCSymbol *&BeginLabel) {
7649   MachineFunction &MF = DAG.getMachineFunction();
7650   MachineModuleInfo &MMI = MF.getMMI();
7651 
7652   // Insert a label before the invoke call to mark the try range.  This can be
7653   // used to detect deletion of the invoke via the MachineModuleInfo.
7654   BeginLabel = MMI.getContext().createTempSymbol();
7655 
7656   // For SjLj, keep track of which landing pads go with which invokes
7657   // so as to maintain the ordering of pads in the LSDA.
7658   unsigned CallSiteIndex = MMI.getCurrentCallSite();
7659   if (CallSiteIndex) {
7660     MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
7661     LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
7662 
7663     // Now that the call site is handled, stop tracking it.
7664     MMI.setCurrentCallSite(0);
7665   }
7666 
7667   return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
7668 }
7669 
7670 SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
7671                                         const BasicBlock *EHPadBB,
7672                                         MCSymbol *BeginLabel) {
7673   assert(BeginLabel && "BeginLabel should've been set");
7674 
7675   MachineFunction &MF = DAG.getMachineFunction();
7676   MachineModuleInfo &MMI = MF.getMMI();
7677 
7678   // Insert a label at the end of the invoke call to mark the try range.  This
7679   // can be used to detect deletion of the invoke via the MachineModuleInfo.
7680   MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
7681   Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
7682 
7683   // Inform MachineModuleInfo of range.
7684   auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
7685   // There is a platform (e.g. wasm) that uses funclet style IR but does not
7686   // actually use outlined funclets and their LSDA info style.
7687   if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
7688     assert(II && "II should've been set");
7689     WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
7690     EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
7691   } else if (!isScopedEHPersonality(Pers)) {
7692     assert(EHPadBB);
7693     MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
7694   }
7695 
7696   return Chain;
7697 }
7698 
7699 std::pair<SDValue, SDValue>
7700 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
7701                                     const BasicBlock *EHPadBB) {
7702   MCSymbol *BeginLabel = nullptr;
7703 
7704   if (EHPadBB) {
7705     // Both PendingLoads and PendingExports must be flushed here;
7706     // this call might not return.
7707     (void)getRoot();
7708     DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
7709     CLI.setChain(getRoot());
7710   }
7711 
7712   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7713   std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7714 
7715   assert((CLI.IsTailCall || Result.second.getNode()) &&
7716          "Non-null chain expected with non-tail call!");
7717   assert((Result.second.getNode() || !Result.first.getNode()) &&
7718          "Null value expected with tail call!");
7719 
7720   if (!Result.second.getNode()) {
7721     // As a special case, a null chain means that a tail call has been emitted
7722     // and the DAG root is already updated.
7723     HasTailCall = true;
7724 
7725     // Since there's no actual continuation from this block, nothing can be
7726     // relying on us setting vregs for them.
7727     PendingExports.clear();
7728   } else {
7729     DAG.setRoot(Result.second);
7730   }
7731 
7732   if (EHPadBB) {
7733     DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
7734                            BeginLabel));
7735   }
7736 
7737   return Result;
7738 }
7739 
7740 void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
7741                                       bool isTailCall,
7742                                       bool isMustTailCall,
7743                                       const BasicBlock *EHPadBB) {
7744   auto &DL = DAG.getDataLayout();
7745   FunctionType *FTy = CB.getFunctionType();
7746   Type *RetTy = CB.getType();
7747 
7748   TargetLowering::ArgListTy Args;
7749   Args.reserve(CB.arg_size());
7750 
7751   const Value *SwiftErrorVal = nullptr;
7752   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7753 
7754   if (isTailCall) {
7755     // Avoid emitting tail calls in functions with the disable-tail-calls
7756     // attribute.
7757     auto *Caller = CB.getParent()->getParent();
7758     if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
7759         "true" && !isMustTailCall)
7760       isTailCall = false;
7761 
7762     // We can't tail call inside a function with a swifterror argument. Lowering
7763     // does not support this yet. It would have to move into the swifterror
7764     // register before the call.
7765     if (TLI.supportSwiftError() &&
7766         Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
7767       isTailCall = false;
7768   }
7769 
7770   for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
7771     TargetLowering::ArgListEntry Entry;
7772     const Value *V = *I;
7773 
7774     // Skip empty types
7775     if (V->getType()->isEmptyTy())
7776       continue;
7777 
7778     SDValue ArgNode = getValue(V);
7779     Entry.Node = ArgNode; Entry.Ty = V->getType();
7780 
7781     Entry.setAttributes(&CB, I - CB.arg_begin());
7782 
7783     // Use swifterror virtual register as input to the call.
7784     if (Entry.IsSwiftError && TLI.supportSwiftError()) {
7785       SwiftErrorVal = V;
7786       // We find the virtual register for the actual swifterror argument.
7787       // Instead of using the Value, we use the virtual register instead.
7788       Entry.Node =
7789           DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
7790                           EVT(TLI.getPointerTy(DL)));
7791     }
7792 
7793     Args.push_back(Entry);
7794 
7795     // If we have an explicit sret argument that is an Instruction, (i.e., it
7796     // might point to function-local memory), we can't meaningfully tail-call.
7797     if (Entry.IsSRet && isa<Instruction>(V))
7798       isTailCall = false;
7799   }
7800 
7801   // If call site has a cfguardtarget operand bundle, create and add an
7802   // additional ArgListEntry.
7803   if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
7804     TargetLowering::ArgListEntry Entry;
7805     Value *V = Bundle->Inputs[0];
7806     SDValue ArgNode = getValue(V);
7807     Entry.Node = ArgNode;
7808     Entry.Ty = V->getType();
7809     Entry.IsCFGuardTarget = true;
7810     Args.push_back(Entry);
7811   }
7812 
7813   // Check if target-independent constraints permit a tail call here.
7814   // Target-dependent constraints are checked within TLI->LowerCallTo.
7815   if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
7816     isTailCall = false;
7817 
7818   // Disable tail calls if there is an swifterror argument. Targets have not
7819   // been updated to support tail calls.
7820   if (TLI.supportSwiftError() && SwiftErrorVal)
7821     isTailCall = false;
7822 
7823   TargetLowering::CallLoweringInfo CLI(DAG);
7824   CLI.setDebugLoc(getCurSDLoc())
7825       .setChain(getRoot())
7826       .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
7827       .setTailCall(isTailCall)
7828       .setConvergent(CB.isConvergent())
7829       .setIsPreallocated(
7830           CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
7831   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7832 
7833   if (Result.first.getNode()) {
7834     Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
7835     setValue(&CB, Result.first);
7836   }
7837 
7838   // The last element of CLI.InVals has the SDValue for swifterror return.
7839   // Here we copy it to a virtual register and update SwiftErrorMap for
7840   // book-keeping.
7841   if (SwiftErrorVal && TLI.supportSwiftError()) {
7842     // Get the last element of InVals.
7843     SDValue Src = CLI.InVals.back();
7844     Register VReg =
7845         SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
7846     SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
7847     DAG.setRoot(CopyNode);
7848   }
7849 }
7850 
7851 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
7852                              SelectionDAGBuilder &Builder) {
7853   // Check to see if this load can be trivially constant folded, e.g. if the
7854   // input is from a string literal.
7855   if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
7856     // Cast pointer to the type we really want to load.
7857     Type *LoadTy =
7858         Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
7859     if (LoadVT.isVector())
7860       LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
7861 
7862     LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
7863                                          PointerType::getUnqual(LoadTy));
7864 
7865     if (const Constant *LoadCst =
7866             ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
7867                                          LoadTy, Builder.DAG.getDataLayout()))
7868       return Builder.getValue(LoadCst);
7869   }
7870 
7871   // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
7872   // still constant memory, the input chain can be the entry node.
7873   SDValue Root;
7874   bool ConstantMemory = false;
7875 
7876   // Do not serialize (non-volatile) loads of constant memory with anything.
7877   if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
7878     Root = Builder.DAG.getEntryNode();
7879     ConstantMemory = true;
7880   } else {
7881     // Do not serialize non-volatile loads against each other.
7882     Root = Builder.DAG.getRoot();
7883   }
7884 
7885   SDValue Ptr = Builder.getValue(PtrVal);
7886   SDValue LoadVal =
7887       Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
7888                           MachinePointerInfo(PtrVal), Align(1));
7889 
7890   if (!ConstantMemory)
7891     Builder.PendingLoads.push_back(LoadVal.getValue(1));
7892   return LoadVal;
7893 }
7894 
7895 /// Record the value for an instruction that produces an integer result,
7896 /// converting the type where necessary.
7897 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
7898                                                   SDValue Value,
7899                                                   bool IsSigned) {
7900   EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7901                                                     I.getType(), true);
7902   if (IsSigned)
7903     Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
7904   else
7905     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
7906   setValue(&I, Value);
7907 }
7908 
7909 /// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
7910 /// true and lower it. Otherwise return false, and it will be lowered like a
7911 /// normal call.
7912 /// The caller already checked that \p I calls the appropriate LibFunc with a
7913 /// correct prototype.
7914 bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
7915   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
7916   const Value *Size = I.getArgOperand(2);
7917   const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
7918   if (CSize && CSize->getZExtValue() == 0) {
7919     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
7920                                                           I.getType(), true);
7921     setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
7922     return true;
7923   }
7924 
7925   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7926   std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
7927       DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
7928       getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
7929   if (Res.first.getNode()) {
7930     processIntegerCallValue(I, Res.first, true);
7931     PendingLoads.push_back(Res.second);
7932     return true;
7933   }
7934 
7935   // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
7936   // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
7937   if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
7938     return false;
7939 
7940   // If the target has a fast compare for the given size, it will return a
7941   // preferred load type for that size. Require that the load VT is legal and
7942   // that the target supports unaligned loads of that type. Otherwise, return
7943   // INVALID.
7944   auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
7945     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7946     MVT LVT = TLI.hasFastEqualityCompare(NumBits);
7947     if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
7948       // TODO: Handle 5 byte compare as 4-byte + 1 byte.
7949       // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
7950       // TODO: Check alignment of src and dest ptrs.
7951       unsigned DstAS = LHS->getType()->getPointerAddressSpace();
7952       unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
7953       if (!TLI.isTypeLegal(LVT) ||
7954           !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
7955           !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
7956         LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
7957     }
7958 
7959     return LVT;
7960   };
7961 
7962   // This turns into unaligned loads. We only do this if the target natively
7963   // supports the MVT we'll be loading or if it is small enough (<= 4) that
7964   // we'll only produce a small number of byte loads.
7965   MVT LoadVT;
7966   unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
7967   switch (NumBitsToCompare) {
7968   default:
7969     return false;
7970   case 16:
7971     LoadVT = MVT::i16;
7972     break;
7973   case 32:
7974     LoadVT = MVT::i32;
7975     break;
7976   case 64:
7977   case 128:
7978   case 256:
7979     LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
7980     break;
7981   }
7982 
7983   if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
7984     return false;
7985 
7986   SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
7987   SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
7988 
7989   // Bitcast to a wide integer type if the loads are vectors.
7990   if (LoadVT.isVector()) {
7991     EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
7992     LoadL = DAG.getBitcast(CmpVT, LoadL);
7993     LoadR = DAG.getBitcast(CmpVT, LoadR);
7994   }
7995 
7996   SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
7997   processIntegerCallValue(I, Cmp, false);
7998   return true;
7999 }
8000 
8001 /// See if we can lower a memchr call into an optimized form. If so, return
8002 /// true and lower it. Otherwise return false, and it will be lowered like a
8003 /// normal call.
8004 /// The caller already checked that \p I calls the appropriate LibFunc with a
8005 /// correct prototype.
8006 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
8007   const Value *Src = I.getArgOperand(0);
8008   const Value *Char = I.getArgOperand(1);
8009   const Value *Length = I.getArgOperand(2);
8010 
8011   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8012   std::pair<SDValue, SDValue> Res =
8013     TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
8014                                 getValue(Src), getValue(Char), getValue(Length),
8015                                 MachinePointerInfo(Src));
8016   if (Res.first.getNode()) {
8017     setValue(&I, Res.first);
8018     PendingLoads.push_back(Res.second);
8019     return true;
8020   }
8021 
8022   return false;
8023 }
8024 
8025 /// See if we can lower a mempcpy call into an optimized form. If so, return
8026 /// true and lower it. Otherwise return false, and it will be lowered like a
8027 /// normal call.
8028 /// The caller already checked that \p I calls the appropriate LibFunc with a
8029 /// correct prototype.
8030 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
8031   SDValue Dst = getValue(I.getArgOperand(0));
8032   SDValue Src = getValue(I.getArgOperand(1));
8033   SDValue Size = getValue(I.getArgOperand(2));
8034 
8035   Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
8036   Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
8037   // DAG::getMemcpy needs Alignment to be defined.
8038   Align Alignment = std::min(DstAlign, SrcAlign);
8039 
8040   bool isVol = false;
8041   SDLoc sdl = getCurSDLoc();
8042 
8043   // In the mempcpy context we need to pass in a false value for isTailCall
8044   // because the return pointer needs to be adjusted by the size of
8045   // the copied memory.
8046   SDValue Root = isVol ? getRoot() : getMemoryRoot();
8047   SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false,
8048                              /*isTailCall=*/false,
8049                              MachinePointerInfo(I.getArgOperand(0)),
8050                              MachinePointerInfo(I.getArgOperand(1)),
8051                              I.getAAMetadata());
8052   assert(MC.getNode() != nullptr &&
8053          "** memcpy should not be lowered as TailCall in mempcpy context **");
8054   DAG.setRoot(MC);
8055 
8056   // Check if Size needs to be truncated or extended.
8057   Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
8058 
8059   // Adjust return pointer to point just past the last dst byte.
8060   SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
8061                                     Dst, Size);
8062   setValue(&I, DstPlusSize);
8063   return true;
8064 }
8065 
8066 /// See if we can lower a strcpy call into an optimized form.  If so, return
8067 /// true and lower it, otherwise return false and it will be lowered like a
8068 /// normal call.
8069 /// The caller already checked that \p I calls the appropriate LibFunc with a
8070 /// correct prototype.
8071 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
8072   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8073 
8074   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8075   std::pair<SDValue, SDValue> Res =
8076     TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
8077                                 getValue(Arg0), getValue(Arg1),
8078                                 MachinePointerInfo(Arg0),
8079                                 MachinePointerInfo(Arg1), isStpcpy);
8080   if (Res.first.getNode()) {
8081     setValue(&I, Res.first);
8082     DAG.setRoot(Res.second);
8083     return true;
8084   }
8085 
8086   return false;
8087 }
8088 
8089 /// See if we can lower a strcmp call into an optimized form.  If so, return
8090 /// true and lower it, otherwise return false and it will be lowered like a
8091 /// normal call.
8092 /// The caller already checked that \p I calls the appropriate LibFunc with a
8093 /// correct prototype.
8094 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
8095   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8096 
8097   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8098   std::pair<SDValue, SDValue> Res =
8099     TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
8100                                 getValue(Arg0), getValue(Arg1),
8101                                 MachinePointerInfo(Arg0),
8102                                 MachinePointerInfo(Arg1));
8103   if (Res.first.getNode()) {
8104     processIntegerCallValue(I, Res.first, true);
8105     PendingLoads.push_back(Res.second);
8106     return true;
8107   }
8108 
8109   return false;
8110 }
8111 
8112 /// See if we can lower a strlen call into an optimized form.  If so, return
8113 /// true and lower it, otherwise return false and it will be lowered like a
8114 /// normal call.
8115 /// The caller already checked that \p I calls the appropriate LibFunc with a
8116 /// correct prototype.
8117 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
8118   const Value *Arg0 = I.getArgOperand(0);
8119 
8120   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8121   std::pair<SDValue, SDValue> Res =
8122     TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
8123                                 getValue(Arg0), MachinePointerInfo(Arg0));
8124   if (Res.first.getNode()) {
8125     processIntegerCallValue(I, Res.first, false);
8126     PendingLoads.push_back(Res.second);
8127     return true;
8128   }
8129 
8130   return false;
8131 }
8132 
8133 /// See if we can lower a strnlen call into an optimized form.  If so, return
8134 /// true and lower it, otherwise return false and it will be lowered like a
8135 /// normal call.
8136 /// The caller already checked that \p I calls the appropriate LibFunc with a
8137 /// correct prototype.
8138 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
8139   const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8140 
8141   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8142   std::pair<SDValue, SDValue> Res =
8143     TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
8144                                  getValue(Arg0), getValue(Arg1),
8145                                  MachinePointerInfo(Arg0));
8146   if (Res.first.getNode()) {
8147     processIntegerCallValue(I, Res.first, false);
8148     PendingLoads.push_back(Res.second);
8149     return true;
8150   }
8151 
8152   return false;
8153 }
8154 
8155 /// See if we can lower a unary floating-point operation into an SDNode with
8156 /// the specified Opcode.  If so, return true and lower it, otherwise return
8157 /// false and it will be lowered like a normal call.
8158 /// The caller already checked that \p I calls the appropriate LibFunc with a
8159 /// correct prototype.
8160 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
8161                                               unsigned Opcode) {
8162   // We already checked this call's prototype; verify it doesn't modify errno.
8163   if (!I.onlyReadsMemory())
8164     return false;
8165 
8166   SDNodeFlags Flags;
8167   Flags.copyFMF(cast<FPMathOperator>(I));
8168 
8169   SDValue Tmp = getValue(I.getArgOperand(0));
8170   setValue(&I,
8171            DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
8172   return true;
8173 }
8174 
8175 /// See if we can lower a binary floating-point operation into an SDNode with
8176 /// the specified Opcode. If so, return true and lower it. Otherwise return
8177 /// false, and it will be lowered like a normal call.
8178 /// The caller already checked that \p I calls the appropriate LibFunc with a
8179 /// correct prototype.
8180 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
8181                                                unsigned Opcode) {
8182   // We already checked this call's prototype; verify it doesn't modify errno.
8183   if (!I.onlyReadsMemory())
8184     return false;
8185 
8186   SDNodeFlags Flags;
8187   Flags.copyFMF(cast<FPMathOperator>(I));
8188 
8189   SDValue Tmp0 = getValue(I.getArgOperand(0));
8190   SDValue Tmp1 = getValue(I.getArgOperand(1));
8191   EVT VT = Tmp0.getValueType();
8192   setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
8193   return true;
8194 }
8195 
8196 void SelectionDAGBuilder::visitCall(const CallInst &I) {
8197   // Handle inline assembly differently.
8198   if (I.isInlineAsm()) {
8199     visitInlineAsm(I);
8200     return;
8201   }
8202 
8203   if (Function *F = I.getCalledFunction()) {
8204     diagnoseDontCall(I);
8205 
8206     if (F->isDeclaration()) {
8207       // Is this an LLVM intrinsic or a target-specific intrinsic?
8208       unsigned IID = F->getIntrinsicID();
8209       if (!IID)
8210         if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
8211           IID = II->getIntrinsicID(F);
8212 
8213       if (IID) {
8214         visitIntrinsicCall(I, IID);
8215         return;
8216       }
8217     }
8218 
8219     // Check for well-known libc/libm calls.  If the function is internal, it
8220     // can't be a library call.  Don't do the check if marked as nobuiltin for
8221     // some reason or the call site requires strict floating point semantics.
8222     LibFunc Func;
8223     if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
8224         F->hasName() && LibInfo->getLibFunc(*F, Func) &&
8225         LibInfo->hasOptimizedCodeGen(Func)) {
8226       switch (Func) {
8227       default: break;
8228       case LibFunc_bcmp:
8229         if (visitMemCmpBCmpCall(I))
8230           return;
8231         break;
8232       case LibFunc_copysign:
8233       case LibFunc_copysignf:
8234       case LibFunc_copysignl:
8235         // We already checked this call's prototype; verify it doesn't modify
8236         // errno.
8237         if (I.onlyReadsMemory()) {
8238           SDValue LHS = getValue(I.getArgOperand(0));
8239           SDValue RHS = getValue(I.getArgOperand(1));
8240           setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
8241                                    LHS.getValueType(), LHS, RHS));
8242           return;
8243         }
8244         break;
8245       case LibFunc_fabs:
8246       case LibFunc_fabsf:
8247       case LibFunc_fabsl:
8248         if (visitUnaryFloatCall(I, ISD::FABS))
8249           return;
8250         break;
8251       case LibFunc_fmin:
8252       case LibFunc_fminf:
8253       case LibFunc_fminl:
8254         if (visitBinaryFloatCall(I, ISD::FMINNUM))
8255           return;
8256         break;
8257       case LibFunc_fmax:
8258       case LibFunc_fmaxf:
8259       case LibFunc_fmaxl:
8260         if (visitBinaryFloatCall(I, ISD::FMAXNUM))
8261           return;
8262         break;
8263       case LibFunc_sin:
8264       case LibFunc_sinf:
8265       case LibFunc_sinl:
8266         if (visitUnaryFloatCall(I, ISD::FSIN))
8267           return;
8268         break;
8269       case LibFunc_cos:
8270       case LibFunc_cosf:
8271       case LibFunc_cosl:
8272         if (visitUnaryFloatCall(I, ISD::FCOS))
8273           return;
8274         break;
8275       case LibFunc_sqrt:
8276       case LibFunc_sqrtf:
8277       case LibFunc_sqrtl:
8278       case LibFunc_sqrt_finite:
8279       case LibFunc_sqrtf_finite:
8280       case LibFunc_sqrtl_finite:
8281         if (visitUnaryFloatCall(I, ISD::FSQRT))
8282           return;
8283         break;
8284       case LibFunc_floor:
8285       case LibFunc_floorf:
8286       case LibFunc_floorl:
8287         if (visitUnaryFloatCall(I, ISD::FFLOOR))
8288           return;
8289         break;
8290       case LibFunc_nearbyint:
8291       case LibFunc_nearbyintf:
8292       case LibFunc_nearbyintl:
8293         if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
8294           return;
8295         break;
8296       case LibFunc_ceil:
8297       case LibFunc_ceilf:
8298       case LibFunc_ceill:
8299         if (visitUnaryFloatCall(I, ISD::FCEIL))
8300           return;
8301         break;
8302       case LibFunc_rint:
8303       case LibFunc_rintf:
8304       case LibFunc_rintl:
8305         if (visitUnaryFloatCall(I, ISD::FRINT))
8306           return;
8307         break;
8308       case LibFunc_round:
8309       case LibFunc_roundf:
8310       case LibFunc_roundl:
8311         if (visitUnaryFloatCall(I, ISD::FROUND))
8312           return;
8313         break;
8314       case LibFunc_trunc:
8315       case LibFunc_truncf:
8316       case LibFunc_truncl:
8317         if (visitUnaryFloatCall(I, ISD::FTRUNC))
8318           return;
8319         break;
8320       case LibFunc_log2:
8321       case LibFunc_log2f:
8322       case LibFunc_log2l:
8323         if (visitUnaryFloatCall(I, ISD::FLOG2))
8324           return;
8325         break;
8326       case LibFunc_exp2:
8327       case LibFunc_exp2f:
8328       case LibFunc_exp2l:
8329         if (visitUnaryFloatCall(I, ISD::FEXP2))
8330           return;
8331         break;
8332       case LibFunc_memcmp:
8333         if (visitMemCmpBCmpCall(I))
8334           return;
8335         break;
8336       case LibFunc_mempcpy:
8337         if (visitMemPCpyCall(I))
8338           return;
8339         break;
8340       case LibFunc_memchr:
8341         if (visitMemChrCall(I))
8342           return;
8343         break;
8344       case LibFunc_strcpy:
8345         if (visitStrCpyCall(I, false))
8346           return;
8347         break;
8348       case LibFunc_stpcpy:
8349         if (visitStrCpyCall(I, true))
8350           return;
8351         break;
8352       case LibFunc_strcmp:
8353         if (visitStrCmpCall(I))
8354           return;
8355         break;
8356       case LibFunc_strlen:
8357         if (visitStrLenCall(I))
8358           return;
8359         break;
8360       case LibFunc_strnlen:
8361         if (visitStrNLenCall(I))
8362           return;
8363         break;
8364       }
8365     }
8366   }
8367 
8368   // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
8369   // have to do anything here to lower funclet bundles.
8370   // CFGuardTarget bundles are lowered in LowerCallTo.
8371   assert(!I.hasOperandBundlesOtherThan(
8372              {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
8373               LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
8374               LLVMContext::OB_clang_arc_attachedcall}) &&
8375          "Cannot lower calls with arbitrary operand bundles!");
8376 
8377   SDValue Callee = getValue(I.getCalledOperand());
8378 
8379   if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
8380     LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
8381   else
8382     // Check if we can potentially perform a tail call. More detailed checking
8383     // is be done within LowerCallTo, after more information about the call is
8384     // known.
8385     LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
8386 }
8387 
8388 namespace {
8389 
8390 /// AsmOperandInfo - This contains information for each constraint that we are
8391 /// lowering.
8392 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
8393 public:
8394   /// CallOperand - If this is the result output operand or a clobber
8395   /// this is null, otherwise it is the incoming operand to the CallInst.
8396   /// This gets modified as the asm is processed.
8397   SDValue CallOperand;
8398 
8399   /// AssignedRegs - If this is a register or register class operand, this
8400   /// contains the set of register corresponding to the operand.
8401   RegsForValue AssignedRegs;
8402 
8403   explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
8404     : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
8405   }
8406 
8407   /// Whether or not this operand accesses memory
8408   bool hasMemory(const TargetLowering &TLI) const {
8409     // Indirect operand accesses access memory.
8410     if (isIndirect)
8411       return true;
8412 
8413     for (const auto &Code : Codes)
8414       if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
8415         return true;
8416 
8417     return false;
8418   }
8419 };
8420 
8421 
8422 } // end anonymous namespace
8423 
8424 /// Make sure that the output operand \p OpInfo and its corresponding input
8425 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error
8426 /// out).
8427 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
8428                                SDISelAsmOperandInfo &MatchingOpInfo,
8429                                SelectionDAG &DAG) {
8430   if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
8431     return;
8432 
8433   const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
8434   const auto &TLI = DAG.getTargetLoweringInfo();
8435 
8436   std::pair<unsigned, const TargetRegisterClass *> MatchRC =
8437       TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
8438                                        OpInfo.ConstraintVT);
8439   std::pair<unsigned, const TargetRegisterClass *> InputRC =
8440       TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
8441                                        MatchingOpInfo.ConstraintVT);
8442   if ((OpInfo.ConstraintVT.isInteger() !=
8443        MatchingOpInfo.ConstraintVT.isInteger()) ||
8444       (MatchRC.second != InputRC.second)) {
8445     // FIXME: error out in a more elegant fashion
8446     report_fatal_error("Unsupported asm: input constraint"
8447                        " with a matching output constraint of"
8448                        " incompatible type!");
8449   }
8450   MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
8451 }
8452 
8453 /// Get a direct memory input to behave well as an indirect operand.
8454 /// This may introduce stores, hence the need for a \p Chain.
8455 /// \return The (possibly updated) chain.
8456 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
8457                                         SDISelAsmOperandInfo &OpInfo,
8458                                         SelectionDAG &DAG) {
8459   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8460 
8461   // If we don't have an indirect input, put it in the constpool if we can,
8462   // otherwise spill it to a stack slot.
8463   // TODO: This isn't quite right. We need to handle these according to
8464   // the addressing mode that the constraint wants. Also, this may take
8465   // an additional register for the computation and we don't want that
8466   // either.
8467 
8468   // If the operand is a float, integer, or vector constant, spill to a
8469   // constant pool entry to get its address.
8470   const Value *OpVal = OpInfo.CallOperandVal;
8471   if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
8472       isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
8473     OpInfo.CallOperand = DAG.getConstantPool(
8474         cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
8475     return Chain;
8476   }
8477 
8478   // Otherwise, create a stack slot and emit a store to it before the asm.
8479   Type *Ty = OpVal->getType();
8480   auto &DL = DAG.getDataLayout();
8481   uint64_t TySize = DL.getTypeAllocSize(Ty);
8482   MachineFunction &MF = DAG.getMachineFunction();
8483   int SSFI = MF.getFrameInfo().CreateStackObject(
8484       TySize, DL.getPrefTypeAlign(Ty), false);
8485   SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
8486   Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
8487                             MachinePointerInfo::getFixedStack(MF, SSFI),
8488                             TLI.getMemValueType(DL, Ty));
8489   OpInfo.CallOperand = StackSlot;
8490 
8491   return Chain;
8492 }
8493 
8494 /// GetRegistersForValue - Assign registers (virtual or physical) for the
8495 /// specified operand.  We prefer to assign virtual registers, to allow the
8496 /// register allocator to handle the assignment process.  However, if the asm
8497 /// uses features that we can't model on machineinstrs, we have SDISel do the
8498 /// allocation.  This produces generally horrible, but correct, code.
8499 ///
8500 ///   OpInfo describes the operand
8501 ///   RefOpInfo describes the matching operand if any, the operand otherwise
8502 static llvm::Optional<unsigned>
8503 getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
8504                      SDISelAsmOperandInfo &OpInfo,
8505                      SDISelAsmOperandInfo &RefOpInfo) {
8506   LLVMContext &Context = *DAG.getContext();
8507   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8508 
8509   MachineFunction &MF = DAG.getMachineFunction();
8510   SmallVector<unsigned, 4> Regs;
8511   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8512 
8513   // No work to do for memory/address operands.
8514   if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8515       OpInfo.ConstraintType == TargetLowering::C_Address)
8516     return None;
8517 
8518   // If this is a constraint for a single physreg, or a constraint for a
8519   // register class, find it.
8520   unsigned AssignedReg;
8521   const TargetRegisterClass *RC;
8522   std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
8523       &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
8524   // RC is unset only on failure. Return immediately.
8525   if (!RC)
8526     return None;
8527 
8528   // Get the actual register value type.  This is important, because the user
8529   // may have asked for (e.g.) the AX register in i32 type.  We need to
8530   // remember that AX is actually i16 to get the right extension.
8531   const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
8532 
8533   if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
8534     // If this is an FP operand in an integer register (or visa versa), or more
8535     // generally if the operand value disagrees with the register class we plan
8536     // to stick it in, fix the operand type.
8537     //
8538     // If this is an input value, the bitcast to the new type is done now.
8539     // Bitcast for output value is done at the end of visitInlineAsm().
8540     if ((OpInfo.Type == InlineAsm::isOutput ||
8541          OpInfo.Type == InlineAsm::isInput) &&
8542         !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
8543       // Try to convert to the first EVT that the reg class contains.  If the
8544       // types are identical size, use a bitcast to convert (e.g. two differing
8545       // vector types).  Note: output bitcast is done at the end of
8546       // visitInlineAsm().
8547       if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
8548         // Exclude indirect inputs while they are unsupported because the code
8549         // to perform the load is missing and thus OpInfo.CallOperand still
8550         // refers to the input address rather than the pointed-to value.
8551         if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
8552           OpInfo.CallOperand =
8553               DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
8554         OpInfo.ConstraintVT = RegVT;
8555         // If the operand is an FP value and we want it in integer registers,
8556         // use the corresponding integer type. This turns an f64 value into
8557         // i64, which can be passed with two i32 values on a 32-bit machine.
8558       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
8559         MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
8560         if (OpInfo.Type == InlineAsm::isInput)
8561           OpInfo.CallOperand =
8562               DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
8563         OpInfo.ConstraintVT = VT;
8564       }
8565     }
8566   }
8567 
8568   // No need to allocate a matching input constraint since the constraint it's
8569   // matching to has already been allocated.
8570   if (OpInfo.isMatchingInputConstraint())
8571     return None;
8572 
8573   EVT ValueVT = OpInfo.ConstraintVT;
8574   if (OpInfo.ConstraintVT == MVT::Other)
8575     ValueVT = RegVT;
8576 
8577   // Initialize NumRegs.
8578   unsigned NumRegs = 1;
8579   if (OpInfo.ConstraintVT != MVT::Other)
8580     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
8581 
8582   // If this is a constraint for a specific physical register, like {r17},
8583   // assign it now.
8584 
8585   // If this associated to a specific register, initialize iterator to correct
8586   // place. If virtual, make sure we have enough registers
8587 
8588   // Initialize iterator if necessary
8589   TargetRegisterClass::iterator I = RC->begin();
8590   MachineRegisterInfo &RegInfo = MF.getRegInfo();
8591 
8592   // Do not check for single registers.
8593   if (AssignedReg) {
8594     I = std::find(I, RC->end(), AssignedReg);
8595     if (I == RC->end()) {
8596       // RC does not contain the selected register, which indicates a
8597       // mismatch between the register and the required type/bitwidth.
8598       return {AssignedReg};
8599     }
8600   }
8601 
8602   for (; NumRegs; --NumRegs, ++I) {
8603     assert(I != RC->end() && "Ran out of registers to allocate!");
8604     Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
8605     Regs.push_back(R);
8606   }
8607 
8608   OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
8609   return None;
8610 }
8611 
8612 static unsigned
8613 findMatchingInlineAsmOperand(unsigned OperandNo,
8614                              const std::vector<SDValue> &AsmNodeOperands) {
8615   // Scan until we find the definition we already emitted of this operand.
8616   unsigned CurOp = InlineAsm::Op_FirstOperand;
8617   for (; OperandNo; --OperandNo) {
8618     // Advance to the next operand.
8619     unsigned OpFlag =
8620         cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
8621     assert((InlineAsm::isRegDefKind(OpFlag) ||
8622             InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||
8623             InlineAsm::isMemKind(OpFlag)) &&
8624            "Skipped past definitions?");
8625     CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
8626   }
8627   return CurOp;
8628 }
8629 
8630 namespace {
8631 
8632 class ExtraFlags {
8633   unsigned Flags = 0;
8634 
8635 public:
8636   explicit ExtraFlags(const CallBase &Call) {
8637     const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
8638     if (IA->hasSideEffects())
8639       Flags |= InlineAsm::Extra_HasSideEffects;
8640     if (IA->isAlignStack())
8641       Flags |= InlineAsm::Extra_IsAlignStack;
8642     if (Call.isConvergent())
8643       Flags |= InlineAsm::Extra_IsConvergent;
8644     Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
8645   }
8646 
8647   void update(const TargetLowering::AsmOperandInfo &OpInfo) {
8648     // Ideally, we would only check against memory constraints.  However, the
8649     // meaning of an Other constraint can be target-specific and we can't easily
8650     // reason about it.  Therefore, be conservative and set MayLoad/MayStore
8651     // for Other constraints as well.
8652     if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8653         OpInfo.ConstraintType == TargetLowering::C_Other) {
8654       if (OpInfo.Type == InlineAsm::isInput)
8655         Flags |= InlineAsm::Extra_MayLoad;
8656       else if (OpInfo.Type == InlineAsm::isOutput)
8657         Flags |= InlineAsm::Extra_MayStore;
8658       else if (OpInfo.Type == InlineAsm::isClobber)
8659         Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
8660     }
8661   }
8662 
8663   unsigned get() const { return Flags; }
8664 };
8665 
8666 } // end anonymous namespace
8667 
8668 /// visitInlineAsm - Handle a call to an InlineAsm object.
8669 void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
8670                                          const BasicBlock *EHPadBB) {
8671   const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
8672 
8673   /// ConstraintOperands - Information about all of the constraints.
8674   SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
8675 
8676   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8677   TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
8678       DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);
8679 
8680   // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
8681   // AsmDialect, MayLoad, MayStore).
8682   bool HasSideEffect = IA->hasSideEffects();
8683   ExtraFlags ExtraInfo(Call);
8684 
8685   for (auto &T : TargetConstraints) {
8686     ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
8687     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
8688 
8689     if (OpInfo.CallOperandVal)
8690       OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
8691 
8692     if (!HasSideEffect)
8693       HasSideEffect = OpInfo.hasMemory(TLI);
8694 
8695     // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
8696     // FIXME: Could we compute this on OpInfo rather than T?
8697 
8698     // Compute the constraint code and ConstraintType to use.
8699     TLI.ComputeConstraintToUse(T, SDValue());
8700 
8701     if (T.ConstraintType == TargetLowering::C_Immediate &&
8702         OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
8703       // We've delayed emitting a diagnostic like the "n" constraint because
8704       // inlining could cause an integer showing up.
8705       return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
8706                                           "' expects an integer constant "
8707                                           "expression");
8708 
8709     ExtraInfo.update(T);
8710   }
8711 
8712   // We won't need to flush pending loads if this asm doesn't touch
8713   // memory and is nonvolatile.
8714   SDValue Flag, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
8715 
8716   bool EmitEHLabels = isa<InvokeInst>(Call) && IA->canThrow();
8717   if (EmitEHLabels) {
8718     assert(EHPadBB && "InvokeInst must have an EHPadBB");
8719   }
8720   bool IsCallBr = isa<CallBrInst>(Call);
8721 
8722   if (IsCallBr || EmitEHLabels) {
8723     // If this is a callbr or invoke we need to flush pending exports since
8724     // inlineasm_br and invoke are terminators.
8725     // We need to do this before nodes are glued to the inlineasm_br node.
8726     Chain = getControlRoot();
8727   }
8728 
8729   MCSymbol *BeginLabel = nullptr;
8730   if (EmitEHLabels) {
8731     Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
8732   }
8733 
8734   // Second pass over the constraints: compute which constraint option to use.
8735   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
8736     // If this is an output operand with a matching input operand, look up the
8737     // matching input. If their types mismatch, e.g. one is an integer, the
8738     // other is floating point, or their sizes are different, flag it as an
8739     // error.
8740     if (OpInfo.hasMatchingInput()) {
8741       SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
8742       patchMatchingInput(OpInfo, Input, DAG);
8743     }
8744 
8745     // Compute the constraint code and ConstraintType to use.
8746     TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
8747 
8748     if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
8749          OpInfo.Type == InlineAsm::isClobber) ||
8750         OpInfo.ConstraintType == TargetLowering::C_Address)
8751       continue;
8752 
8753     // If this is a memory input, and if the operand is not indirect, do what we
8754     // need to provide an address for the memory input.
8755     if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
8756         !OpInfo.isIndirect) {
8757       assert((OpInfo.isMultipleAlternative ||
8758               (OpInfo.Type == InlineAsm::isInput)) &&
8759              "Can only indirectify direct input operands!");
8760 
8761       // Memory operands really want the address of the value.
8762       Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
8763 
8764       // There is no longer a Value* corresponding to this operand.
8765       OpInfo.CallOperandVal = nullptr;
8766 
8767       // It is now an indirect operand.
8768       OpInfo.isIndirect = true;
8769     }
8770 
8771   }
8772 
8773   // AsmNodeOperands - The operands for the ISD::INLINEASM node.
8774   std::vector<SDValue> AsmNodeOperands;
8775   AsmNodeOperands.push_back(SDValue());  // reserve space for input chain
8776   AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
8777       IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
8778 
8779   // If we have a !srcloc metadata node associated with it, we want to attach
8780   // this to the ultimately generated inline asm machineinstr.  To do this, we
8781   // pass in the third operand as this (potentially null) inline asm MDNode.
8782   const MDNode *SrcLoc = Call.getMetadata("srcloc");
8783   AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
8784 
8785   // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
8786   // bits as operand 3.
8787   AsmNodeOperands.push_back(DAG.getTargetConstant(
8788       ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8789 
8790   // Third pass: Loop over operands to prepare DAG-level operands.. As part of
8791   // this, assign virtual and physical registers for inputs and otput.
8792   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
8793     // Assign Registers.
8794     SDISelAsmOperandInfo &RefOpInfo =
8795         OpInfo.isMatchingInputConstraint()
8796             ? ConstraintOperands[OpInfo.getMatchedOperand()]
8797             : OpInfo;
8798     const auto RegError =
8799         getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
8800     if (RegError) {
8801       const MachineFunction &MF = DAG.getMachineFunction();
8802       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8803       const char *RegName = TRI.getName(RegError.value());
8804       emitInlineAsmError(Call, "register '" + Twine(RegName) +
8805                                    "' allocated for constraint '" +
8806                                    Twine(OpInfo.ConstraintCode) +
8807                                    "' does not match required type");
8808       return;
8809     }
8810 
8811     auto DetectWriteToReservedRegister = [&]() {
8812       const MachineFunction &MF = DAG.getMachineFunction();
8813       const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8814       for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
8815         if (Register::isPhysicalRegister(Reg) &&
8816             TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
8817           const char *RegName = TRI.getName(Reg);
8818           emitInlineAsmError(Call, "write to reserved register '" +
8819                                        Twine(RegName) + "'");
8820           return true;
8821         }
8822       }
8823       return false;
8824     };
8825     assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
8826             (OpInfo.Type == InlineAsm::isInput &&
8827              !OpInfo.isMatchingInputConstraint())) &&
8828            "Only address as input operand is allowed.");
8829 
8830     switch (OpInfo.Type) {
8831     case InlineAsm::isOutput:
8832       if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
8833         unsigned ConstraintID =
8834             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
8835         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
8836                "Failed to convert memory constraint code to constraint id.");
8837 
8838         // Add information to the INLINEASM node to know about this output.
8839         unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
8840         OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
8841         AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
8842                                                         MVT::i32));
8843         AsmNodeOperands.push_back(OpInfo.CallOperand);
8844       } else {
8845         // Otherwise, this outputs to a register (directly for C_Register /
8846         // C_RegisterClass, and a target-defined fashion for
8847         // C_Immediate/C_Other). Find a register that we can use.
8848         if (OpInfo.AssignedRegs.Regs.empty()) {
8849           emitInlineAsmError(
8850               Call, "couldn't allocate output register for constraint '" +
8851                         Twine(OpInfo.ConstraintCode) + "'");
8852           return;
8853         }
8854 
8855         if (DetectWriteToReservedRegister())
8856           return;
8857 
8858         // Add information to the INLINEASM node to know that this register is
8859         // set.
8860         OpInfo.AssignedRegs.AddInlineAsmOperands(
8861             OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber
8862                                   : InlineAsm::Kind_RegDef,
8863             false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
8864       }
8865       break;
8866 
8867     case InlineAsm::isInput:
8868     case InlineAsm::isLabel: {
8869       SDValue InOperandVal = OpInfo.CallOperand;
8870 
8871       if (OpInfo.isMatchingInputConstraint()) {
8872         // If this is required to match an output register we have already set,
8873         // just use its register.
8874         auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
8875                                                   AsmNodeOperands);
8876         unsigned OpFlag =
8877           cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
8878         if (InlineAsm::isRegDefKind(OpFlag) ||
8879             InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
8880           // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
8881           if (OpInfo.isIndirect) {
8882             // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
8883             emitInlineAsmError(Call, "inline asm not supported yet: "
8884                                      "don't know how to handle tied "
8885                                      "indirect register inputs");
8886             return;
8887           }
8888 
8889           SmallVector<unsigned, 4> Regs;
8890           MachineFunction &MF = DAG.getMachineFunction();
8891           MachineRegisterInfo &MRI = MF.getRegInfo();
8892           const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8893           auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
8894           Register TiedReg = R->getReg();
8895           MVT RegVT = R->getSimpleValueType(0);
8896           const TargetRegisterClass *RC =
8897               TiedReg.isVirtual()     ? MRI.getRegClass(TiedReg)
8898               : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
8899                                       : TRI.getMinimalPhysRegClass(TiedReg);
8900           unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag);
8901           for (unsigned i = 0; i != NumRegs; ++i)
8902             Regs.push_back(MRI.createVirtualRegister(RC));
8903 
8904           RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
8905 
8906           SDLoc dl = getCurSDLoc();
8907           // Use the produced MatchedRegs object to
8908           MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag, &Call);
8909           MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
8910                                            true, OpInfo.getMatchedOperand(), dl,
8911                                            DAG, AsmNodeOperands);
8912           break;
8913         }
8914 
8915         assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!");
8916         assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&
8917                "Unexpected number of operands");
8918         // Add information to the INLINEASM node to know about this input.
8919         // See InlineAsm.h isUseOperandTiedToDef.
8920         OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
8921         OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
8922                                                     OpInfo.getMatchedOperand());
8923         AsmNodeOperands.push_back(DAG.getTargetConstant(
8924             OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8925         AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
8926         break;
8927       }
8928 
8929       // Treat indirect 'X' constraint as memory.
8930       if (OpInfo.ConstraintType == TargetLowering::C_Other &&
8931           OpInfo.isIndirect)
8932         OpInfo.ConstraintType = TargetLowering::C_Memory;
8933 
8934       if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
8935           OpInfo.ConstraintType == TargetLowering::C_Other) {
8936         std::vector<SDValue> Ops;
8937         TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
8938                                           Ops, DAG);
8939         if (Ops.empty()) {
8940           if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
8941             if (isa<ConstantSDNode>(InOperandVal)) {
8942               emitInlineAsmError(Call, "value out of range for constraint '" +
8943                                            Twine(OpInfo.ConstraintCode) + "'");
8944               return;
8945             }
8946 
8947           emitInlineAsmError(Call,
8948                              "invalid operand for inline asm constraint '" +
8949                                  Twine(OpInfo.ConstraintCode) + "'");
8950           return;
8951         }
8952 
8953         // Add information to the INLINEASM node to know about this input.
8954         unsigned ResOpType =
8955           InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
8956         AsmNodeOperands.push_back(DAG.getTargetConstant(
8957             ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
8958         llvm::append_range(AsmNodeOperands, Ops);
8959         break;
8960       }
8961 
8962       if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
8963           OpInfo.ConstraintType == TargetLowering::C_Address) {
8964         assert((OpInfo.isIndirect ||
8965                 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
8966                "Operand must be indirect to be a mem!");
8967         assert(InOperandVal.getValueType() ==
8968                    TLI.getPointerTy(DAG.getDataLayout()) &&
8969                "Memory operands expect pointer values");
8970 
8971         unsigned ConstraintID =
8972             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
8973         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
8974                "Failed to convert memory constraint code to constraint id.");
8975 
8976         // Add information to the INLINEASM node to know about this input.
8977         unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
8978         ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
8979         AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
8980                                                         getCurSDLoc(),
8981                                                         MVT::i32));
8982         AsmNodeOperands.push_back(InOperandVal);
8983         break;
8984       }
8985 
8986       assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
8987               OpInfo.ConstraintType == TargetLowering::C_Register) &&
8988              "Unknown constraint type!");
8989 
8990       // TODO: Support this.
8991       if (OpInfo.isIndirect) {
8992         emitInlineAsmError(
8993             Call, "Don't know how to handle indirect register inputs yet "
8994                   "for constraint '" +
8995                       Twine(OpInfo.ConstraintCode) + "'");
8996         return;
8997       }
8998 
8999       // Copy the input into the appropriate registers.
9000       if (OpInfo.AssignedRegs.Regs.empty()) {
9001         emitInlineAsmError(Call,
9002                            "couldn't allocate input reg for constraint '" +
9003                                Twine(OpInfo.ConstraintCode) + "'");
9004         return;
9005       }
9006 
9007       if (DetectWriteToReservedRegister())
9008         return;
9009 
9010       SDLoc dl = getCurSDLoc();
9011 
9012       OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag,
9013                                         &Call);
9014 
9015       OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
9016                                                dl, DAG, AsmNodeOperands);
9017       break;
9018     }
9019     case InlineAsm::isClobber:
9020       // Add the clobbered value to the operand list, so that the register
9021       // allocator is aware that the physreg got clobbered.
9022       if (!OpInfo.AssignedRegs.Regs.empty())
9023         OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
9024                                                  false, 0, getCurSDLoc(), DAG,
9025                                                  AsmNodeOperands);
9026       break;
9027     }
9028   }
9029 
9030   // Finish up input operands.  Set the input chain and add the flag last.
9031   AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
9032   if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
9033 
9034   unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
9035   Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
9036                       DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
9037   Flag = Chain.getValue(1);
9038 
9039   // Do additional work to generate outputs.
9040 
9041   SmallVector<EVT, 1> ResultVTs;
9042   SmallVector<SDValue, 1> ResultValues;
9043   SmallVector<SDValue, 8> OutChains;
9044 
9045   llvm::Type *CallResultType = Call.getType();
9046   ArrayRef<Type *> ResultTypes;
9047   if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
9048     ResultTypes = StructResult->elements();
9049   else if (!CallResultType->isVoidTy())
9050     ResultTypes = makeArrayRef(CallResultType);
9051 
9052   auto CurResultType = ResultTypes.begin();
9053   auto handleRegAssign = [&](SDValue V) {
9054     assert(CurResultType != ResultTypes.end() && "Unexpected value");
9055     assert((*CurResultType)->isSized() && "Unexpected unsized type");
9056     EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
9057     ++CurResultType;
9058     // If the type of the inline asm call site return value is different but has
9059     // same size as the type of the asm output bitcast it.  One example of this
9060     // is for vectors with different width / number of elements.  This can
9061     // happen for register classes that can contain multiple different value
9062     // types.  The preg or vreg allocated may not have the same VT as was
9063     // expected.
9064     //
9065     // This can also happen for a return value that disagrees with the register
9066     // class it is put in, eg. a double in a general-purpose register on a
9067     // 32-bit machine.
9068     if (ResultVT != V.getValueType() &&
9069         ResultVT.getSizeInBits() == V.getValueSizeInBits())
9070       V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
9071     else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
9072              V.getValueType().isInteger()) {
9073       // If a result value was tied to an input value, the computed result
9074       // may have a wider width than the expected result.  Extract the
9075       // relevant portion.
9076       V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
9077     }
9078     assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
9079     ResultVTs.push_back(ResultVT);
9080     ResultValues.push_back(V);
9081   };
9082 
9083   // Deal with output operands.
9084   for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9085     if (OpInfo.Type == InlineAsm::isOutput) {
9086       SDValue Val;
9087       // Skip trivial output operands.
9088       if (OpInfo.AssignedRegs.Regs.empty())
9089         continue;
9090 
9091       switch (OpInfo.ConstraintType) {
9092       case TargetLowering::C_Register:
9093       case TargetLowering::C_RegisterClass:
9094         Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
9095                                                   Chain, &Flag, &Call);
9096         break;
9097       case TargetLowering::C_Immediate:
9098       case TargetLowering::C_Other:
9099         Val = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
9100                                               OpInfo, DAG);
9101         break;
9102       case TargetLowering::C_Memory:
9103         break; // Already handled.
9104       case TargetLowering::C_Address:
9105         break; // Silence warning.
9106       case TargetLowering::C_Unknown:
9107         assert(false && "Unexpected unknown constraint");
9108       }
9109 
9110       // Indirect output manifest as stores. Record output chains.
9111       if (OpInfo.isIndirect) {
9112         const Value *Ptr = OpInfo.CallOperandVal;
9113         assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
9114         SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
9115                                      MachinePointerInfo(Ptr));
9116         OutChains.push_back(Store);
9117       } else {
9118         // generate CopyFromRegs to associated registers.
9119         assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
9120         if (Val.getOpcode() == ISD::MERGE_VALUES) {
9121           for (const SDValue &V : Val->op_values())
9122             handleRegAssign(V);
9123         } else
9124           handleRegAssign(Val);
9125       }
9126     }
9127   }
9128 
9129   // Set results.
9130   if (!ResultValues.empty()) {
9131     assert(CurResultType == ResultTypes.end() &&
9132            "Mismatch in number of ResultTypes");
9133     assert(ResultValues.size() == ResultTypes.size() &&
9134            "Mismatch in number of output operands in asm result");
9135 
9136     SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
9137                             DAG.getVTList(ResultVTs), ResultValues);
9138     setValue(&Call, V);
9139   }
9140 
9141   // Collect store chains.
9142   if (!OutChains.empty())
9143     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
9144 
9145   if (EmitEHLabels) {
9146     Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
9147   }
9148 
9149   // Only Update Root if inline assembly has a memory effect.
9150   if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
9151       EmitEHLabels)
9152     DAG.setRoot(Chain);
9153 }
9154 
9155 void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
9156                                              const Twine &Message) {
9157   LLVMContext &Ctx = *DAG.getContext();
9158   Ctx.emitError(&Call, Message);
9159 
9160   // Make sure we leave the DAG in a valid state
9161   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9162   SmallVector<EVT, 1> ValueVTs;
9163   ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
9164 
9165   if (ValueVTs.empty())
9166     return;
9167 
9168   SmallVector<SDValue, 1> Ops;
9169   for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
9170     Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
9171 
9172   setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
9173 }
9174 
9175 void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
9176   DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
9177                           MVT::Other, getRoot(),
9178                           getValue(I.getArgOperand(0)),
9179                           DAG.getSrcValue(I.getArgOperand(0))));
9180 }
9181 
9182 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
9183   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9184   const DataLayout &DL = DAG.getDataLayout();
9185   SDValue V = DAG.getVAArg(
9186       TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
9187       getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
9188       DL.getABITypeAlign(I.getType()).value());
9189   DAG.setRoot(V.getValue(1));
9190 
9191   if (I.getType()->isPointerTy())
9192     V = DAG.getPtrExtOrTrunc(
9193         V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
9194   setValue(&I, V);
9195 }
9196 
9197 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
9198   DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
9199                           MVT::Other, getRoot(),
9200                           getValue(I.getArgOperand(0)),
9201                           DAG.getSrcValue(I.getArgOperand(0))));
9202 }
9203 
9204 void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
9205   DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
9206                           MVT::Other, getRoot(),
9207                           getValue(I.getArgOperand(0)),
9208                           getValue(I.getArgOperand(1)),
9209                           DAG.getSrcValue(I.getArgOperand(0)),
9210                           DAG.getSrcValue(I.getArgOperand(1))));
9211 }
9212 
9213 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
9214                                                     const Instruction &I,
9215                                                     SDValue Op) {
9216   const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
9217   if (!Range)
9218     return Op;
9219 
9220   ConstantRange CR = getConstantRangeFromMetadata(*Range);
9221   if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
9222     return Op;
9223 
9224   APInt Lo = CR.getUnsignedMin();
9225   if (!Lo.isMinValue())
9226     return Op;
9227 
9228   APInt Hi = CR.getUnsignedMax();
9229   unsigned Bits = std::max(Hi.getActiveBits(),
9230                            static_cast<unsigned>(IntegerType::MIN_INT_BITS));
9231 
9232   EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
9233 
9234   SDLoc SL = getCurSDLoc();
9235 
9236   SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
9237                              DAG.getValueType(SmallVT));
9238   unsigned NumVals = Op.getNode()->getNumValues();
9239   if (NumVals == 1)
9240     return ZExt;
9241 
9242   SmallVector<SDValue, 4> Ops;
9243 
9244   Ops.push_back(ZExt);
9245   for (unsigned I = 1; I != NumVals; ++I)
9246     Ops.push_back(Op.getValue(I));
9247 
9248   return DAG.getMergeValues(Ops, SL);
9249 }
9250 
9251 /// Populate a CallLowerinInfo (into \p CLI) based on the properties of
9252 /// the call being lowered.
9253 ///
9254 /// This is a helper for lowering intrinsics that follow a target calling
9255 /// convention or require stack pointer adjustment. Only a subset of the
9256 /// intrinsic's operands need to participate in the calling convention.
9257 void SelectionDAGBuilder::populateCallLoweringInfo(
9258     TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
9259     unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
9260     bool IsPatchPoint) {
9261   TargetLowering::ArgListTy Args;
9262   Args.reserve(NumArgs);
9263 
9264   // Populate the argument list.
9265   // Attributes for args start at offset 1, after the return attribute.
9266   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
9267        ArgI != ArgE; ++ArgI) {
9268     const Value *V = Call->getOperand(ArgI);
9269 
9270     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
9271 
9272     TargetLowering::ArgListEntry Entry;
9273     Entry.Node = getValue(V);
9274     Entry.Ty = V->getType();
9275     Entry.setAttributes(Call, ArgI);
9276     Args.push_back(Entry);
9277   }
9278 
9279   CLI.setDebugLoc(getCurSDLoc())
9280       .setChain(getRoot())
9281       .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args))
9282       .setDiscardResult(Call->use_empty())
9283       .setIsPatchPoint(IsPatchPoint)
9284       .setIsPreallocated(
9285           Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
9286 }
9287 
9288 /// Add a stack map intrinsic call's live variable operands to a stackmap
9289 /// or patchpoint target node's operand list.
9290 ///
9291 /// Constants are converted to TargetConstants purely as an optimization to
9292 /// avoid constant materialization and register allocation.
9293 ///
9294 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
9295 /// generate addess computation nodes, and so FinalizeISel can convert the
9296 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
9297 /// address materialization and register allocation, but may also be required
9298 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
9299 /// alloca in the entry block, then the runtime may assume that the alloca's
9300 /// StackMap location can be read immediately after compilation and that the
9301 /// location is valid at any point during execution (this is similar to the
9302 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
9303 /// only available in a register, then the runtime would need to trap when
9304 /// execution reaches the StackMap in order to read the alloca's location.
9305 static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
9306                                 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
9307                                 SelectionDAGBuilder &Builder) {
9308   SelectionDAG &DAG = Builder.DAG;
9309   for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
9310     SDValue Op = Builder.getValue(Call.getArgOperand(I));
9311 
9312     // Things on the stack are pointer-typed, meaning that they are already
9313     // legal and can be emitted directly to target nodes.
9314     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
9315       Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
9316     } else {
9317       // Otherwise emit a target independent node to be legalised.
9318       Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
9319     }
9320   }
9321 }
9322 
9323 /// Lower llvm.experimental.stackmap.
9324 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
9325   // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
9326   //                                  [live variables...])
9327 
9328   assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
9329 
9330   SDValue Chain, InFlag, Callee, NullPtr;
9331   SmallVector<SDValue, 32> Ops;
9332 
9333   SDLoc DL = getCurSDLoc();
9334   Callee = getValue(CI.getCalledOperand());
9335   NullPtr = DAG.getIntPtrConstant(0, DL, true);
9336 
9337   // The stackmap intrinsic only records the live variables (the arguments
9338   // passed to it) and emits NOPS (if requested). Unlike the patchpoint
9339   // intrinsic, this won't be lowered to a function call. This means we don't
9340   // have to worry about calling conventions and target specific lowering code.
9341   // Instead we perform the call lowering right here.
9342   //
9343   // chain, flag = CALLSEQ_START(chain, 0, 0)
9344   // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
9345   // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
9346   //
9347   Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
9348   InFlag = Chain.getValue(1);
9349 
9350   // Add the STACKMAP operands, starting with DAG house-keeping.
9351   Ops.push_back(Chain);
9352   Ops.push_back(InFlag);
9353 
9354   // Add the <id>, <numShadowBytes> operands.
9355   //
9356   // These do not require legalisation, and can be emitted directly to target
9357   // constant nodes.
9358   SDValue ID = getValue(CI.getArgOperand(0));
9359   assert(ID.getValueType() == MVT::i64);
9360   SDValue IDConst = DAG.getTargetConstant(
9361       cast<ConstantSDNode>(ID)->getZExtValue(), DL, ID.getValueType());
9362   Ops.push_back(IDConst);
9363 
9364   SDValue Shad = getValue(CI.getArgOperand(1));
9365   assert(Shad.getValueType() == MVT::i32);
9366   SDValue ShadConst = DAG.getTargetConstant(
9367       cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType());
9368   Ops.push_back(ShadConst);
9369 
9370   // Add the live variables.
9371   addStackMapLiveVars(CI, 2, DL, Ops, *this);
9372 
9373   // Create the STACKMAP node.
9374   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9375   Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
9376   InFlag = Chain.getValue(1);
9377 
9378   Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
9379 
9380   // Stackmaps don't generate values, so nothing goes into the NodeMap.
9381 
9382   // Set the root to the target-lowered call chain.
9383   DAG.setRoot(Chain);
9384 
9385   // Inform the Frame Information that we have a stackmap in this function.
9386   FuncInfo.MF->getFrameInfo().setHasStackMap();
9387 }
9388 
9389 /// Lower llvm.experimental.patchpoint directly to its target opcode.
9390 void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
9391                                           const BasicBlock *EHPadBB) {
9392   // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
9393   //                                                 i32 <numBytes>,
9394   //                                                 i8* <target>,
9395   //                                                 i32 <numArgs>,
9396   //                                                 [Args...],
9397   //                                                 [live variables...])
9398 
9399   CallingConv::ID CC = CB.getCallingConv();
9400   bool IsAnyRegCC = CC == CallingConv::AnyReg;
9401   bool HasDef = !CB.getType()->isVoidTy();
9402   SDLoc dl = getCurSDLoc();
9403   SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos));
9404 
9405   // Handle immediate and symbolic callees.
9406   if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
9407     Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
9408                                    /*isTarget=*/true);
9409   else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
9410     Callee =  DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
9411                                          SDLoc(SymbolicCallee),
9412                                          SymbolicCallee->getValueType(0));
9413 
9414   // Get the real number of arguments participating in the call <numArgs>
9415   SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos));
9416   unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
9417 
9418   // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
9419   // Intrinsics include all meta-operands up to but not including CC.
9420   unsigned NumMetaOpers = PatchPointOpers::CCPos;
9421   assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
9422          "Not enough arguments provided to the patchpoint intrinsic");
9423 
9424   // For AnyRegCC the arguments are lowered later on manually.
9425   unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
9426   Type *ReturnTy =
9427       IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
9428 
9429   TargetLowering::CallLoweringInfo CLI(DAG);
9430   populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
9431                            ReturnTy, true);
9432   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9433 
9434   SDNode *CallEnd = Result.second.getNode();
9435   if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
9436     CallEnd = CallEnd->getOperand(0).getNode();
9437 
9438   /// Get a call instruction from the call sequence chain.
9439   /// Tail calls are not allowed.
9440   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
9441          "Expected a callseq node.");
9442   SDNode *Call = CallEnd->getOperand(0).getNode();
9443   bool HasGlue = Call->getGluedNode();
9444 
9445   // Replace the target specific call node with the patchable intrinsic.
9446   SmallVector<SDValue, 8> Ops;
9447 
9448   // Push the chain.
9449   Ops.push_back(*(Call->op_begin()));
9450 
9451   // Optionally, push the glue (if any).
9452   if (HasGlue)
9453     Ops.push_back(*(Call->op_end() - 1));
9454 
9455   // Push the register mask info.
9456   if (HasGlue)
9457     Ops.push_back(*(Call->op_end() - 2));
9458   else
9459     Ops.push_back(*(Call->op_end() - 1));
9460 
9461   // Add the <id> and <numBytes> constants.
9462   SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos));
9463   Ops.push_back(DAG.getTargetConstant(
9464                   cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
9465   SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos));
9466   Ops.push_back(DAG.getTargetConstant(
9467                   cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
9468                   MVT::i32));
9469 
9470   // Add the callee.
9471   Ops.push_back(Callee);
9472 
9473   // Adjust <numArgs> to account for any arguments that have been passed on the
9474   // stack instead.
9475   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
9476   unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
9477   NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
9478   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
9479 
9480   // Add the calling convention
9481   Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
9482 
9483   // Add the arguments we omitted previously. The register allocator should
9484   // place these in any free register.
9485   if (IsAnyRegCC)
9486     for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
9487       Ops.push_back(getValue(CB.getArgOperand(i)));
9488 
9489   // Push the arguments from the call instruction.
9490   SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
9491   Ops.append(Call->op_begin() + 2, e);
9492 
9493   // Push live variables for the stack map.
9494   addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
9495 
9496   SDVTList NodeTys;
9497   if (IsAnyRegCC && HasDef) {
9498     // Create the return types based on the intrinsic definition
9499     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9500     SmallVector<EVT, 3> ValueVTs;
9501     ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
9502     assert(ValueVTs.size() == 1 && "Expected only one return value type.");
9503 
9504     // There is always a chain and a glue type at the end
9505     ValueVTs.push_back(MVT::Other);
9506     ValueVTs.push_back(MVT::Glue);
9507     NodeTys = DAG.getVTList(ValueVTs);
9508   } else
9509     NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
9510 
9511   // Replace the target specific call node with a PATCHPOINT node.
9512   SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
9513 
9514   // Update the NodeMap.
9515   if (HasDef) {
9516     if (IsAnyRegCC)
9517       setValue(&CB, SDValue(PPV.getNode(), 0));
9518     else
9519       setValue(&CB, Result.first);
9520   }
9521 
9522   // Fixup the consumers of the intrinsic. The chain and glue may be used in the
9523   // call sequence. Furthermore the location of the chain and glue can change
9524   // when the AnyReg calling convention is used and the intrinsic returns a
9525   // value.
9526   if (IsAnyRegCC && HasDef) {
9527     SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
9528     SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
9529     DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
9530   } else
9531     DAG.ReplaceAllUsesWith(Call, PPV.getNode());
9532   DAG.DeleteNode(Call);
9533 
9534   // Inform the Frame Information that we have a patchpoint in this function.
9535   FuncInfo.MF->getFrameInfo().setHasPatchPoint();
9536 }
9537 
9538 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
9539                                             unsigned Intrinsic) {
9540   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9541   SDValue Op1 = getValue(I.getArgOperand(0));
9542   SDValue Op2;
9543   if (I.arg_size() > 1)
9544     Op2 = getValue(I.getArgOperand(1));
9545   SDLoc dl = getCurSDLoc();
9546   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
9547   SDValue Res;
9548   SDNodeFlags SDFlags;
9549   if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
9550     SDFlags.copyFMF(*FPMO);
9551 
9552   switch (Intrinsic) {
9553   case Intrinsic::vector_reduce_fadd:
9554     if (SDFlags.hasAllowReassociation())
9555       Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
9556                         DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
9557                         SDFlags);
9558     else
9559       Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
9560     break;
9561   case Intrinsic::vector_reduce_fmul:
9562     if (SDFlags.hasAllowReassociation())
9563       Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
9564                         DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
9565                         SDFlags);
9566     else
9567       Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
9568     break;
9569   case Intrinsic::vector_reduce_add:
9570     Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
9571     break;
9572   case Intrinsic::vector_reduce_mul:
9573     Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
9574     break;
9575   case Intrinsic::vector_reduce_and:
9576     Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
9577     break;
9578   case Intrinsic::vector_reduce_or:
9579     Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
9580     break;
9581   case Intrinsic::vector_reduce_xor:
9582     Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
9583     break;
9584   case Intrinsic::vector_reduce_smax:
9585     Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
9586     break;
9587   case Intrinsic::vector_reduce_smin:
9588     Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
9589     break;
9590   case Intrinsic::vector_reduce_umax:
9591     Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
9592     break;
9593   case Intrinsic::vector_reduce_umin:
9594     Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
9595     break;
9596   case Intrinsic::vector_reduce_fmax:
9597     Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
9598     break;
9599   case Intrinsic::vector_reduce_fmin:
9600     Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
9601     break;
9602   default:
9603     llvm_unreachable("Unhandled vector reduce intrinsic");
9604   }
9605   setValue(&I, Res);
9606 }
9607 
9608 /// Returns an AttributeList representing the attributes applied to the return
9609 /// value of the given call.
9610 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
9611   SmallVector<Attribute::AttrKind, 2> Attrs;
9612   if (CLI.RetSExt)
9613     Attrs.push_back(Attribute::SExt);
9614   if (CLI.RetZExt)
9615     Attrs.push_back(Attribute::ZExt);
9616   if (CLI.IsInReg)
9617     Attrs.push_back(Attribute::InReg);
9618 
9619   return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
9620                             Attrs);
9621 }
9622 
9623 /// TargetLowering::LowerCallTo - This is the default LowerCallTo
9624 /// implementation, which just calls LowerCall.
9625 /// FIXME: When all targets are
9626 /// migrated to using LowerCall, this hook should be integrated into SDISel.
9627 std::pair<SDValue, SDValue>
9628 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
9629   // Handle the incoming return values from the call.
9630   CLI.Ins.clear();
9631   Type *OrigRetTy = CLI.RetTy;
9632   SmallVector<EVT, 4> RetTys;
9633   SmallVector<uint64_t, 4> Offsets;
9634   auto &DL = CLI.DAG.getDataLayout();
9635   ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
9636 
9637   if (CLI.IsPostTypeLegalization) {
9638     // If we are lowering a libcall after legalization, split the return type.
9639     SmallVector<EVT, 4> OldRetTys;
9640     SmallVector<uint64_t, 4> OldOffsets;
9641     RetTys.swap(OldRetTys);
9642     Offsets.swap(OldOffsets);
9643 
9644     for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
9645       EVT RetVT = OldRetTys[i];
9646       uint64_t Offset = OldOffsets[i];
9647       MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
9648       unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
9649       unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
9650       RetTys.append(NumRegs, RegisterVT);
9651       for (unsigned j = 0; j != NumRegs; ++j)
9652         Offsets.push_back(Offset + j * RegisterVTByteSZ);
9653     }
9654   }
9655 
9656   SmallVector<ISD::OutputArg, 4> Outs;
9657   GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
9658 
9659   bool CanLowerReturn =
9660       this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
9661                            CLI.IsVarArg, Outs, CLI.RetTy->getContext());
9662 
9663   SDValue DemoteStackSlot;
9664   int DemoteStackIdx = -100;
9665   if (!CanLowerReturn) {
9666     // FIXME: equivalent assert?
9667     // assert(!CS.hasInAllocaArgument() &&
9668     //        "sret demotion is incompatible with inalloca");
9669     uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
9670     Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
9671     MachineFunction &MF = CLI.DAG.getMachineFunction();
9672     DemoteStackIdx =
9673         MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
9674     Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
9675                                               DL.getAllocaAddrSpace());
9676 
9677     DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
9678     ArgListEntry Entry;
9679     Entry.Node = DemoteStackSlot;
9680     Entry.Ty = StackSlotPtrType;
9681     Entry.IsSExt = false;
9682     Entry.IsZExt = false;
9683     Entry.IsInReg = false;
9684     Entry.IsSRet = true;
9685     Entry.IsNest = false;
9686     Entry.IsByVal = false;
9687     Entry.IsByRef = false;
9688     Entry.IsReturned = false;
9689     Entry.IsSwiftSelf = false;
9690     Entry.IsSwiftAsync = false;
9691     Entry.IsSwiftError = false;
9692     Entry.IsCFGuardTarget = false;
9693     Entry.Alignment = Alignment;
9694     CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
9695     CLI.NumFixedArgs += 1;
9696     CLI.getArgs()[0].IndirectType = CLI.RetTy;
9697     CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
9698 
9699     // sret demotion isn't compatible with tail-calls, since the sret argument
9700     // points into the callers stack frame.
9701     CLI.IsTailCall = false;
9702   } else {
9703     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
9704         CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
9705     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
9706       ISD::ArgFlagsTy Flags;
9707       if (NeedsRegBlock) {
9708         Flags.setInConsecutiveRegs();
9709         if (I == RetTys.size() - 1)
9710           Flags.setInConsecutiveRegsLast();
9711       }
9712       EVT VT = RetTys[I];
9713       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
9714                                                      CLI.CallConv, VT);
9715       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
9716                                                        CLI.CallConv, VT);
9717       for (unsigned i = 0; i != NumRegs; ++i) {
9718         ISD::InputArg MyFlags;
9719         MyFlags.Flags = Flags;
9720         MyFlags.VT = RegisterVT;
9721         MyFlags.ArgVT = VT;
9722         MyFlags.Used = CLI.IsReturnValueUsed;
9723         if (CLI.RetTy->isPointerTy()) {
9724           MyFlags.Flags.setPointer();
9725           MyFlags.Flags.setPointerAddrSpace(
9726               cast<PointerType>(CLI.RetTy)->getAddressSpace());
9727         }
9728         if (CLI.RetSExt)
9729           MyFlags.Flags.setSExt();
9730         if (CLI.RetZExt)
9731           MyFlags.Flags.setZExt();
9732         if (CLI.IsInReg)
9733           MyFlags.Flags.setInReg();
9734         CLI.Ins.push_back(MyFlags);
9735       }
9736     }
9737   }
9738 
9739   // We push in swifterror return as the last element of CLI.Ins.
9740   ArgListTy &Args = CLI.getArgs();
9741   if (supportSwiftError()) {
9742     for (const ArgListEntry &Arg : Args) {
9743       if (Arg.IsSwiftError) {
9744         ISD::InputArg MyFlags;
9745         MyFlags.VT = getPointerTy(DL);
9746         MyFlags.ArgVT = EVT(getPointerTy(DL));
9747         MyFlags.Flags.setSwiftError();
9748         CLI.Ins.push_back(MyFlags);
9749       }
9750     }
9751   }
9752 
9753   // Handle all of the outgoing arguments.
9754   CLI.Outs.clear();
9755   CLI.OutVals.clear();
9756   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
9757     SmallVector<EVT, 4> ValueVTs;
9758     ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
9759     // FIXME: Split arguments if CLI.IsPostTypeLegalization
9760     Type *FinalType = Args[i].Ty;
9761     if (Args[i].IsByVal)
9762       FinalType = Args[i].IndirectType;
9763     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
9764         FinalType, CLI.CallConv, CLI.IsVarArg, DL);
9765     for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
9766          ++Value) {
9767       EVT VT = ValueVTs[Value];
9768       Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
9769       SDValue Op = SDValue(Args[i].Node.getNode(),
9770                            Args[i].Node.getResNo() + Value);
9771       ISD::ArgFlagsTy Flags;
9772 
9773       // Certain targets (such as MIPS), may have a different ABI alignment
9774       // for a type depending on the context. Give the target a chance to
9775       // specify the alignment it wants.
9776       const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
9777       Flags.setOrigAlign(OriginalAlignment);
9778 
9779       if (Args[i].Ty->isPointerTy()) {
9780         Flags.setPointer();
9781         Flags.setPointerAddrSpace(
9782             cast<PointerType>(Args[i].Ty)->getAddressSpace());
9783       }
9784       if (Args[i].IsZExt)
9785         Flags.setZExt();
9786       if (Args[i].IsSExt)
9787         Flags.setSExt();
9788       if (Args[i].IsInReg) {
9789         // If we are using vectorcall calling convention, a structure that is
9790         // passed InReg - is surely an HVA
9791         if (CLI.CallConv == CallingConv::X86_VectorCall &&
9792             isa<StructType>(FinalType)) {
9793           // The first value of a structure is marked
9794           if (0 == Value)
9795             Flags.setHvaStart();
9796           Flags.setHva();
9797         }
9798         // Set InReg Flag
9799         Flags.setInReg();
9800       }
9801       if (Args[i].IsSRet)
9802         Flags.setSRet();
9803       if (Args[i].IsSwiftSelf)
9804         Flags.setSwiftSelf();
9805       if (Args[i].IsSwiftAsync)
9806         Flags.setSwiftAsync();
9807       if (Args[i].IsSwiftError)
9808         Flags.setSwiftError();
9809       if (Args[i].IsCFGuardTarget)
9810         Flags.setCFGuardTarget();
9811       if (Args[i].IsByVal)
9812         Flags.setByVal();
9813       if (Args[i].IsByRef)
9814         Flags.setByRef();
9815       if (Args[i].IsPreallocated) {
9816         Flags.setPreallocated();
9817         // Set the byval flag for CCAssignFn callbacks that don't know about
9818         // preallocated.  This way we can know how many bytes we should've
9819         // allocated and how many bytes a callee cleanup function will pop.  If
9820         // we port preallocated to more targets, we'll have to add custom
9821         // preallocated handling in the various CC lowering callbacks.
9822         Flags.setByVal();
9823       }
9824       if (Args[i].IsInAlloca) {
9825         Flags.setInAlloca();
9826         // Set the byval flag for CCAssignFn callbacks that don't know about
9827         // inalloca.  This way we can know how many bytes we should've allocated
9828         // and how many bytes a callee cleanup function will pop.  If we port
9829         // inalloca to more targets, we'll have to add custom inalloca handling
9830         // in the various CC lowering callbacks.
9831         Flags.setByVal();
9832       }
9833       Align MemAlign;
9834       if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
9835         unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
9836         Flags.setByValSize(FrameSize);
9837 
9838         // info is not there but there are cases it cannot get right.
9839         if (auto MA = Args[i].Alignment)
9840           MemAlign = *MA;
9841         else
9842           MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
9843       } else if (auto MA = Args[i].Alignment) {
9844         MemAlign = *MA;
9845       } else {
9846         MemAlign = OriginalAlignment;
9847       }
9848       Flags.setMemAlign(MemAlign);
9849       if (Args[i].IsNest)
9850         Flags.setNest();
9851       if (NeedsRegBlock)
9852         Flags.setInConsecutiveRegs();
9853 
9854       MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
9855                                                  CLI.CallConv, VT);
9856       unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
9857                                                         CLI.CallConv, VT);
9858       SmallVector<SDValue, 4> Parts(NumParts);
9859       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
9860 
9861       if (Args[i].IsSExt)
9862         ExtendKind = ISD::SIGN_EXTEND;
9863       else if (Args[i].IsZExt)
9864         ExtendKind = ISD::ZERO_EXTEND;
9865 
9866       // Conservatively only handle 'returned' on non-vectors that can be lowered,
9867       // for now.
9868       if (Args[i].IsReturned && !Op.getValueType().isVector() &&
9869           CanLowerReturn) {
9870         assert((CLI.RetTy == Args[i].Ty ||
9871                 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
9872                  CLI.RetTy->getPointerAddressSpace() ==
9873                      Args[i].Ty->getPointerAddressSpace())) &&
9874                RetTys.size() == NumValues && "unexpected use of 'returned'");
9875         // Before passing 'returned' to the target lowering code, ensure that
9876         // either the register MVT and the actual EVT are the same size or that
9877         // the return value and argument are extended in the same way; in these
9878         // cases it's safe to pass the argument register value unchanged as the
9879         // return register value (although it's at the target's option whether
9880         // to do so)
9881         // TODO: allow code generation to take advantage of partially preserved
9882         // registers rather than clobbering the entire register when the
9883         // parameter extension method is not compatible with the return
9884         // extension method
9885         if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
9886             (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
9887              CLI.RetZExt == Args[i].IsZExt))
9888           Flags.setReturned();
9889       }
9890 
9891       getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
9892                      CLI.CallConv, ExtendKind);
9893 
9894       for (unsigned j = 0; j != NumParts; ++j) {
9895         // if it isn't first piece, alignment must be 1
9896         // For scalable vectors the scalable part is currently handled
9897         // by individual targets, so we just use the known minimum size here.
9898         ISD::OutputArg MyFlags(
9899             Flags, Parts[j].getValueType().getSimpleVT(), VT,
9900             i < CLI.NumFixedArgs, i,
9901             j * Parts[j].getValueType().getStoreSize().getKnownMinSize());
9902         if (NumParts > 1 && j == 0)
9903           MyFlags.Flags.setSplit();
9904         else if (j != 0) {
9905           MyFlags.Flags.setOrigAlign(Align(1));
9906           if (j == NumParts - 1)
9907             MyFlags.Flags.setSplitEnd();
9908         }
9909 
9910         CLI.Outs.push_back(MyFlags);
9911         CLI.OutVals.push_back(Parts[j]);
9912       }
9913 
9914       if (NeedsRegBlock && Value == NumValues - 1)
9915         CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
9916     }
9917   }
9918 
9919   SmallVector<SDValue, 4> InVals;
9920   CLI.Chain = LowerCall(CLI, InVals);
9921 
9922   // Update CLI.InVals to use outside of this function.
9923   CLI.InVals = InVals;
9924 
9925   // Verify that the target's LowerCall behaved as expected.
9926   assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
9927          "LowerCall didn't return a valid chain!");
9928   assert((!CLI.IsTailCall || InVals.empty()) &&
9929          "LowerCall emitted a return value for a tail call!");
9930   assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
9931          "LowerCall didn't emit the correct number of values!");
9932 
9933   // For a tail call, the return value is merely live-out and there aren't
9934   // any nodes in the DAG representing it. Return a special value to
9935   // indicate that a tail call has been emitted and no more Instructions
9936   // should be processed in the current block.
9937   if (CLI.IsTailCall) {
9938     CLI.DAG.setRoot(CLI.Chain);
9939     return std::make_pair(SDValue(), SDValue());
9940   }
9941 
9942 #ifndef NDEBUG
9943   for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
9944     assert(InVals[i].getNode() && "LowerCall emitted a null value!");
9945     assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
9946            "LowerCall emitted a value with the wrong type!");
9947   }
9948 #endif
9949 
9950   SmallVector<SDValue, 4> ReturnValues;
9951   if (!CanLowerReturn) {
9952     // The instruction result is the result of loading from the
9953     // hidden sret parameter.
9954     SmallVector<EVT, 1> PVTs;
9955     Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace());
9956 
9957     ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
9958     assert(PVTs.size() == 1 && "Pointers should fit in one register");
9959     EVT PtrVT = PVTs[0];
9960 
9961     unsigned NumValues = RetTys.size();
9962     ReturnValues.resize(NumValues);
9963     SmallVector<SDValue, 4> Chains(NumValues);
9964 
9965     // An aggregate return value cannot wrap around the address space, so
9966     // offsets to its parts don't wrap either.
9967     SDNodeFlags Flags;
9968     Flags.setNoUnsignedWrap(true);
9969 
9970     MachineFunction &MF = CLI.DAG.getMachineFunction();
9971     Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
9972     for (unsigned i = 0; i < NumValues; ++i) {
9973       SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
9974                                     CLI.DAG.getConstant(Offsets[i], CLI.DL,
9975                                                         PtrVT), Flags);
9976       SDValue L = CLI.DAG.getLoad(
9977           RetTys[i], CLI.DL, CLI.Chain, Add,
9978           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
9979                                             DemoteStackIdx, Offsets[i]),
9980           HiddenSRetAlign);
9981       ReturnValues[i] = L;
9982       Chains[i] = L.getValue(1);
9983     }
9984 
9985     CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
9986   } else {
9987     // Collect the legal value parts into potentially illegal values
9988     // that correspond to the original function's return values.
9989     Optional<ISD::NodeType> AssertOp;
9990     if (CLI.RetSExt)
9991       AssertOp = ISD::AssertSext;
9992     else if (CLI.RetZExt)
9993       AssertOp = ISD::AssertZext;
9994     unsigned CurReg = 0;
9995     for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
9996       EVT VT = RetTys[I];
9997       MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
9998                                                      CLI.CallConv, VT);
9999       unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10000                                                        CLI.CallConv, VT);
10001 
10002       ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
10003                                               NumRegs, RegisterVT, VT, nullptr,
10004                                               CLI.CallConv, AssertOp));
10005       CurReg += NumRegs;
10006     }
10007 
10008     // For a function returning void, there is no return value. We can't create
10009     // such a node, so we just return a null return value in that case. In
10010     // that case, nothing will actually look at the value.
10011     if (ReturnValues.empty())
10012       return std::make_pair(SDValue(), CLI.Chain);
10013   }
10014 
10015   SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
10016                                 CLI.DAG.getVTList(RetTys), ReturnValues);
10017   return std::make_pair(Res, CLI.Chain);
10018 }
10019 
10020 /// Places new result values for the node in Results (their number
10021 /// and types must exactly match those of the original return values of
10022 /// the node), or leaves Results empty, which indicates that the node is not
10023 /// to be custom lowered after all.
10024 void TargetLowering::LowerOperationWrapper(SDNode *N,
10025                                            SmallVectorImpl<SDValue> &Results,
10026                                            SelectionDAG &DAG) const {
10027   SDValue Res = LowerOperation(SDValue(N, 0), DAG);
10028 
10029   if (!Res.getNode())
10030     return;
10031 
10032   // If the original node has one result, take the return value from
10033   // LowerOperation as is. It might not be result number 0.
10034   if (N->getNumValues() == 1) {
10035     Results.push_back(Res);
10036     return;
10037   }
10038 
10039   // If the original node has multiple results, then the return node should
10040   // have the same number of results.
10041   assert((N->getNumValues() == Res->getNumValues()) &&
10042       "Lowering returned the wrong number of results!");
10043 
10044   // Places new result values base on N result number.
10045   for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
10046     Results.push_back(Res.getValue(I));
10047 }
10048 
10049 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
10050   llvm_unreachable("LowerOperation not implemented for this target!");
10051 }
10052 
10053 void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
10054                                                      unsigned Reg,
10055                                                      ISD::NodeType ExtendType) {
10056   SDValue Op = getNonRegisterValue(V);
10057   assert((Op.getOpcode() != ISD::CopyFromReg ||
10058           cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
10059          "Copy from a reg to the same reg!");
10060   assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
10061 
10062   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10063   // If this is an InlineAsm we have to match the registers required, not the
10064   // notional registers required by the type.
10065 
10066   RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
10067                    None); // This is not an ABI copy.
10068   SDValue Chain = DAG.getEntryNode();
10069 
10070   if (ExtendType == ISD::ANY_EXTEND) {
10071     auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
10072     if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
10073       ExtendType = PreferredExtendIt->second;
10074   }
10075   RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
10076   PendingExports.push_back(Chain);
10077 }
10078 
10079 #include "llvm/CodeGen/SelectionDAGISel.h"
10080 
10081 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10082 /// entry block, return true.  This includes arguments used by switches, since
10083 /// the switch may expand into multiple basic blocks.
10084 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
10085   // With FastISel active, we may be splitting blocks, so force creation
10086   // of virtual registers for all non-dead arguments.
10087   if (FastISel)
10088     return A->use_empty();
10089 
10090   const BasicBlock &Entry = A->getParent()->front();
10091   for (const User *U : A->users())
10092     if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
10093       return false;  // Use not in entry block.
10094 
10095   return true;
10096 }
10097 
10098 using ArgCopyElisionMapTy =
10099     DenseMap<const Argument *,
10100              std::pair<const AllocaInst *, const StoreInst *>>;
10101 
10102 /// Scan the entry block of the function in FuncInfo for arguments that look
10103 /// like copies into a local alloca. Record any copied arguments in
10104 /// ArgCopyElisionCandidates.
10105 static void
10106 findArgumentCopyElisionCandidates(const DataLayout &DL,
10107                                   FunctionLoweringInfo *FuncInfo,
10108                                   ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
10109   // Record the state of every static alloca used in the entry block. Argument
10110   // allocas are all used in the entry block, so we need approximately as many
10111   // entries as we have arguments.
10112   enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
10113   SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
10114   unsigned NumArgs = FuncInfo->Fn->arg_size();
10115   StaticAllocas.reserve(NumArgs * 2);
10116 
10117   auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
10118     if (!V)
10119       return nullptr;
10120     V = V->stripPointerCasts();
10121     const auto *AI = dyn_cast<AllocaInst>(V);
10122     if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
10123       return nullptr;
10124     auto Iter = StaticAllocas.insert({AI, Unknown});
10125     return &Iter.first->second;
10126   };
10127 
10128   // Look for stores of arguments to static allocas. Look through bitcasts and
10129   // GEPs to handle type coercions, as long as the alloca is fully initialized
10130   // by the store. Any non-store use of an alloca escapes it and any subsequent
10131   // unanalyzed store might write it.
10132   // FIXME: Handle structs initialized with multiple stores.
10133   for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
10134     // Look for stores, and handle non-store uses conservatively.
10135     const auto *SI = dyn_cast<StoreInst>(&I);
10136     if (!SI) {
10137       // We will look through cast uses, so ignore them completely.
10138       if (I.isCast())
10139         continue;
10140       // Ignore debug info and pseudo op intrinsics, they don't escape or store
10141       // to allocas.
10142       if (I.isDebugOrPseudoInst())
10143         continue;
10144       // This is an unknown instruction. Assume it escapes or writes to all
10145       // static alloca operands.
10146       for (const Use &U : I.operands()) {
10147         if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
10148           *Info = StaticAllocaInfo::Clobbered;
10149       }
10150       continue;
10151     }
10152 
10153     // If the stored value is a static alloca, mark it as escaped.
10154     if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
10155       *Info = StaticAllocaInfo::Clobbered;
10156 
10157     // Check if the destination is a static alloca.
10158     const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
10159     StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
10160     if (!Info)
10161       continue;
10162     const AllocaInst *AI = cast<AllocaInst>(Dst);
10163 
10164     // Skip allocas that have been initialized or clobbered.
10165     if (*Info != StaticAllocaInfo::Unknown)
10166       continue;
10167 
10168     // Check if the stored value is an argument, and that this store fully
10169     // initializes the alloca.
10170     // If the argument type has padding bits we can't directly forward a pointer
10171     // as the upper bits may contain garbage.
10172     // Don't elide copies from the same argument twice.
10173     const Value *Val = SI->getValueOperand()->stripPointerCasts();
10174     const auto *Arg = dyn_cast<Argument>(Val);
10175     if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
10176         Arg->getType()->isEmptyTy() ||
10177         DL.getTypeStoreSize(Arg->getType()) !=
10178             DL.getTypeAllocSize(AI->getAllocatedType()) ||
10179         !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
10180         ArgCopyElisionCandidates.count(Arg)) {
10181       *Info = StaticAllocaInfo::Clobbered;
10182       continue;
10183     }
10184 
10185     LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
10186                       << '\n');
10187 
10188     // Mark this alloca and store for argument copy elision.
10189     *Info = StaticAllocaInfo::Elidable;
10190     ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
10191 
10192     // Stop scanning if we've seen all arguments. This will happen early in -O0
10193     // builds, which is useful, because -O0 builds have large entry blocks and
10194     // many allocas.
10195     if (ArgCopyElisionCandidates.size() == NumArgs)
10196       break;
10197   }
10198 }
10199 
10200 /// Try to elide argument copies from memory into a local alloca. Succeeds if
10201 /// ArgVal is a load from a suitable fixed stack object.
10202 static void tryToElideArgumentCopy(
10203     FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
10204     DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
10205     SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
10206     ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
10207     SDValue ArgVal, bool &ArgHasUses) {
10208   // Check if this is a load from a fixed stack object.
10209   auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
10210   if (!LNode)
10211     return;
10212   auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
10213   if (!FINode)
10214     return;
10215 
10216   // Check that the fixed stack object is the right size and alignment.
10217   // Look at the alignment that the user wrote on the alloca instead of looking
10218   // at the stack object.
10219   auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
10220   assert(ArgCopyIter != ArgCopyElisionCandidates.end());
10221   const AllocaInst *AI = ArgCopyIter->second.first;
10222   int FixedIndex = FINode->getIndex();
10223   int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
10224   int OldIndex = AllocaIndex;
10225   MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
10226   if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
10227     LLVM_DEBUG(
10228         dbgs() << "  argument copy elision failed due to bad fixed stack "
10229                   "object size\n");
10230     return;
10231   }
10232   Align RequiredAlignment = AI->getAlign();
10233   if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
10234     LLVM_DEBUG(dbgs() << "  argument copy elision failed: alignment of alloca "
10235                          "greater than stack argument alignment ("
10236                       << DebugStr(RequiredAlignment) << " vs "
10237                       << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
10238     return;
10239   }
10240 
10241   // Perform the elision. Delete the old stack object and replace its only use
10242   // in the variable info map. Mark the stack object as mutable.
10243   LLVM_DEBUG({
10244     dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
10245            << "  Replacing frame index " << OldIndex << " with " << FixedIndex
10246            << '\n';
10247   });
10248   MFI.RemoveStackObject(OldIndex);
10249   MFI.setIsImmutableObjectIndex(FixedIndex, false);
10250   AllocaIndex = FixedIndex;
10251   ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
10252   Chains.push_back(ArgVal.getValue(1));
10253 
10254   // Avoid emitting code for the store implementing the copy.
10255   const StoreInst *SI = ArgCopyIter->second.second;
10256   ElidedArgCopyInstrs.insert(SI);
10257 
10258   // Check for uses of the argument again so that we can avoid exporting ArgVal
10259   // if it is't used by anything other than the store.
10260   for (const Value *U : Arg.users()) {
10261     if (U != SI) {
10262       ArgHasUses = true;
10263       break;
10264     }
10265   }
10266 }
10267 
10268 void SelectionDAGISel::LowerArguments(const Function &F) {
10269   SelectionDAG &DAG = SDB->DAG;
10270   SDLoc dl = SDB->getCurSDLoc();
10271   const DataLayout &DL = DAG.getDataLayout();
10272   SmallVector<ISD::InputArg, 16> Ins;
10273 
10274   // In Naked functions we aren't going to save any registers.
10275   if (F.hasFnAttribute(Attribute::Naked))
10276     return;
10277 
10278   if (!FuncInfo->CanLowerReturn) {
10279     // Put in an sret pointer parameter before all the other parameters.
10280     SmallVector<EVT, 1> ValueVTs;
10281     ComputeValueVTs(*TLI, DAG.getDataLayout(),
10282                     F.getReturnType()->getPointerTo(
10283                         DAG.getDataLayout().getAllocaAddrSpace()),
10284                     ValueVTs);
10285 
10286     // NOTE: Assuming that a pointer will never break down to more than one VT
10287     // or one register.
10288     ISD::ArgFlagsTy Flags;
10289     Flags.setSRet();
10290     MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
10291     ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
10292                          ISD::InputArg::NoArgIndex, 0);
10293     Ins.push_back(RetArg);
10294   }
10295 
10296   // Look for stores of arguments to static allocas. Mark such arguments with a
10297   // flag to ask the target to give us the memory location of that argument if
10298   // available.
10299   ArgCopyElisionMapTy ArgCopyElisionCandidates;
10300   findArgumentCopyElisionCandidates(DL, FuncInfo.get(),
10301                                     ArgCopyElisionCandidates);
10302 
10303   // Set up the incoming argument description vector.
10304   for (const Argument &Arg : F.args()) {
10305     unsigned ArgNo = Arg.getArgNo();
10306     SmallVector<EVT, 4> ValueVTs;
10307     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
10308     bool isArgValueUsed = !Arg.use_empty();
10309     unsigned PartBase = 0;
10310     Type *FinalType = Arg.getType();
10311     if (Arg.hasAttribute(Attribute::ByVal))
10312       FinalType = Arg.getParamByValType();
10313     bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
10314         FinalType, F.getCallingConv(), F.isVarArg(), DL);
10315     for (unsigned Value = 0, NumValues = ValueVTs.size();
10316          Value != NumValues; ++Value) {
10317       EVT VT = ValueVTs[Value];
10318       Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
10319       ISD::ArgFlagsTy Flags;
10320 
10321 
10322       if (Arg.getType()->isPointerTy()) {
10323         Flags.setPointer();
10324         Flags.setPointerAddrSpace(
10325             cast<PointerType>(Arg.getType())->getAddressSpace());
10326       }
10327       if (Arg.hasAttribute(Attribute::ZExt))
10328         Flags.setZExt();
10329       if (Arg.hasAttribute(Attribute::SExt))
10330         Flags.setSExt();
10331       if (Arg.hasAttribute(Attribute::InReg)) {
10332         // If we are using vectorcall calling convention, a structure that is
10333         // passed InReg - is surely an HVA
10334         if (F.getCallingConv() == CallingConv::X86_VectorCall &&
10335             isa<StructType>(Arg.getType())) {
10336           // The first value of a structure is marked
10337           if (0 == Value)
10338             Flags.setHvaStart();
10339           Flags.setHva();
10340         }
10341         // Set InReg Flag
10342         Flags.setInReg();
10343       }
10344       if (Arg.hasAttribute(Attribute::StructRet))
10345         Flags.setSRet();
10346       if (Arg.hasAttribute(Attribute::SwiftSelf))
10347         Flags.setSwiftSelf();
10348       if (Arg.hasAttribute(Attribute::SwiftAsync))
10349         Flags.setSwiftAsync();
10350       if (Arg.hasAttribute(Attribute::SwiftError))
10351         Flags.setSwiftError();
10352       if (Arg.hasAttribute(Attribute::ByVal))
10353         Flags.setByVal();
10354       if (Arg.hasAttribute(Attribute::ByRef))
10355         Flags.setByRef();
10356       if (Arg.hasAttribute(Attribute::InAlloca)) {
10357         Flags.setInAlloca();
10358         // Set the byval flag for CCAssignFn callbacks that don't know about
10359         // inalloca.  This way we can know how many bytes we should've allocated
10360         // and how many bytes a callee cleanup function will pop.  If we port
10361         // inalloca to more targets, we'll have to add custom inalloca handling
10362         // in the various CC lowering callbacks.
10363         Flags.setByVal();
10364       }
10365       if (Arg.hasAttribute(Attribute::Preallocated)) {
10366         Flags.setPreallocated();
10367         // Set the byval flag for CCAssignFn callbacks that don't know about
10368         // preallocated.  This way we can know how many bytes we should've
10369         // allocated and how many bytes a callee cleanup function will pop.  If
10370         // we port preallocated to more targets, we'll have to add custom
10371         // preallocated handling in the various CC lowering callbacks.
10372         Flags.setByVal();
10373       }
10374 
10375       // Certain targets (such as MIPS), may have a different ABI alignment
10376       // for a type depending on the context. Give the target a chance to
10377       // specify the alignment it wants.
10378       const Align OriginalAlignment(
10379           TLI->getABIAlignmentForCallingConv(ArgTy, DL));
10380       Flags.setOrigAlign(OriginalAlignment);
10381 
10382       Align MemAlign;
10383       Type *ArgMemTy = nullptr;
10384       if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
10385           Flags.isByRef()) {
10386         if (!ArgMemTy)
10387           ArgMemTy = Arg.getPointeeInMemoryValueType();
10388 
10389         uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
10390 
10391         // For in-memory arguments, size and alignment should be passed from FE.
10392         // BE will guess if this info is not there but there are cases it cannot
10393         // get right.
10394         if (auto ParamAlign = Arg.getParamStackAlign())
10395           MemAlign = *ParamAlign;
10396         else if ((ParamAlign = Arg.getParamAlign()))
10397           MemAlign = *ParamAlign;
10398         else
10399           MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
10400         if (Flags.isByRef())
10401           Flags.setByRefSize(MemSize);
10402         else
10403           Flags.setByValSize(MemSize);
10404       } else if (auto ParamAlign = Arg.getParamStackAlign()) {
10405         MemAlign = *ParamAlign;
10406       } else {
10407         MemAlign = OriginalAlignment;
10408       }
10409       Flags.setMemAlign(MemAlign);
10410 
10411       if (Arg.hasAttribute(Attribute::Nest))
10412         Flags.setNest();
10413       if (NeedsRegBlock)
10414         Flags.setInConsecutiveRegs();
10415       if (ArgCopyElisionCandidates.count(&Arg))
10416         Flags.setCopyElisionCandidate();
10417       if (Arg.hasAttribute(Attribute::Returned))
10418         Flags.setReturned();
10419 
10420       MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
10421           *CurDAG->getContext(), F.getCallingConv(), VT);
10422       unsigned NumRegs = TLI->getNumRegistersForCallingConv(
10423           *CurDAG->getContext(), F.getCallingConv(), VT);
10424       for (unsigned i = 0; i != NumRegs; ++i) {
10425         // For scalable vectors, use the minimum size; individual targets
10426         // are responsible for handling scalable vector arguments and
10427         // return values.
10428         ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
10429                  ArgNo, PartBase+i*RegisterVT.getStoreSize().getKnownMinSize());
10430         if (NumRegs > 1 && i == 0)
10431           MyFlags.Flags.setSplit();
10432         // if it isn't first piece, alignment must be 1
10433         else if (i > 0) {
10434           MyFlags.Flags.setOrigAlign(Align(1));
10435           if (i == NumRegs - 1)
10436             MyFlags.Flags.setSplitEnd();
10437         }
10438         Ins.push_back(MyFlags);
10439       }
10440       if (NeedsRegBlock && Value == NumValues - 1)
10441         Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
10442       PartBase += VT.getStoreSize().getKnownMinSize();
10443     }
10444   }
10445 
10446   // Call the target to set up the argument values.
10447   SmallVector<SDValue, 8> InVals;
10448   SDValue NewRoot = TLI->LowerFormalArguments(
10449       DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
10450 
10451   // Verify that the target's LowerFormalArguments behaved as expected.
10452   assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
10453          "LowerFormalArguments didn't return a valid chain!");
10454   assert(InVals.size() == Ins.size() &&
10455          "LowerFormalArguments didn't emit the correct number of values!");
10456   LLVM_DEBUG({
10457     for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
10458       assert(InVals[i].getNode() &&
10459              "LowerFormalArguments emitted a null value!");
10460       assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
10461              "LowerFormalArguments emitted a value with the wrong type!");
10462     }
10463   });
10464 
10465   // Update the DAG with the new chain value resulting from argument lowering.
10466   DAG.setRoot(NewRoot);
10467 
10468   // Set up the argument values.
10469   unsigned i = 0;
10470   if (!FuncInfo->CanLowerReturn) {
10471     // Create a virtual register for the sret pointer, and put in a copy
10472     // from the sret argument into it.
10473     SmallVector<EVT, 1> ValueVTs;
10474     ComputeValueVTs(*TLI, DAG.getDataLayout(),
10475                     F.getReturnType()->getPointerTo(
10476                         DAG.getDataLayout().getAllocaAddrSpace()),
10477                     ValueVTs);
10478     MVT VT = ValueVTs[0].getSimpleVT();
10479     MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
10480     Optional<ISD::NodeType> AssertOp = None;
10481     SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
10482                                         nullptr, F.getCallingConv(), AssertOp);
10483 
10484     MachineFunction& MF = SDB->DAG.getMachineFunction();
10485     MachineRegisterInfo& RegInfo = MF.getRegInfo();
10486     Register SRetReg =
10487         RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
10488     FuncInfo->DemoteRegister = SRetReg;
10489     NewRoot =
10490         SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
10491     DAG.setRoot(NewRoot);
10492 
10493     // i indexes lowered arguments.  Bump it past the hidden sret argument.
10494     ++i;
10495   }
10496 
10497   SmallVector<SDValue, 4> Chains;
10498   DenseMap<int, int> ArgCopyElisionFrameIndexMap;
10499   for (const Argument &Arg : F.args()) {
10500     SmallVector<SDValue, 4> ArgValues;
10501     SmallVector<EVT, 4> ValueVTs;
10502     ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
10503     unsigned NumValues = ValueVTs.size();
10504     if (NumValues == 0)
10505       continue;
10506 
10507     bool ArgHasUses = !Arg.use_empty();
10508 
10509     // Elide the copying store if the target loaded this argument from a
10510     // suitable fixed stack object.
10511     if (Ins[i].Flags.isCopyElisionCandidate()) {
10512       tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
10513                              ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
10514                              InVals[i], ArgHasUses);
10515     }
10516 
10517     // If this argument is unused then remember its value. It is used to generate
10518     // debugging information.
10519     bool isSwiftErrorArg =
10520         TLI->supportSwiftError() &&
10521         Arg.hasAttribute(Attribute::SwiftError);
10522     if (!ArgHasUses && !isSwiftErrorArg) {
10523       SDB->setUnusedArgValue(&Arg, InVals[i]);
10524 
10525       // Also remember any frame index for use in FastISel.
10526       if (FrameIndexSDNode *FI =
10527           dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
10528         FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10529     }
10530 
10531     for (unsigned Val = 0; Val != NumValues; ++Val) {
10532       EVT VT = ValueVTs[Val];
10533       MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
10534                                                       F.getCallingConv(), VT);
10535       unsigned NumParts = TLI->getNumRegistersForCallingConv(
10536           *CurDAG->getContext(), F.getCallingConv(), VT);
10537 
10538       // Even an apparent 'unused' swifterror argument needs to be returned. So
10539       // we do generate a copy for it that can be used on return from the
10540       // function.
10541       if (ArgHasUses || isSwiftErrorArg) {
10542         Optional<ISD::NodeType> AssertOp;
10543         if (Arg.hasAttribute(Attribute::SExt))
10544           AssertOp = ISD::AssertSext;
10545         else if (Arg.hasAttribute(Attribute::ZExt))
10546           AssertOp = ISD::AssertZext;
10547 
10548         ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
10549                                              PartVT, VT, nullptr,
10550                                              F.getCallingConv(), AssertOp));
10551       }
10552 
10553       i += NumParts;
10554     }
10555 
10556     // We don't need to do anything else for unused arguments.
10557     if (ArgValues.empty())
10558       continue;
10559 
10560     // Note down frame index.
10561     if (FrameIndexSDNode *FI =
10562         dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
10563       FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10564 
10565     SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
10566                                      SDB->getCurSDLoc());
10567 
10568     SDB->setValue(&Arg, Res);
10569     if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
10570       // We want to associate the argument with the frame index, among
10571       // involved operands, that correspond to the lowest address. The
10572       // getCopyFromParts function, called earlier, is swapping the order of
10573       // the operands to BUILD_PAIR depending on endianness. The result of
10574       // that swapping is that the least significant bits of the argument will
10575       // be in the first operand of the BUILD_PAIR node, and the most
10576       // significant bits will be in the second operand.
10577       unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
10578       if (LoadSDNode *LNode =
10579           dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
10580         if (FrameIndexSDNode *FI =
10581             dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
10582           FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
10583     }
10584 
10585     // Analyses past this point are naive and don't expect an assertion.
10586     if (Res.getOpcode() == ISD::AssertZext)
10587       Res = Res.getOperand(0);
10588 
10589     // Update the SwiftErrorVRegDefMap.
10590     if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
10591       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
10592       if (Register::isVirtualRegister(Reg))
10593         SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
10594                                    Reg);
10595     }
10596 
10597     // If this argument is live outside of the entry block, insert a copy from
10598     // wherever we got it to the vreg that other BB's will reference it as.
10599     if (Res.getOpcode() == ISD::CopyFromReg) {
10600       // If we can, though, try to skip creating an unnecessary vreg.
10601       // FIXME: This isn't very clean... it would be nice to make this more
10602       // general.
10603       unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
10604       if (Register::isVirtualRegister(Reg)) {
10605         FuncInfo->ValueMap[&Arg] = Reg;
10606         continue;
10607       }
10608     }
10609     if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
10610       FuncInfo->InitializeRegForValue(&Arg);
10611       SDB->CopyToExportRegsIfNeeded(&Arg);
10612     }
10613   }
10614 
10615   if (!Chains.empty()) {
10616     Chains.push_back(NewRoot);
10617     NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
10618   }
10619 
10620   DAG.setRoot(NewRoot);
10621 
10622   assert(i == InVals.size() && "Argument register count mismatch!");
10623 
10624   // If any argument copy elisions occurred and we have debug info, update the
10625   // stale frame indices used in the dbg.declare variable info table.
10626   MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
10627   if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
10628     for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
10629       auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
10630       if (I != ArgCopyElisionFrameIndexMap.end())
10631         VI.Slot = I->second;
10632     }
10633   }
10634 
10635   // Finally, if the target has anything special to do, allow it to do so.
10636   emitFunctionEntryCode();
10637 }
10638 
10639 /// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
10640 /// ensure constants are generated when needed.  Remember the virtual registers
10641 /// that need to be added to the Machine PHI nodes as input.  We cannot just
10642 /// directly add them, because expansion might result in multiple MBB's for one
10643 /// BB.  As such, the start of the BB might correspond to a different MBB than
10644 /// the end.
10645 void
10646 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
10647   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10648   const Instruction *TI = LLVMBB->getTerminator();
10649 
10650   SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
10651 
10652   // Check PHI nodes in successors that expect a value to be available from this
10653   // block.
10654   for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
10655     const BasicBlock *SuccBB = TI->getSuccessor(succ);
10656     if (!isa<PHINode>(SuccBB->begin())) continue;
10657     MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
10658 
10659     // If this terminator has multiple identical successors (common for
10660     // switches), only handle each succ once.
10661     if (!SuccsHandled.insert(SuccMBB).second)
10662       continue;
10663 
10664     MachineBasicBlock::iterator MBBI = SuccMBB->begin();
10665 
10666     // At this point we know that there is a 1-1 correspondence between LLVM PHI
10667     // nodes and Machine PHI nodes, but the incoming operands have not been
10668     // emitted yet.
10669     for (const PHINode &PN : SuccBB->phis()) {
10670       // Ignore dead phi's.
10671       if (PN.use_empty())
10672         continue;
10673 
10674       // Skip empty types
10675       if (PN.getType()->isEmptyTy())
10676         continue;
10677 
10678       unsigned Reg;
10679       const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
10680 
10681       if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
10682         unsigned &RegOut = ConstantsOut[C];
10683         if (RegOut == 0) {
10684           RegOut = FuncInfo.CreateRegs(C);
10685           // We need to zero/sign extend ConstantInt phi operands to match
10686           // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
10687           ISD::NodeType ExtendType = ISD::ANY_EXTEND;
10688           if (auto *CI = dyn_cast<ConstantInt>(C))
10689             ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
10690                                                     : ISD::ZERO_EXTEND;
10691           CopyValueToVirtualRegister(C, RegOut, ExtendType);
10692         }
10693         Reg = RegOut;
10694       } else {
10695         DenseMap<const Value *, Register>::iterator I =
10696           FuncInfo.ValueMap.find(PHIOp);
10697         if (I != FuncInfo.ValueMap.end())
10698           Reg = I->second;
10699         else {
10700           assert(isa<AllocaInst>(PHIOp) &&
10701                  FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
10702                  "Didn't codegen value into a register!??");
10703           Reg = FuncInfo.CreateRegs(PHIOp);
10704           CopyValueToVirtualRegister(PHIOp, Reg);
10705         }
10706       }
10707 
10708       // Remember that this register needs to added to the machine PHI node as
10709       // the input for this MBB.
10710       SmallVector<EVT, 4> ValueVTs;
10711       ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
10712       for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
10713         EVT VT = ValueVTs[vti];
10714         unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
10715         for (unsigned i = 0, e = NumRegisters; i != e; ++i)
10716           FuncInfo.PHINodesToUpdate.push_back(
10717               std::make_pair(&*MBBI++, Reg + i));
10718         Reg += NumRegisters;
10719       }
10720     }
10721   }
10722 
10723   ConstantsOut.clear();
10724 }
10725 
10726 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
10727   MachineFunction::iterator I(MBB);
10728   if (++I == FuncInfo.MF->end())
10729     return nullptr;
10730   return &*I;
10731 }
10732 
10733 /// During lowering new call nodes can be created (such as memset, etc.).
10734 /// Those will become new roots of the current DAG, but complications arise
10735 /// when they are tail calls. In such cases, the call lowering will update
10736 /// the root, but the builder still needs to know that a tail call has been
10737 /// lowered in order to avoid generating an additional return.
10738 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
10739   // If the node is null, we do have a tail call.
10740   if (MaybeTC.getNode() != nullptr)
10741     DAG.setRoot(MaybeTC);
10742   else
10743     HasTailCall = true;
10744 }
10745 
10746 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
10747                                         MachineBasicBlock *SwitchMBB,
10748                                         MachineBasicBlock *DefaultMBB) {
10749   MachineFunction *CurMF = FuncInfo.MF;
10750   MachineBasicBlock *NextMBB = nullptr;
10751   MachineFunction::iterator BBI(W.MBB);
10752   if (++BBI != FuncInfo.MF->end())
10753     NextMBB = &*BBI;
10754 
10755   unsigned Size = W.LastCluster - W.FirstCluster + 1;
10756 
10757   BranchProbabilityInfo *BPI = FuncInfo.BPI;
10758 
10759   if (Size == 2 && W.MBB == SwitchMBB) {
10760     // If any two of the cases has the same destination, and if one value
10761     // is the same as the other, but has one bit unset that the other has set,
10762     // use bit manipulation to do two compares at once.  For example:
10763     // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
10764     // TODO: This could be extended to merge any 2 cases in switches with 3
10765     // cases.
10766     // TODO: Handle cases where W.CaseBB != SwitchBB.
10767     CaseCluster &Small = *W.FirstCluster;
10768     CaseCluster &Big = *W.LastCluster;
10769 
10770     if (Small.Low == Small.High && Big.Low == Big.High &&
10771         Small.MBB == Big.MBB) {
10772       const APInt &SmallValue = Small.Low->getValue();
10773       const APInt &BigValue = Big.Low->getValue();
10774 
10775       // Check that there is only one bit different.
10776       APInt CommonBit = BigValue ^ SmallValue;
10777       if (CommonBit.isPowerOf2()) {
10778         SDValue CondLHS = getValue(Cond);
10779         EVT VT = CondLHS.getValueType();
10780         SDLoc DL = getCurSDLoc();
10781 
10782         SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
10783                                  DAG.getConstant(CommonBit, DL, VT));
10784         SDValue Cond = DAG.getSetCC(
10785             DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
10786             ISD::SETEQ);
10787 
10788         // Update successor info.
10789         // Both Small and Big will jump to Small.BB, so we sum up the
10790         // probabilities.
10791         addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
10792         if (BPI)
10793           addSuccessorWithProb(
10794               SwitchMBB, DefaultMBB,
10795               // The default destination is the first successor in IR.
10796               BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
10797         else
10798           addSuccessorWithProb(SwitchMBB, DefaultMBB);
10799 
10800         // Insert the true branch.
10801         SDValue BrCond =
10802             DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
10803                         DAG.getBasicBlock(Small.MBB));
10804         // Insert the false branch.
10805         BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
10806                              DAG.getBasicBlock(DefaultMBB));
10807 
10808         DAG.setRoot(BrCond);
10809         return;
10810       }
10811     }
10812   }
10813 
10814   if (TM.getOptLevel() != CodeGenOpt::None) {
10815     // Here, we order cases by probability so the most likely case will be
10816     // checked first. However, two clusters can have the same probability in
10817     // which case their relative ordering is non-deterministic. So we use Low
10818     // as a tie-breaker as clusters are guaranteed to never overlap.
10819     llvm::sort(W.FirstCluster, W.LastCluster + 1,
10820                [](const CaseCluster &a, const CaseCluster &b) {
10821       return a.Prob != b.Prob ?
10822              a.Prob > b.Prob :
10823              a.Low->getValue().slt(b.Low->getValue());
10824     });
10825 
10826     // Rearrange the case blocks so that the last one falls through if possible
10827     // without changing the order of probabilities.
10828     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
10829       --I;
10830       if (I->Prob > W.LastCluster->Prob)
10831         break;
10832       if (I->Kind == CC_Range && I->MBB == NextMBB) {
10833         std::swap(*I, *W.LastCluster);
10834         break;
10835       }
10836     }
10837   }
10838 
10839   // Compute total probability.
10840   BranchProbability DefaultProb = W.DefaultProb;
10841   BranchProbability UnhandledProbs = DefaultProb;
10842   for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
10843     UnhandledProbs += I->Prob;
10844 
10845   MachineBasicBlock *CurMBB = W.MBB;
10846   for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
10847     bool FallthroughUnreachable = false;
10848     MachineBasicBlock *Fallthrough;
10849     if (I == W.LastCluster) {
10850       // For the last cluster, fall through to the default destination.
10851       Fallthrough = DefaultMBB;
10852       FallthroughUnreachable = isa<UnreachableInst>(
10853           DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
10854     } else {
10855       Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
10856       CurMF->insert(BBI, Fallthrough);
10857       // Put Cond in a virtual register to make it available from the new blocks.
10858       ExportFromCurrentBlock(Cond);
10859     }
10860     UnhandledProbs -= I->Prob;
10861 
10862     switch (I->Kind) {
10863       case CC_JumpTable: {
10864         // FIXME: Optimize away range check based on pivot comparisons.
10865         JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
10866         SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
10867 
10868         // The jump block hasn't been inserted yet; insert it here.
10869         MachineBasicBlock *JumpMBB = JT->MBB;
10870         CurMF->insert(BBI, JumpMBB);
10871 
10872         auto JumpProb = I->Prob;
10873         auto FallthroughProb = UnhandledProbs;
10874 
10875         // If the default statement is a target of the jump table, we evenly
10876         // distribute the default probability to successors of CurMBB. Also
10877         // update the probability on the edge from JumpMBB to Fallthrough.
10878         for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
10879                                               SE = JumpMBB->succ_end();
10880              SI != SE; ++SI) {
10881           if (*SI == DefaultMBB) {
10882             JumpProb += DefaultProb / 2;
10883             FallthroughProb -= DefaultProb / 2;
10884             JumpMBB->setSuccProbability(SI, DefaultProb / 2);
10885             JumpMBB->normalizeSuccProbs();
10886             break;
10887           }
10888         }
10889 
10890         if (FallthroughUnreachable)
10891           JTH->FallthroughUnreachable = true;
10892 
10893         if (!JTH->FallthroughUnreachable)
10894           addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
10895         addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
10896         CurMBB->normalizeSuccProbs();
10897 
10898         // The jump table header will be inserted in our current block, do the
10899         // range check, and fall through to our fallthrough block.
10900         JTH->HeaderBB = CurMBB;
10901         JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
10902 
10903         // If we're in the right place, emit the jump table header right now.
10904         if (CurMBB == SwitchMBB) {
10905           visitJumpTableHeader(*JT, *JTH, SwitchMBB);
10906           JTH->Emitted = true;
10907         }
10908         break;
10909       }
10910       case CC_BitTests: {
10911         // FIXME: Optimize away range check based on pivot comparisons.
10912         BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
10913 
10914         // The bit test blocks haven't been inserted yet; insert them here.
10915         for (BitTestCase &BTC : BTB->Cases)
10916           CurMF->insert(BBI, BTC.ThisBB);
10917 
10918         // Fill in fields of the BitTestBlock.
10919         BTB->Parent = CurMBB;
10920         BTB->Default = Fallthrough;
10921 
10922         BTB->DefaultProb = UnhandledProbs;
10923         // If the cases in bit test don't form a contiguous range, we evenly
10924         // distribute the probability on the edge to Fallthrough to two
10925         // successors of CurMBB.
10926         if (!BTB->ContiguousRange) {
10927           BTB->Prob += DefaultProb / 2;
10928           BTB->DefaultProb -= DefaultProb / 2;
10929         }
10930 
10931         if (FallthroughUnreachable)
10932           BTB->FallthroughUnreachable = true;
10933 
10934         // If we're in the right place, emit the bit test header right now.
10935         if (CurMBB == SwitchMBB) {
10936           visitBitTestHeader(*BTB, SwitchMBB);
10937           BTB->Emitted = true;
10938         }
10939         break;
10940       }
10941       case CC_Range: {
10942         const Value *RHS, *LHS, *MHS;
10943         ISD::CondCode CC;
10944         if (I->Low == I->High) {
10945           // Check Cond == I->Low.
10946           CC = ISD::SETEQ;
10947           LHS = Cond;
10948           RHS=I->Low;
10949           MHS = nullptr;
10950         } else {
10951           // Check I->Low <= Cond <= I->High.
10952           CC = ISD::SETLE;
10953           LHS = I->Low;
10954           MHS = Cond;
10955           RHS = I->High;
10956         }
10957 
10958         // If Fallthrough is unreachable, fold away the comparison.
10959         if (FallthroughUnreachable)
10960           CC = ISD::SETTRUE;
10961 
10962         // The false probability is the sum of all unhandled cases.
10963         CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
10964                      getCurSDLoc(), I->Prob, UnhandledProbs);
10965 
10966         if (CurMBB == SwitchMBB)
10967           visitSwitchCase(CB, SwitchMBB);
10968         else
10969           SL->SwitchCases.push_back(CB);
10970 
10971         break;
10972       }
10973     }
10974     CurMBB = Fallthrough;
10975   }
10976 }
10977 
10978 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
10979                                               CaseClusterIt First,
10980                                               CaseClusterIt Last) {
10981   return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
10982     if (X.Prob != CC.Prob)
10983       return X.Prob > CC.Prob;
10984 
10985     // Ties are broken by comparing the case value.
10986     return X.Low->getValue().slt(CC.Low->getValue());
10987   });
10988 }
10989 
10990 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
10991                                         const SwitchWorkListItem &W,
10992                                         Value *Cond,
10993                                         MachineBasicBlock *SwitchMBB) {
10994   assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
10995          "Clusters not sorted?");
10996 
10997   assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
10998 
10999   // Balance the tree based on branch probabilities to create a near-optimal (in
11000   // terms of search time given key frequency) binary search tree. See e.g. Kurt
11001   // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
11002   CaseClusterIt LastLeft = W.FirstCluster;
11003   CaseClusterIt FirstRight = W.LastCluster;
11004   auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
11005   auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
11006 
11007   // Move LastLeft and FirstRight towards each other from opposite directions to
11008   // find a partitioning of the clusters which balances the probability on both
11009   // sides. If LeftProb and RightProb are equal, alternate which side is
11010   // taken to ensure 0-probability nodes are distributed evenly.
11011   unsigned I = 0;
11012   while (LastLeft + 1 < FirstRight) {
11013     if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
11014       LeftProb += (++LastLeft)->Prob;
11015     else
11016       RightProb += (--FirstRight)->Prob;
11017     I++;
11018   }
11019 
11020   while (true) {
11021     // Our binary search tree differs from a typical BST in that ours can have up
11022     // to three values in each leaf. The pivot selection above doesn't take that
11023     // into account, which means the tree might require more nodes and be less
11024     // efficient. We compensate for this here.
11025 
11026     unsigned NumLeft = LastLeft - W.FirstCluster + 1;
11027     unsigned NumRight = W.LastCluster - FirstRight + 1;
11028 
11029     if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
11030       // If one side has less than 3 clusters, and the other has more than 3,
11031       // consider taking a cluster from the other side.
11032 
11033       if (NumLeft < NumRight) {
11034         // Consider moving the first cluster on the right to the left side.
11035         CaseCluster &CC = *FirstRight;
11036         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
11037         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
11038         if (LeftSideRank <= RightSideRank) {
11039           // Moving the cluster to the left does not demote it.
11040           ++LastLeft;
11041           ++FirstRight;
11042           continue;
11043         }
11044       } else {
11045         assert(NumRight < NumLeft);
11046         // Consider moving the last element on the left to the right side.
11047         CaseCluster &CC = *LastLeft;
11048         unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
11049         unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
11050         if (RightSideRank <= LeftSideRank) {
11051           // Moving the cluster to the right does not demot it.
11052           --LastLeft;
11053           --FirstRight;
11054           continue;
11055         }
11056       }
11057     }
11058     break;
11059   }
11060 
11061   assert(LastLeft + 1 == FirstRight);
11062   assert(LastLeft >= W.FirstCluster);
11063   assert(FirstRight <= W.LastCluster);
11064 
11065   // Use the first element on the right as pivot since we will make less-than
11066   // comparisons against it.
11067   CaseClusterIt PivotCluster = FirstRight;
11068   assert(PivotCluster > W.FirstCluster);
11069   assert(PivotCluster <= W.LastCluster);
11070 
11071   CaseClusterIt FirstLeft = W.FirstCluster;
11072   CaseClusterIt LastRight = W.LastCluster;
11073 
11074   const ConstantInt *Pivot = PivotCluster->Low;
11075 
11076   // New blocks will be inserted immediately after the current one.
11077   MachineFunction::iterator BBI(W.MBB);
11078   ++BBI;
11079 
11080   // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
11081   // we can branch to its destination directly if it's squeezed exactly in
11082   // between the known lower bound and Pivot - 1.
11083   MachineBasicBlock *LeftMBB;
11084   if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
11085       FirstLeft->Low == W.GE &&
11086       (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
11087     LeftMBB = FirstLeft->MBB;
11088   } else {
11089     LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11090     FuncInfo.MF->insert(BBI, LeftMBB);
11091     WorkList.push_back(
11092         {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
11093     // Put Cond in a virtual register to make it available from the new blocks.
11094     ExportFromCurrentBlock(Cond);
11095   }
11096 
11097   // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
11098   // single cluster, RHS.Low == Pivot, and we can branch to its destination
11099   // directly if RHS.High equals the current upper bound.
11100   MachineBasicBlock *RightMBB;
11101   if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
11102       W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
11103     RightMBB = FirstRight->MBB;
11104   } else {
11105     RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11106     FuncInfo.MF->insert(BBI, RightMBB);
11107     WorkList.push_back(
11108         {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
11109     // Put Cond in a virtual register to make it available from the new blocks.
11110     ExportFromCurrentBlock(Cond);
11111   }
11112 
11113   // Create the CaseBlock record that will be used to lower the branch.
11114   CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
11115                getCurSDLoc(), LeftProb, RightProb);
11116 
11117   if (W.MBB == SwitchMBB)
11118     visitSwitchCase(CB, SwitchMBB);
11119   else
11120     SL->SwitchCases.push_back(CB);
11121 }
11122 
11123 // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11124 // from the swith statement.
11125 static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
11126                                             BranchProbability PeeledCaseProb) {
11127   if (PeeledCaseProb == BranchProbability::getOne())
11128     return BranchProbability::getZero();
11129   BranchProbability SwitchProb = PeeledCaseProb.getCompl();
11130 
11131   uint32_t Numerator = CaseProb.getNumerator();
11132   uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
11133   return BranchProbability(Numerator, std::max(Numerator, Denominator));
11134 }
11135 
11136 // Try to peel the top probability case if it exceeds the threshold.
11137 // Return current MachineBasicBlock for the switch statement if the peeling
11138 // does not occur.
11139 // If the peeling is performed, return the newly created MachineBasicBlock
11140 // for the peeled switch statement. Also update Clusters to remove the peeled
11141 // case. PeeledCaseProb is the BranchProbability for the peeled case.
11142 MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
11143     const SwitchInst &SI, CaseClusterVector &Clusters,
11144     BranchProbability &PeeledCaseProb) {
11145   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11146   // Don't perform if there is only one cluster or optimizing for size.
11147   if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
11148       TM.getOptLevel() == CodeGenOpt::None ||
11149       SwitchMBB->getParent()->getFunction().hasMinSize())
11150     return SwitchMBB;
11151 
11152   BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
11153   unsigned PeeledCaseIndex = 0;
11154   bool SwitchPeeled = false;
11155   for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
11156     CaseCluster &CC = Clusters[Index];
11157     if (CC.Prob < TopCaseProb)
11158       continue;
11159     TopCaseProb = CC.Prob;
11160     PeeledCaseIndex = Index;
11161     SwitchPeeled = true;
11162   }
11163   if (!SwitchPeeled)
11164     return SwitchMBB;
11165 
11166   LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
11167                     << TopCaseProb << "\n");
11168 
11169   // Record the MBB for the peeled switch statement.
11170   MachineFunction::iterator BBI(SwitchMBB);
11171   ++BBI;
11172   MachineBasicBlock *PeeledSwitchMBB =
11173       FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
11174   FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
11175 
11176   ExportFromCurrentBlock(SI.getCondition());
11177   auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
11178   SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
11179                           nullptr,   nullptr,      TopCaseProb.getCompl()};
11180   lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
11181 
11182   Clusters.erase(PeeledCaseIt);
11183   for (CaseCluster &CC : Clusters) {
11184     LLVM_DEBUG(
11185         dbgs() << "Scale the probablity for one cluster, before scaling: "
11186                << CC.Prob << "\n");
11187     CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
11188     LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
11189   }
11190   PeeledCaseProb = TopCaseProb;
11191   return PeeledSwitchMBB;
11192 }
11193 
11194 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
11195   // Extract cases from the switch.
11196   BranchProbabilityInfo *BPI = FuncInfo.BPI;
11197   CaseClusterVector Clusters;
11198   Clusters.reserve(SI.getNumCases());
11199   for (auto I : SI.cases()) {
11200     MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
11201     const ConstantInt *CaseVal = I.getCaseValue();
11202     BranchProbability Prob =
11203         BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
11204             : BranchProbability(1, SI.getNumCases() + 1);
11205     Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
11206   }
11207 
11208   MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
11209 
11210   // Cluster adjacent cases with the same destination. We do this at all
11211   // optimization levels because it's cheap to do and will make codegen faster
11212   // if there are many clusters.
11213   sortAndRangeify(Clusters);
11214 
11215   // The branch probablity of the peeled case.
11216   BranchProbability PeeledCaseProb = BranchProbability::getZero();
11217   MachineBasicBlock *PeeledSwitchMBB =
11218       peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
11219 
11220   // If there is only the default destination, jump there directly.
11221   MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11222   if (Clusters.empty()) {
11223     assert(PeeledSwitchMBB == SwitchMBB);
11224     SwitchMBB->addSuccessor(DefaultMBB);
11225     if (DefaultMBB != NextBlock(SwitchMBB)) {
11226       DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
11227                               getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
11228     }
11229     return;
11230   }
11231 
11232   SL->findJumpTables(Clusters, &SI, DefaultMBB, DAG.getPSI(), DAG.getBFI());
11233   SL->findBitTestClusters(Clusters, &SI);
11234 
11235   LLVM_DEBUG({
11236     dbgs() << "Case clusters: ";
11237     for (const CaseCluster &C : Clusters) {
11238       if (C.Kind == CC_JumpTable)
11239         dbgs() << "JT:";
11240       if (C.Kind == CC_BitTests)
11241         dbgs() << "BT:";
11242 
11243       C.Low->getValue().print(dbgs(), true);
11244       if (C.Low != C.High) {
11245         dbgs() << '-';
11246         C.High->getValue().print(dbgs(), true);
11247       }
11248       dbgs() << ' ';
11249     }
11250     dbgs() << '\n';
11251   });
11252 
11253   assert(!Clusters.empty());
11254   SwitchWorkList WorkList;
11255   CaseClusterIt First = Clusters.begin();
11256   CaseClusterIt Last = Clusters.end() - 1;
11257   auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
11258   // Scale the branchprobability for DefaultMBB if the peel occurs and
11259   // DefaultMBB is not replaced.
11260   if (PeeledCaseProb != BranchProbability::getZero() &&
11261       DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
11262     DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
11263   WorkList.push_back(
11264       {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
11265 
11266   while (!WorkList.empty()) {
11267     SwitchWorkListItem W = WorkList.pop_back_val();
11268     unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
11269 
11270     if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
11271         !DefaultMBB->getParent()->getFunction().hasMinSize()) {
11272       // For optimized builds, lower large range as a balanced binary tree.
11273       splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
11274       continue;
11275     }
11276 
11277     lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
11278   }
11279 }
11280 
11281 void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
11282   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11283   auto DL = getCurSDLoc();
11284   EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11285   setValue(&I, DAG.getStepVector(DL, ResultVT));
11286 }
11287 
11288 void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
11289   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11290   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11291 
11292   SDLoc DL = getCurSDLoc();
11293   SDValue V = getValue(I.getOperand(0));
11294   assert(VT == V.getValueType() && "Malformed vector.reverse!");
11295 
11296   if (VT.isScalableVector()) {
11297     setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
11298     return;
11299   }
11300 
11301   // Use VECTOR_SHUFFLE for the fixed-length vector
11302   // to maintain existing behavior.
11303   SmallVector<int, 8> Mask;
11304   unsigned NumElts = VT.getVectorMinNumElements();
11305   for (unsigned i = 0; i != NumElts; ++i)
11306     Mask.push_back(NumElts - 1 - i);
11307 
11308   setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
11309 }
11310 
11311 void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
11312   SmallVector<EVT, 4> ValueVTs;
11313   ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
11314                   ValueVTs);
11315   unsigned NumValues = ValueVTs.size();
11316   if (NumValues == 0) return;
11317 
11318   SmallVector<SDValue, 4> Values(NumValues);
11319   SDValue Op = getValue(I.getOperand(0));
11320 
11321   for (unsigned i = 0; i != NumValues; ++i)
11322     Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
11323                             SDValue(Op.getNode(), Op.getResNo() + i));
11324 
11325   setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
11326                            DAG.getVTList(ValueVTs), Values));
11327 }
11328 
11329 void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
11330   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11331   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11332 
11333   SDLoc DL = getCurSDLoc();
11334   SDValue V1 = getValue(I.getOperand(0));
11335   SDValue V2 = getValue(I.getOperand(1));
11336   int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
11337 
11338   // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
11339   if (VT.isScalableVector()) {
11340     MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
11341     setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
11342                              DAG.getConstant(Imm, DL, IdxVT)));
11343     return;
11344   }
11345 
11346   unsigned NumElts = VT.getVectorNumElements();
11347 
11348   uint64_t Idx = (NumElts + Imm) % NumElts;
11349 
11350   // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
11351   SmallVector<int, 8> Mask;
11352   for (unsigned i = 0; i < NumElts; ++i)
11353     Mask.push_back(Idx + i);
11354   setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
11355 }
11356