xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp (revision 85868e8a1daeaae7a0e48effb2ea2310ae3b02c6)
1 //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file includes support code use by SelectionDAGBuilder when lowering a
10 // statepoint sequence in SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "StatepointLowering.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/CodeGen/FunctionLoweringInfo.h"
24 #include "llvm/CodeGen/GCMetadata.h"
25 #include "llvm/CodeGen/GCStrategy.h"
26 #include "llvm/CodeGen/ISDOpcodes.h"
27 #include "llvm/CodeGen/MachineFrameInfo.h"
28 #include "llvm/CodeGen/MachineFunction.h"
29 #include "llvm/CodeGen/MachineMemOperand.h"
30 #include "llvm/CodeGen/RuntimeLibcalls.h"
31 #include "llvm/CodeGen/SelectionDAG.h"
32 #include "llvm/CodeGen/SelectionDAGNodes.h"
33 #include "llvm/CodeGen/StackMaps.h"
34 #include "llvm/CodeGen/TargetLowering.h"
35 #include "llvm/CodeGen/TargetOpcodes.h"
36 #include "llvm/IR/CallingConv.h"
37 #include "llvm/IR/DerivedTypes.h"
38 #include "llvm/IR/Instruction.h"
39 #include "llvm/IR/Instructions.h"
40 #include "llvm/IR/LLVMContext.h"
41 #include "llvm/IR/Statepoint.h"
42 #include "llvm/IR/Type.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/MachineValueType.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
47 #include <cassert>
48 #include <cstddef>
49 #include <cstdint>
50 #include <iterator>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "statepoint-lowering"
57 
58 STATISTIC(NumSlotsAllocatedForStatepoints,
59           "Number of stack slots allocated for statepoints");
60 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
61 STATISTIC(StatepointMaxSlotsRequired,
62           "Maximum number of stack slots required for a singe statepoint");
63 
64 static void pushStackMapConstant(SmallVectorImpl<SDValue>& Ops,
65                                  SelectionDAGBuilder &Builder, uint64_t Value) {
66   SDLoc L = Builder.getCurSDLoc();
67   Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
68                                               MVT::i64));
69   Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
70 }
71 
72 void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
73   // Consistency check
74   assert(PendingGCRelocateCalls.empty() &&
75          "Trying to visit statepoint before finished processing previous one");
76   Locations.clear();
77   NextSlotToAllocate = 0;
78   // Need to resize this on each safepoint - we need the two to stay in sync and
79   // the clear patterns of a SelectionDAGBuilder have no relation to
80   // FunctionLoweringInfo.  Also need to ensure used bits get cleared.
81   AllocatedStackSlots.clear();
82   AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
83 }
84 
85 void StatepointLoweringState::clear() {
86   Locations.clear();
87   AllocatedStackSlots.clear();
88   assert(PendingGCRelocateCalls.empty() &&
89          "cleared before statepoint sequence completed");
90 }
91 
92 SDValue
93 StatepointLoweringState::allocateStackSlot(EVT ValueType,
94                                            SelectionDAGBuilder &Builder) {
95   NumSlotsAllocatedForStatepoints++;
96   MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
97 
98   unsigned SpillSize = ValueType.getStoreSize();
99   assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
100 
101   // First look for a previously created stack slot which is not in
102   // use (accounting for the fact arbitrary slots may already be
103   // reserved), or to create a new stack slot and use it.
104 
105   const size_t NumSlots = AllocatedStackSlots.size();
106   assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
107 
108   assert(AllocatedStackSlots.size() ==
109          Builder.FuncInfo.StatepointStackSlots.size() &&
110          "Broken invariant");
111 
112   for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
113     if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
114       const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
115       if (MFI.getObjectSize(FI) == SpillSize) {
116         AllocatedStackSlots.set(NextSlotToAllocate);
117         // TODO: Is ValueType the right thing to use here?
118         return Builder.DAG.getFrameIndex(FI, ValueType);
119       }
120     }
121   }
122 
123   // Couldn't find a free slot, so create a new one:
124 
125   SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
126   const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
127   MFI.markAsStatepointSpillSlotObjectIndex(FI);
128 
129   Builder.FuncInfo.StatepointStackSlots.push_back(FI);
130   AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
131   assert(AllocatedStackSlots.size() ==
132          Builder.FuncInfo.StatepointStackSlots.size() &&
133          "Broken invariant");
134 
135   StatepointMaxSlotsRequired.updateMax(
136       Builder.FuncInfo.StatepointStackSlots.size());
137 
138   return SpillSlot;
139 }
140 
141 /// Utility function for reservePreviousStackSlotForValue. Tries to find
142 /// stack slot index to which we have spilled value for previous statepoints.
143 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
144 static Optional<int> findPreviousSpillSlot(const Value *Val,
145                                            SelectionDAGBuilder &Builder,
146                                            int LookUpDepth) {
147   // Can not look any further - give up now
148   if (LookUpDepth <= 0)
149     return None;
150 
151   // Spill location is known for gc relocates
152   if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
153     const auto &SpillMap =
154         Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
155 
156     auto It = SpillMap.find(Relocate->getDerivedPtr());
157     if (It == SpillMap.end())
158       return None;
159 
160     return It->second;
161   }
162 
163   // Look through bitcast instructions.
164   if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
165     return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
166 
167   // Look through phi nodes
168   // All incoming values should have same known stack slot, otherwise result
169   // is unknown.
170   if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
171     Optional<int> MergedResult = None;
172 
173     for (auto &IncomingValue : Phi->incoming_values()) {
174       Optional<int> SpillSlot =
175           findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
176       if (!SpillSlot.hasValue())
177         return None;
178 
179       if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
180         return None;
181 
182       MergedResult = SpillSlot;
183     }
184     return MergedResult;
185   }
186 
187   // TODO: We can do better for PHI nodes. In cases like this:
188   //   ptr = phi(relocated_pointer, not_relocated_pointer)
189   //   statepoint(ptr)
190   // We will return that stack slot for ptr is unknown. And later we might
191   // assign different stack slots for ptr and relocated_pointer. This limits
192   // llvm's ability to remove redundant stores.
193   // Unfortunately it's hard to accomplish in current infrastructure.
194   // We use this function to eliminate spill store completely, while
195   // in example we still need to emit store, but instead of any location
196   // we need to use special "preferred" location.
197 
198   // TODO: handle simple updates.  If a value is modified and the original
199   // value is no longer live, it would be nice to put the modified value in the
200   // same slot.  This allows folding of the memory accesses for some
201   // instructions types (like an increment).
202   //   statepoint (i)
203   //   i1 = i+1
204   //   statepoint (i1)
205   // However we need to be careful for cases like this:
206   //   statepoint(i)
207   //   i1 = i+1
208   //   statepoint(i, i1)
209   // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
210   // put handling of simple modifications in this function like it's done
211   // for bitcasts we might end up reserving i's slot for 'i+1' because order in
212   // which we visit values is unspecified.
213 
214   // Don't know any information about this instruction
215   return None;
216 }
217 
218 /// Try to find existing copies of the incoming values in stack slots used for
219 /// statepoint spilling.  If we can find a spill slot for the incoming value,
220 /// mark that slot as allocated, and reuse the same slot for this safepoint.
221 /// This helps to avoid series of loads and stores that only serve to reshuffle
222 /// values on the stack between calls.
223 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
224                                              SelectionDAGBuilder &Builder) {
225   SDValue Incoming = Builder.getValue(IncomingValue);
226 
227   if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
228     // We won't need to spill this, so no need to check for previously
229     // allocated stack slots
230     return;
231   }
232 
233   SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
234   if (OldLocation.getNode())
235     // Duplicates in input
236     return;
237 
238   const int LookUpDepth = 6;
239   Optional<int> Index =
240       findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
241   if (!Index.hasValue())
242     return;
243 
244   const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
245 
246   auto SlotIt = find(StatepointSlots, *Index);
247   assert(SlotIt != StatepointSlots.end() &&
248          "Value spilled to the unknown stack slot");
249 
250   // This is one of our dedicated lowering slots
251   const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
252   if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
253     // stack slot already assigned to someone else, can't use it!
254     // TODO: currently we reserve space for gc arguments after doing
255     // normal allocation for deopt arguments.  We should reserve for
256     // _all_ deopt and gc arguments, then start allocating.  This
257     // will prevent some moves being inserted when vm state changes,
258     // but gc state doesn't between two calls.
259     return;
260   }
261   // Reserve this stack slot
262   Builder.StatepointLowering.reserveStackSlot(Offset);
263 
264   // Cache this slot so we find it when going through the normal
265   // assignment loop.
266   SDValue Loc =
267       Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
268   Builder.StatepointLowering.setLocation(Incoming, Loc);
269 }
270 
271 /// Remove any duplicate (as SDValues) from the derived pointer pairs.  This
272 /// is not required for correctness.  It's purpose is to reduce the size of
273 /// StackMap section.  It has no effect on the number of spill slots required
274 /// or the actual lowering.
275 static void
276 removeDuplicateGCPtrs(SmallVectorImpl<const Value *> &Bases,
277                       SmallVectorImpl<const Value *> &Ptrs,
278                       SmallVectorImpl<const GCRelocateInst *> &Relocs,
279                       SelectionDAGBuilder &Builder,
280                       FunctionLoweringInfo::StatepointSpillMap &SSM) {
281   DenseMap<SDValue, const Value *> Seen;
282 
283   SmallVector<const Value *, 64> NewBases, NewPtrs;
284   SmallVector<const GCRelocateInst *, 64> NewRelocs;
285   for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
286     SDValue SD = Builder.getValue(Ptrs[i]);
287     auto SeenIt = Seen.find(SD);
288 
289     if (SeenIt == Seen.end()) {
290       // Only add non-duplicates
291       NewBases.push_back(Bases[i]);
292       NewPtrs.push_back(Ptrs[i]);
293       NewRelocs.push_back(Relocs[i]);
294       Seen[SD] = Ptrs[i];
295     } else {
296       // Duplicate pointer found, note in SSM and move on:
297       SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
298     }
299   }
300   assert(Bases.size() >= NewBases.size());
301   assert(Ptrs.size() >= NewPtrs.size());
302   assert(Relocs.size() >= NewRelocs.size());
303   Bases = NewBases;
304   Ptrs = NewPtrs;
305   Relocs = NewRelocs;
306   assert(Ptrs.size() == Bases.size());
307   assert(Ptrs.size() == Relocs.size());
308 }
309 
310 /// Extract call from statepoint, lower it and return pointer to the
311 /// call node. Also update NodeMap so that getValue(statepoint) will
312 /// reference lowered call result
313 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
314     SelectionDAGBuilder::StatepointLoweringInfo &SI,
315     SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
316   SDValue ReturnValue, CallEndVal;
317   std::tie(ReturnValue, CallEndVal) =
318       Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
319   SDNode *CallEnd = CallEndVal.getNode();
320 
321   // Get a call instruction from the call sequence chain.  Tail calls are not
322   // allowed.  The following code is essentially reverse engineering X86's
323   // LowerCallTo.
324   //
325   // We are expecting DAG to have the following form:
326   //
327   // ch = eh_label (only in case of invoke statepoint)
328   //   ch, glue = callseq_start ch
329   //   ch, glue = X86::Call ch, glue
330   //   ch, glue = callseq_end ch, glue
331   //   get_return_value ch, glue
332   //
333   // get_return_value can either be a sequence of CopyFromReg instructions
334   // to grab the return value from the return register(s), or it can be a LOAD
335   // to load a value returned by reference via a stack slot.
336 
337   bool HasDef = !SI.CLI.RetTy->isVoidTy();
338   if (HasDef) {
339     if (CallEnd->getOpcode() == ISD::LOAD)
340       CallEnd = CallEnd->getOperand(0).getNode();
341     else
342       while (CallEnd->getOpcode() == ISD::CopyFromReg)
343         CallEnd = CallEnd->getOperand(0).getNode();
344   }
345 
346   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
347   return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
348 }
349 
350 static MachineMemOperand* getMachineMemOperand(MachineFunction &MF,
351                                                FrameIndexSDNode &FI) {
352   auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex());
353   auto MMOFlags = MachineMemOperand::MOStore |
354     MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
355   auto &MFI = MF.getFrameInfo();
356   return MF.getMachineMemOperand(PtrInfo, MMOFlags,
357                                  MFI.getObjectSize(FI.getIndex()),
358                                  MFI.getObjectAlignment(FI.getIndex()));
359 }
360 
361 /// Spill a value incoming to the statepoint. It might be either part of
362 /// vmstate
363 /// or gcstate. In both cases unconditionally spill it on the stack unless it
364 /// is a null constant. Return pair with first element being frame index
365 /// containing saved value and second element with outgoing chain from the
366 /// emitted store
367 static std::tuple<SDValue, SDValue, MachineMemOperand*>
368 spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
369                              SelectionDAGBuilder &Builder) {
370   SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
371   MachineMemOperand* MMO = nullptr;
372 
373   // Emit new store if we didn't do it for this ptr before
374   if (!Loc.getNode()) {
375     Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
376                                                        Builder);
377     int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
378     // We use TargetFrameIndex so that isel will not select it into LEA
379     Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
380 
381     // Right now we always allocate spill slots that are of the same
382     // size as the value we're about to spill (the size of spillee can
383     // vary since we spill vectors of pointers too).  At some point we
384     // can consider allowing spills of smaller values to larger slots
385     // (i.e. change the '==' in the assert below to a '>=').
386     MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
387     assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
388            "Bad spill:  stack slot does not match!");
389 
390     // Note: Using the alignment of the spill slot (rather than the abi or
391     // preferred alignment) is required for correctness when dealing with spill
392     // slots with preferred alignments larger than frame alignment..
393     auto &MF = Builder.DAG.getMachineFunction();
394     auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
395     auto *StoreMMO =
396       MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
397                               MFI.getObjectSize(Index),
398                               MFI.getObjectAlignment(Index));
399     Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
400                                  StoreMMO);
401 
402     MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
403 
404     Builder.StatepointLowering.setLocation(Incoming, Loc);
405   }
406 
407   assert(Loc.getNode());
408   return std::make_tuple(Loc, Chain, MMO);
409 }
410 
411 /// Lower a single value incoming to a statepoint node.  This value can be
412 /// either a deopt value or a gc value, the handling is the same.  We special
413 /// case constants and allocas, then fall back to spilling if required.
414 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
415                                          SmallVectorImpl<SDValue> &Ops,
416                                          SmallVectorImpl<MachineMemOperand*> &MemRefs,
417                                          SelectionDAGBuilder &Builder) {
418   // Note: We know all of these spills are independent, but don't bother to
419   // exploit that chain wise.  DAGCombine will happily do so as needed, so
420   // doing it here would be a small compile time win at most.
421   SDValue Chain = Builder.getRoot();
422 
423   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
424     // If the original value was a constant, make sure it gets recorded as
425     // such in the stackmap.  This is required so that the consumer can
426     // parse any internal format to the deopt state.  It also handles null
427     // pointers and other constant pointers in GC states.  Note the constant
428     // vectors do not appear to actually hit this path and that anything larger
429     // than an i64 value (not type!) will fail asserts here.
430     pushStackMapConstant(Ops, Builder, C->getSExtValue());
431   } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
432     // This handles allocas as arguments to the statepoint (this is only
433     // really meaningful for a deopt value.  For GC, we'd be trying to
434     // relocate the address of the alloca itself?)
435     assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
436            "Incoming value is a frame index!");
437     Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
438                                                   Builder.getFrameIndexTy()));
439 
440     auto &MF = Builder.DAG.getMachineFunction();
441     auto *MMO = getMachineMemOperand(MF, *FI);
442     MemRefs.push_back(MMO);
443 
444   } else if (LiveInOnly) {
445     // If this value is live in (not live-on-return, or live-through), we can
446     // treat it the same way patchpoint treats it's "live in" values.  We'll
447     // end up folding some of these into stack references, but they'll be
448     // handled by the register allocator.  Note that we do not have the notion
449     // of a late use so these values might be placed in registers which are
450     // clobbered by the call.  This is fine for live-in.
451     Ops.push_back(Incoming);
452   } else {
453     // Otherwise, locate a spill slot and explicitly spill it so it
454     // can be found by the runtime later.  We currently do not support
455     // tracking values through callee saved registers to their eventual
456     // spill location.  This would be a useful optimization, but would
457     // need to be optional since it requires a lot of complexity on the
458     // runtime side which not all would support.
459     auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
460     Ops.push_back(std::get<0>(Res));
461     if (auto *MMO = std::get<2>(Res))
462       MemRefs.push_back(MMO);
463     Chain = std::get<1>(Res);;
464   }
465 
466   Builder.DAG.setRoot(Chain);
467 }
468 
469 /// Lower deopt state and gc pointer arguments of the statepoint.  The actual
470 /// lowering is described in lowerIncomingStatepointValue.  This function is
471 /// responsible for lowering everything in the right position and playing some
472 /// tricks to avoid redundant stack manipulation where possible.  On
473 /// completion, 'Ops' will contain ready to use operands for machine code
474 /// statepoint. The chain nodes will have already been created and the DAG root
475 /// will be set to the last value spilled (if any were).
476 static void
477 lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
478                         SmallVectorImpl<MachineMemOperand*> &MemRefs,                                    SelectionDAGBuilder::StatepointLoweringInfo &SI,
479                         SelectionDAGBuilder &Builder) {
480   // Lower the deopt and gc arguments for this statepoint.  Layout will be:
481   // deopt argument length, deopt arguments.., gc arguments...
482 #ifndef NDEBUG
483   if (auto *GFI = Builder.GFI) {
484     // Check that each of the gc pointer and bases we've gotten out of the
485     // safepoint is something the strategy thinks might be a pointer (or vector
486     // of pointers) into the GC heap.  This is basically just here to help catch
487     // errors during statepoint insertion. TODO: This should actually be in the
488     // Verifier, but we can't get to the GCStrategy from there (yet).
489     GCStrategy &S = GFI->getStrategy();
490     for (const Value *V : SI.Bases) {
491       auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
492       if (Opt.hasValue()) {
493         assert(Opt.getValue() &&
494                "non gc managed base pointer found in statepoint");
495       }
496     }
497     for (const Value *V : SI.Ptrs) {
498       auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
499       if (Opt.hasValue()) {
500         assert(Opt.getValue() &&
501                "non gc managed derived pointer found in statepoint");
502       }
503     }
504     assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
505   } else {
506     assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
507     assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
508   }
509 #endif
510 
511   // Figure out what lowering strategy we're going to use for each part
512   // Note: Is is conservatively correct to lower both "live-in" and "live-out"
513   // as "live-through". A "live-through" variable is one which is "live-in",
514   // "live-out", and live throughout the lifetime of the call (i.e. we can find
515   // it from any PC within the transitive callee of the statepoint).  In
516   // particular, if the callee spills callee preserved registers we may not
517   // be able to find a value placed in that register during the call.  This is
518   // fine for live-out, but not for live-through.  If we were willing to make
519   // assumptions about the code generator producing the callee, we could
520   // potentially allow live-through values in callee saved registers.
521   const bool LiveInDeopt =
522     SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
523 
524   auto isGCValue =[&](const Value *V) {
525     return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
526   };
527 
528   // Before we actually start lowering (and allocating spill slots for values),
529   // reserve any stack slots which we judge to be profitable to reuse for a
530   // particular value.  This is purely an optimization over the code below and
531   // doesn't change semantics at all.  It is important for performance that we
532   // reserve slots for both deopt and gc values before lowering either.
533   for (const Value *V : SI.DeoptState) {
534     if (!LiveInDeopt || isGCValue(V))
535       reservePreviousStackSlotForValue(V, Builder);
536   }
537   for (unsigned i = 0; i < SI.Bases.size(); ++i) {
538     reservePreviousStackSlotForValue(SI.Bases[i], Builder);
539     reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
540   }
541 
542   // First, prefix the list with the number of unique values to be
543   // lowered.  Note that this is the number of *Values* not the
544   // number of SDValues required to lower them.
545   const int NumVMSArgs = SI.DeoptState.size();
546   pushStackMapConstant(Ops, Builder, NumVMSArgs);
547 
548   // The vm state arguments are lowered in an opaque manner.  We do not know
549   // what type of values are contained within.
550   for (const Value *V : SI.DeoptState) {
551     SDValue Incoming;
552     // If this is a function argument at a static frame index, generate it as
553     // the frame index.
554     if (const Argument *Arg = dyn_cast<Argument>(V)) {
555       int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
556       if (FI != INT_MAX)
557         Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
558     }
559     if (!Incoming.getNode())
560       Incoming = Builder.getValue(V);
561     const bool LiveInValue = LiveInDeopt && !isGCValue(V);
562     lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, MemRefs, Builder);
563   }
564 
565   // Finally, go ahead and lower all the gc arguments.  There's no prefixed
566   // length for this one.  After lowering, we'll have the base and pointer
567   // arrays interwoven with each (lowered) base pointer immediately followed by
568   // it's (lowered) derived pointer.  i.e
569   // (base[0], ptr[0], base[1], ptr[1], ...)
570   for (unsigned i = 0; i < SI.Bases.size(); ++i) {
571     const Value *Base = SI.Bases[i];
572     lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
573                                  Ops, MemRefs, Builder);
574 
575     const Value *Ptr = SI.Ptrs[i];
576     lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
577                                  Ops, MemRefs, Builder);
578   }
579 
580   // If there are any explicit spill slots passed to the statepoint, record
581   // them, but otherwise do not do anything special.  These are user provided
582   // allocas and give control over placement to the consumer.  In this case,
583   // it is the contents of the slot which may get updated, not the pointer to
584   // the alloca
585   for (Value *V : SI.GCArgs) {
586     SDValue Incoming = Builder.getValue(V);
587     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
588       // This handles allocas as arguments to the statepoint
589       assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
590              "Incoming value is a frame index!");
591       Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
592                                                     Builder.getFrameIndexTy()));
593 
594       auto &MF = Builder.DAG.getMachineFunction();
595       auto *MMO = getMachineMemOperand(MF, *FI);
596       MemRefs.push_back(MMO);
597     }
598   }
599 
600   // Record computed locations for all lowered values.
601   // This can not be embedded in lowering loops as we need to record *all*
602   // values, while previous loops account only values with unique SDValues.
603   const Instruction *StatepointInstr = SI.StatepointInstr;
604   auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
605 
606   for (const GCRelocateInst *Relocate : SI.GCRelocates) {
607     const Value *V = Relocate->getDerivedPtr();
608     SDValue SDV = Builder.getValue(V);
609     SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
610 
611     if (Loc.getNode()) {
612       SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
613     } else {
614       // Record value as visited, but not spilled. This is case for allocas
615       // and constants. For this values we can avoid emitting spill load while
616       // visiting corresponding gc_relocate.
617       // Actually we do not need to record them in this map at all.
618       // We do this only to check that we are not relocating any unvisited
619       // value.
620       SpillMap.SlotMap[V] = None;
621 
622       // Default llvm mechanisms for exporting values which are used in
623       // different basic blocks does not work for gc relocates.
624       // Note that it would be incorrect to teach llvm that all relocates are
625       // uses of the corresponding values so that it would automatically
626       // export them. Relocates of the spilled values does not use original
627       // value.
628       if (Relocate->getParent() != StatepointInstr->getParent())
629         Builder.ExportFromCurrentBlock(V);
630     }
631   }
632 }
633 
634 SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
635     SelectionDAGBuilder::StatepointLoweringInfo &SI) {
636   // The basic scheme here is that information about both the original call and
637   // the safepoint is encoded in the CallInst.  We create a temporary call and
638   // lower it, then reverse engineer the calling sequence.
639 
640   NumOfStatepoints++;
641   // Clear state
642   StatepointLowering.startNewStatepoint(*this);
643 
644 #ifndef NDEBUG
645   // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
646   // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
647   for (auto *Reloc : SI.GCRelocates)
648     if (Reloc->getParent() == SI.StatepointInstr->getParent())
649       StatepointLowering.scheduleRelocCall(*Reloc);
650 #endif
651 
652   // Remove any redundant llvm::Values which map to the same SDValue as another
653   // input.  Also has the effect of removing duplicates in the original
654   // llvm::Value input list as well.  This is a useful optimization for
655   // reducing the size of the StackMap section.  It has no other impact.
656   removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
657                         FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
658   assert(SI.Bases.size() == SI.Ptrs.size() &&
659          SI.Ptrs.size() == SI.GCRelocates.size());
660 
661   // Lower statepoint vmstate and gcstate arguments
662   SmallVector<SDValue, 10> LoweredMetaArgs;
663   SmallVector<MachineMemOperand*, 16> MemRefs;
664   lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, SI, *this);
665 
666   // Now that we've emitted the spills, we need to update the root so that the
667   // call sequence is ordered correctly.
668   SI.CLI.setChain(getRoot());
669 
670   // Get call node, we will replace it later with statepoint
671   SDValue ReturnVal;
672   SDNode *CallNode;
673   std::tie(ReturnVal, CallNode) =
674       lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
675 
676   // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
677   // nodes with all the appropriate arguments and return values.
678 
679   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
680   SDValue Chain = CallNode->getOperand(0);
681 
682   SDValue Glue;
683   bool CallHasIncomingGlue = CallNode->getGluedNode();
684   if (CallHasIncomingGlue) {
685     // Glue is always last operand
686     Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
687   }
688 
689   // Build the GC_TRANSITION_START node if necessary.
690   //
691   // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
692   // order in which they appear in the call to the statepoint intrinsic. If
693   // any of the operands is a pointer-typed, that operand is immediately
694   // followed by a SRCVALUE for the pointer that may be used during lowering
695   // (e.g. to form MachinePointerInfo values for loads/stores).
696   const bool IsGCTransition =
697       (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
698       (uint64_t)StatepointFlags::GCTransition;
699   if (IsGCTransition) {
700     SmallVector<SDValue, 8> TSOps;
701 
702     // Add chain
703     TSOps.push_back(Chain);
704 
705     // Add GC transition arguments
706     for (const Value *V : SI.GCTransitionArgs) {
707       TSOps.push_back(getValue(V));
708       if (V->getType()->isPointerTy())
709         TSOps.push_back(DAG.getSrcValue(V));
710     }
711 
712     // Add glue if necessary
713     if (CallHasIncomingGlue)
714       TSOps.push_back(Glue);
715 
716     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
717 
718     SDValue GCTransitionStart =
719         DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
720 
721     Chain = GCTransitionStart.getValue(0);
722     Glue = GCTransitionStart.getValue(1);
723   }
724 
725   // TODO: Currently, all of these operands are being marked as read/write in
726   // PrologEpilougeInserter.cpp, we should special case the VMState arguments
727   // and flags to be read-only.
728   SmallVector<SDValue, 40> Ops;
729 
730   // Add the <id> and <numBytes> constants.
731   Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
732   Ops.push_back(
733       DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
734 
735   // Calculate and push starting position of vmstate arguments
736   // Get number of arguments incoming directly into call node
737   unsigned NumCallRegArgs =
738       CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
739   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
740 
741   // Add call target
742   SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
743   Ops.push_back(CallTarget);
744 
745   // Add call arguments
746   // Get position of register mask in the call
747   SDNode::op_iterator RegMaskIt;
748   if (CallHasIncomingGlue)
749     RegMaskIt = CallNode->op_end() - 2;
750   else
751     RegMaskIt = CallNode->op_end() - 1;
752   Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
753 
754   // Add a constant argument for the calling convention
755   pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
756 
757   // Add a constant argument for the flags
758   uint64_t Flags = SI.StatepointFlags;
759   assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
760          "Unknown flag used");
761   pushStackMapConstant(Ops, *this, Flags);
762 
763   // Insert all vmstate and gcstate arguments
764   Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
765 
766   // Add register mask from call node
767   Ops.push_back(*RegMaskIt);
768 
769   // Add chain
770   Ops.push_back(Chain);
771 
772   // Same for the glue, but we add it only if original call had it
773   if (Glue.getNode())
774     Ops.push_back(Glue);
775 
776   // Compute return values.  Provide a glue output since we consume one as
777   // input.  This allows someone else to chain off us as needed.
778   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
779 
780   MachineSDNode *StatepointMCNode =
781     DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
782   DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
783 
784   SDNode *SinkNode = StatepointMCNode;
785 
786   // Build the GC_TRANSITION_END node if necessary.
787   //
788   // See the comment above regarding GC_TRANSITION_START for the layout of
789   // the operands to the GC_TRANSITION_END node.
790   if (IsGCTransition) {
791     SmallVector<SDValue, 8> TEOps;
792 
793     // Add chain
794     TEOps.push_back(SDValue(StatepointMCNode, 0));
795 
796     // Add GC transition arguments
797     for (const Value *V : SI.GCTransitionArgs) {
798       TEOps.push_back(getValue(V));
799       if (V->getType()->isPointerTy())
800         TEOps.push_back(DAG.getSrcValue(V));
801     }
802 
803     // Add glue
804     TEOps.push_back(SDValue(StatepointMCNode, 1));
805 
806     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
807 
808     SDValue GCTransitionStart =
809         DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
810 
811     SinkNode = GCTransitionStart.getNode();
812   }
813 
814   // Replace original call
815   DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
816   // Remove original call node
817   DAG.DeleteNode(CallNode);
818 
819   // DON'T set the root - under the assumption that it's already set past the
820   // inserted node we created.
821 
822   // TODO: A better future implementation would be to emit a single variable
823   // argument, variable return value STATEPOINT node here and then hookup the
824   // return value of each gc.relocate to the respective output of the
825   // previously emitted STATEPOINT value.  Unfortunately, this doesn't appear
826   // to actually be possible today.
827 
828   return ReturnVal;
829 }
830 
831 void
832 SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
833                                      const BasicBlock *EHPadBB /*= nullptr*/) {
834   assert(ISP.getCall()->getCallingConv() != CallingConv::AnyReg &&
835          "anyregcc is not supported on statepoints!");
836 
837 #ifndef NDEBUG
838   // If this is a malformed statepoint, report it early to simplify debugging.
839   // This should catch any IR level mistake that's made when constructing or
840   // transforming statepoints.
841   ISP.verify();
842 
843   // Check that the associated GCStrategy expects to encounter statepoints.
844   assert(GFI->getStrategy().useStatepoints() &&
845          "GCStrategy does not expect to encounter statepoints");
846 #endif
847 
848   SDValue ActualCallee;
849 
850   if (ISP.getNumPatchBytes() > 0) {
851     // If we've been asked to emit a nop sequence instead of a call instruction
852     // for this statepoint then don't lower the call target, but use a constant
853     // `null` instead.  Not lowering the call target lets statepoint clients get
854     // away without providing a physical address for the symbolic call target at
855     // link time.
856 
857     const auto &TLI = DAG.getTargetLoweringInfo();
858     const auto &DL = DAG.getDataLayout();
859 
860     unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
861     ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
862   } else {
863     ActualCallee = getValue(ISP.getCalledValue());
864   }
865 
866   StatepointLoweringInfo SI(DAG);
867   populateCallLoweringInfo(SI.CLI, ISP.getCall(),
868                            ImmutableStatepoint::CallArgsBeginPos,
869                            ISP.getNumCallArgs(), ActualCallee,
870                            ISP.getActualReturnType(), false /* IsPatchPoint */);
871 
872   for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
873     SI.GCRelocates.push_back(Relocate);
874     SI.Bases.push_back(Relocate->getBasePtr());
875     SI.Ptrs.push_back(Relocate->getDerivedPtr());
876   }
877 
878   SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
879   SI.StatepointInstr = ISP.getInstruction();
880   SI.GCTransitionArgs =
881       ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
882   SI.ID = ISP.getID();
883   SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
884   SI.StatepointFlags = ISP.getFlags();
885   SI.NumPatchBytes = ISP.getNumPatchBytes();
886   SI.EHPadBB = EHPadBB;
887 
888   SDValue ReturnValue = LowerAsSTATEPOINT(SI);
889 
890   // Export the result value if needed
891   const GCResultInst *GCResult = ISP.getGCResult();
892   Type *RetTy = ISP.getActualReturnType();
893   if (!RetTy->isVoidTy() && GCResult) {
894     if (GCResult->getParent() != ISP.getCall()->getParent()) {
895       // Result value will be used in a different basic block so we need to
896       // export it now.  Default exporting mechanism will not work here because
897       // statepoint call has a different type than the actual call. It means
898       // that by default llvm will create export register of the wrong type
899       // (always i32 in our case). So instead we need to create export register
900       // with correct type manually.
901       // TODO: To eliminate this problem we can remove gc.result intrinsics
902       //       completely and make statepoint call to return a tuple.
903       unsigned Reg = FuncInfo.CreateRegs(RetTy);
904       RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
905                        DAG.getDataLayout(), Reg, RetTy,
906                        ISP.getCall()->getCallingConv());
907       SDValue Chain = DAG.getEntryNode();
908 
909       RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
910       PendingExports.push_back(Chain);
911       FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
912     } else {
913       // Result value will be used in a same basic block. Don't export it or
914       // perform any explicit register copies.
915       // We'll replace the actuall call node shortly. gc_result will grab
916       // this value.
917       setValue(ISP.getInstruction(), ReturnValue);
918     }
919   } else {
920     // The token value is never used from here on, just generate a poison value
921     setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
922   }
923 }
924 
925 void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
926     const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
927     bool VarArgDisallowed, bool ForceVoidReturnTy) {
928   StatepointLoweringInfo SI(DAG);
929   unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
930   populateCallLoweringInfo(
931       SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee,
932       ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
933       false);
934   if (!VarArgDisallowed)
935     SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
936 
937   auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
938 
939   unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID;
940 
941   auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes());
942   SI.ID = SD.StatepointID.getValueOr(DefaultID);
943   SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
944 
945   SI.DeoptState =
946       ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
947   SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
948   SI.EHPadBB = EHPadBB;
949 
950   // NB! The GC arguments are deliberately left empty.
951 
952   if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
953     ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
954     setValue(Call, ReturnVal);
955   }
956 }
957 
958 void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
959     const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
960   LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
961                                    /* VarArgDisallowed = */ false,
962                                    /* ForceVoidReturnTy  = */ false);
963 }
964 
965 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
966   // The result value of the gc_result is simply the result of the actual
967   // call.  We've already emitted this, so just grab the value.
968   const Instruction *I = CI.getStatepoint();
969 
970   if (I->getParent() != CI.getParent()) {
971     // Statepoint is in different basic block so we should have stored call
972     // result in a virtual register.
973     // We can not use default getValue() functionality to copy value from this
974     // register because statepoint and actual call return types can be
975     // different, and getValue() will use CopyFromReg of the wrong type,
976     // which is always i32 in our case.
977     PointerType *CalleeType = cast<PointerType>(
978         ImmutableStatepoint(I).getCalledValue()->getType());
979     Type *RetTy =
980         cast<FunctionType>(CalleeType->getElementType())->getReturnType();
981     SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
982 
983     assert(CopyFromReg.getNode());
984     setValue(&CI, CopyFromReg);
985   } else {
986     setValue(&CI, getValue(I));
987   }
988 }
989 
990 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
991 #ifndef NDEBUG
992   // Consistency check
993   // We skip this check for relocates not in the same basic block as their
994   // statepoint. It would be too expensive to preserve validation info through
995   // different basic blocks.
996   if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
997     StatepointLowering.relocCallVisited(Relocate);
998 
999   auto *Ty = Relocate.getType()->getScalarType();
1000   if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
1001     assert(*IsManaged && "Non gc managed pointer relocated!");
1002 #endif
1003 
1004   const Value *DerivedPtr = Relocate.getDerivedPtr();
1005   SDValue SD = getValue(DerivedPtr);
1006 
1007   auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
1008   auto SlotIt = SpillMap.find(DerivedPtr);
1009   assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
1010   Optional<int> DerivedPtrLocation = SlotIt->second;
1011 
1012   // We didn't need to spill these special cases (constants and allocas).
1013   // See the handling in spillIncomingValueForStatepoint for detail.
1014   if (!DerivedPtrLocation) {
1015     setValue(&Relocate, SD);
1016     return;
1017   }
1018 
1019   unsigned Index = *DerivedPtrLocation;
1020   SDValue SpillSlot = DAG.getTargetFrameIndex(Index, getFrameIndexTy());
1021 
1022   // Note: We know all of these reloads are independent, but don't bother to
1023   // exploit that chain wise.  DAGCombine will happily do so as needed, so
1024   // doing it here would be a small compile time win at most.
1025   SDValue Chain = getRoot();
1026 
1027   auto &MF = DAG.getMachineFunction();
1028   auto &MFI = MF.getFrameInfo();
1029   auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
1030   auto *LoadMMO =
1031     MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
1032                             MFI.getObjectSize(Index),
1033                             MFI.getObjectAlignment(Index));
1034 
1035   auto LoadVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
1036                                                          Relocate.getType());
1037 
1038   SDValue SpillLoad = DAG.getLoad(LoadVT, getCurSDLoc(), Chain,
1039                                   SpillSlot, LoadMMO);
1040 
1041   DAG.setRoot(SpillLoad.getValue(1));
1042 
1043   assert(SpillLoad.getNode());
1044   setValue(&Relocate, SpillLoad);
1045 }
1046 
1047 void SelectionDAGBuilder::LowerDeoptimizeCall(const CallInst *CI) {
1048   const auto &TLI = DAG.getTargetLoweringInfo();
1049   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1050                                          TLI.getPointerTy(DAG.getDataLayout()));
1051 
1052   // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1053   // call.  We also do not lower the return value to any virtual register, and
1054   // change the immediately following return to a trap instruction.
1055   LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1056                                    /* VarArgDisallowed = */ true,
1057                                    /* ForceVoidReturnTy = */ true);
1058 }
1059 
1060 void SelectionDAGBuilder::LowerDeoptimizingReturn() {
1061   // We do not lower the return value from llvm.deoptimize to any virtual
1062   // register, and change the immediately following return to a trap
1063   // instruction.
1064   if (DAG.getTarget().Options.TrapUnreachable)
1065     DAG.setRoot(
1066         DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
1067 }
1068