xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp (revision a90b9d0159070121c221b966469c3e36d912bf82)
1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements some simple delegations needed for call lowering.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
15 #include "llvm/CodeGen/Analysis.h"
16 #include "llvm/CodeGen/CallingConvLower.h"
17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18 #include "llvm/CodeGen/GlobalISel/Utils.h"
19 #include "llvm/CodeGen/MachineFrameInfo.h"
20 #include "llvm/CodeGen/MachineOperand.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/TargetLowering.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/LLVMContext.h"
25 #include "llvm/IR/Module.h"
26 #include "llvm/Target/TargetMachine.h"
27 
28 #define DEBUG_TYPE "call-lowering"
29 
30 using namespace llvm;
31 
32 void CallLowering::anchor() {}
33 
34 /// Helper function which updates \p Flags when \p AttrFn returns true.
35 static void
36 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
37                     const std::function<bool(Attribute::AttrKind)> &AttrFn) {
38   if (AttrFn(Attribute::SExt))
39     Flags.setSExt();
40   if (AttrFn(Attribute::ZExt))
41     Flags.setZExt();
42   if (AttrFn(Attribute::InReg))
43     Flags.setInReg();
44   if (AttrFn(Attribute::StructRet))
45     Flags.setSRet();
46   if (AttrFn(Attribute::Nest))
47     Flags.setNest();
48   if (AttrFn(Attribute::ByVal))
49     Flags.setByVal();
50   if (AttrFn(Attribute::Preallocated))
51     Flags.setPreallocated();
52   if (AttrFn(Attribute::InAlloca))
53     Flags.setInAlloca();
54   if (AttrFn(Attribute::Returned))
55     Flags.setReturned();
56   if (AttrFn(Attribute::SwiftSelf))
57     Flags.setSwiftSelf();
58   if (AttrFn(Attribute::SwiftAsync))
59     Flags.setSwiftAsync();
60   if (AttrFn(Attribute::SwiftError))
61     Flags.setSwiftError();
62 }
63 
64 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
65                                                      unsigned ArgIdx) const {
66   ISD::ArgFlagsTy Flags;
67   addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) {
68     return Call.paramHasAttr(ArgIdx, Attr);
69   });
70   return Flags;
71 }
72 
73 ISD::ArgFlagsTy
74 CallLowering::getAttributesForReturn(const CallBase &Call) const {
75   ISD::ArgFlagsTy Flags;
76   addFlagsUsingAttrFn(Flags, [&Call](Attribute::AttrKind Attr) {
77     return Call.hasRetAttr(Attr);
78   });
79   return Flags;
80 }
81 
82 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
83                                              const AttributeList &Attrs,
84                                              unsigned OpIdx) const {
85   addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
86     return Attrs.hasAttributeAtIndex(OpIdx, Attr);
87   });
88 }
89 
90 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
91                              ArrayRef<Register> ResRegs,
92                              ArrayRef<ArrayRef<Register>> ArgRegs,
93                              Register SwiftErrorVReg,
94                              std::function<unsigned()> GetCalleeReg) const {
95   CallLoweringInfo Info;
96   const DataLayout &DL = MIRBuilder.getDataLayout();
97   MachineFunction &MF = MIRBuilder.getMF();
98   MachineRegisterInfo &MRI = MF.getRegInfo();
99   bool CanBeTailCalled = CB.isTailCall() &&
100                          isInTailCallPosition(CB, MF.getTarget()) &&
101                          (MF.getFunction()
102                               .getFnAttribute("disable-tail-calls")
103                               .getValueAsString() != "true");
104 
105   CallingConv::ID CallConv = CB.getCallingConv();
106   Type *RetTy = CB.getType();
107   bool IsVarArg = CB.getFunctionType()->isVarArg();
108 
109   SmallVector<BaseArgInfo, 4> SplitArgs;
110   getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL);
111   Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg);
112 
113   Info.IsConvergent = CB.isConvergent();
114 
115   if (!Info.CanLowerReturn) {
116     // Callee requires sret demotion.
117     insertSRetOutgoingArgument(MIRBuilder, CB, Info);
118 
119     // The sret demotion isn't compatible with tail-calls, since the sret
120     // argument points into the caller's stack frame.
121     CanBeTailCalled = false;
122   }
123 
124 
125   // First step is to marshall all the function's parameters into the correct
126   // physregs and memory locations. Gather the sequence of argument types that
127   // we'll pass to the assigner function.
128   unsigned i = 0;
129   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
130   for (const auto &Arg : CB.args()) {
131     ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i),
132                     i < NumFixedArgs};
133     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
134 
135     // If we have an explicit sret argument that is an Instruction, (i.e., it
136     // might point to function-local memory), we can't meaningfully tail-call.
137     if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg))
138       CanBeTailCalled = false;
139 
140     Info.OrigArgs.push_back(OrigArg);
141     ++i;
142   }
143 
144   // Try looking through a bitcast from one function type to another.
145   // Commonly happens with calls to objc_msgSend().
146   const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
147   if (const Function *F = dyn_cast<Function>(CalleeV))
148     Info.Callee = MachineOperand::CreateGA(F, 0);
149   else if (isa<GlobalIFunc>(CalleeV) || isa<GlobalAlias>(CalleeV)) {
150     // IR IFuncs and Aliases can't be forward declared (only defined), so the
151     // callee must be in the same TU and therefore we can direct-call it without
152     // worrying about it being out of range.
153     Info.Callee = MachineOperand::CreateGA(cast<GlobalValue>(CalleeV), 0);
154   } else
155     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
156 
157   Register ReturnHintAlignReg;
158   Align ReturnHintAlign;
159 
160   Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(CB)};
161 
162   if (!Info.OrigRet.Ty->isVoidTy()) {
163     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);
164 
165     if (MaybeAlign Alignment = CB.getRetAlign()) {
166       if (*Alignment > Align(1)) {
167         ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]);
168         Info.OrigRet.Regs[0] = ReturnHintAlignReg;
169         ReturnHintAlign = *Alignment;
170       }
171     }
172   }
173 
174   auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi);
175   if (Bundle && CB.isIndirectCall()) {
176     Info.CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
177     assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
178   }
179 
180   Info.CB = &CB;
181   Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
182   Info.CallConv = CallConv;
183   Info.SwiftErrorVReg = SwiftErrorVReg;
184   Info.IsMustTailCall = CB.isMustTailCall();
185   Info.IsTailCall = CanBeTailCalled;
186   Info.IsVarArg = IsVarArg;
187   if (!lowerCall(MIRBuilder, Info))
188     return false;
189 
190   if (ReturnHintAlignReg && !Info.IsTailCall) {
191     MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg,
192                                 ReturnHintAlign);
193   }
194 
195   return true;
196 }
197 
198 template <typename FuncInfoTy>
199 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
200                                const DataLayout &DL,
201                                const FuncInfoTy &FuncInfo) const {
202   auto &Flags = Arg.Flags[0];
203   const AttributeList &Attrs = FuncInfo.getAttributes();
204   addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
205 
206   PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType());
207   if (PtrTy) {
208     Flags.setPointer();
209     Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
210   }
211 
212   Align MemAlign = DL.getABITypeAlign(Arg.Ty);
213   if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
214     assert(OpIdx >= AttributeList::FirstArgIndex);
215     unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
216 
217     Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
218     if (!ElementTy)
219       ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
220     if (!ElementTy)
221       ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
222     assert(ElementTy && "Must have byval, inalloca or preallocated type");
223     Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
224 
225     // For ByVal, alignment should be passed from FE.  BE will guess if
226     // this info is not there but there are cases it cannot get right.
227     if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
228       MemAlign = *ParamAlign;
229     else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
230       MemAlign = *ParamAlign;
231     else
232       MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
233   } else if (OpIdx >= AttributeList::FirstArgIndex) {
234     if (auto ParamAlign =
235             FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
236       MemAlign = *ParamAlign;
237   }
238   Flags.setMemAlign(MemAlign);
239   Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
240 
241   // Don't try to use the returned attribute if the argument is marked as
242   // swiftself, since it won't be passed in x0.
243   if (Flags.isSwiftSelf())
244     Flags.setReturned(false);
245 }
246 
247 template void
248 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
249                                     const DataLayout &DL,
250                                     const Function &FuncInfo) const;
251 
252 template void
253 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
254                                     const DataLayout &DL,
255                                     const CallBase &FuncInfo) const;
256 
257 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
258                                      SmallVectorImpl<ArgInfo> &SplitArgs,
259                                      const DataLayout &DL,
260                                      CallingConv::ID CallConv,
261                                      SmallVectorImpl<uint64_t> *Offsets) const {
262   LLVMContext &Ctx = OrigArg.Ty->getContext();
263 
264   SmallVector<EVT, 4> SplitVTs;
265   ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, Offsets, 0);
266 
267   if (SplitVTs.size() == 0)
268     return;
269 
270   if (SplitVTs.size() == 1) {
271     // No splitting to do, but we want to replace the original type (e.g. [1 x
272     // double] -> double).
273     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
274                            OrigArg.OrigArgIndex, OrigArg.Flags[0],
275                            OrigArg.IsFixed, OrigArg.OrigValue);
276     return;
277   }
278 
279   // Create one ArgInfo for each virtual register in the original ArgInfo.
280   assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
281 
282   bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
283       OrigArg.Ty, CallConv, false, DL);
284   for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
285     Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
286     SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex,
287                            OrigArg.Flags[0], OrigArg.IsFixed);
288     if (NeedsRegBlock)
289       SplitArgs.back().Flags[0].setInConsecutiveRegs();
290   }
291 
292   SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
293 }
294 
295 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
296 static MachineInstrBuilder
297 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
298                             ArrayRef<Register> SrcRegs) {
299   MachineRegisterInfo &MRI = *B.getMRI();
300   LLT LLTy = MRI.getType(DstRegs[0]);
301   LLT PartLLT = MRI.getType(SrcRegs[0]);
302 
303   // Deal with v3s16 split into v2s16
304   LLT LCMTy = getCoverTy(LLTy, PartLLT);
305   if (LCMTy == LLTy) {
306     // Common case where no padding is needed.
307     assert(DstRegs.size() == 1);
308     return B.buildConcatVectors(DstRegs[0], SrcRegs);
309   }
310 
311   // We need to create an unmerge to the result registers, which may require
312   // widening the original value.
313   Register UnmergeSrcReg;
314   if (LCMTy != PartLLT) {
315     assert(DstRegs.size() == 1);
316     return B.buildDeleteTrailingVectorElements(
317         DstRegs[0], B.buildMergeLikeInstr(LCMTy, SrcRegs));
318   } else {
319     // We don't need to widen anything if we're extracting a scalar which was
320     // promoted to a vector e.g. s8 -> v4s8 -> s8
321     assert(SrcRegs.size() == 1);
322     UnmergeSrcReg = SrcRegs[0];
323   }
324 
325   int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
326 
327   SmallVector<Register, 8> PadDstRegs(NumDst);
328   std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin());
329 
330   // Create the excess dead defs for the unmerge.
331   for (int I = DstRegs.size(); I != NumDst; ++I)
332     PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);
333 
334   if (PadDstRegs.size() == 1)
335     return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg);
336   return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);
337 }
338 
339 /// Create a sequence of instructions to combine pieces split into register
340 /// typed values to the original IR value. \p OrigRegs contains the destination
341 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces
342 /// with type \p PartLLT. This is used for incoming values (physregs to vregs).
343 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
344                               ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
345                               const ISD::ArgFlagsTy Flags) {
346   MachineRegisterInfo &MRI = *B.getMRI();
347 
348   if (PartLLT == LLTy) {
349     // We should have avoided introducing a new virtual register, and just
350     // directly assigned here.
351     assert(OrigRegs[0] == Regs[0]);
352     return;
353   }
354 
355   if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
356       Regs.size() == 1) {
357     B.buildBitcast(OrigRegs[0], Regs[0]);
358     return;
359   }
360 
361   // A vector PartLLT needs extending to LLTy's element size.
362   // E.g. <2 x s64> = G_SEXT <2 x s32>.
363   if (PartLLT.isVector() == LLTy.isVector() &&
364       PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
365       (!PartLLT.isVector() ||
366        PartLLT.getElementCount() == LLTy.getElementCount()) &&
367       OrigRegs.size() == 1 && Regs.size() == 1) {
368     Register SrcReg = Regs[0];
369 
370     LLT LocTy = MRI.getType(SrcReg);
371 
372     if (Flags.isSExt()) {
373       SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
374                    .getReg(0);
375     } else if (Flags.isZExt()) {
376       SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
377                    .getReg(0);
378     }
379 
380     // Sometimes pointers are passed zero extended.
381     LLT OrigTy = MRI.getType(OrigRegs[0]);
382     if (OrigTy.isPointer()) {
383       LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits());
384       B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg));
385       return;
386     }
387 
388     B.buildTrunc(OrigRegs[0], SrcReg);
389     return;
390   }
391 
392   if (!LLTy.isVector() && !PartLLT.isVector()) {
393     assert(OrigRegs.size() == 1);
394     LLT OrigTy = MRI.getType(OrigRegs[0]);
395 
396     unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
397     if (SrcSize == OrigTy.getSizeInBits())
398       B.buildMergeValues(OrigRegs[0], Regs);
399     else {
400       auto Widened = B.buildMergeLikeInstr(LLT::scalar(SrcSize), Regs);
401       B.buildTrunc(OrigRegs[0], Widened);
402     }
403 
404     return;
405   }
406 
407   if (PartLLT.isVector()) {
408     assert(OrigRegs.size() == 1);
409     SmallVector<Register> CastRegs(Regs.begin(), Regs.end());
410 
411     // If PartLLT is a mismatched vector in both number of elements and element
412     // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
413     // have the same elt type, i.e. v4s32.
414     // TODO: Extend this coersion to element multiples other than just 2.
415     if (PartLLT.getSizeInBits() > LLTy.getSizeInBits() &&
416         PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
417         Regs.size() == 1) {
418       LLT NewTy = PartLLT.changeElementType(LLTy.getElementType())
419                       .changeElementCount(PartLLT.getElementCount() * 2);
420       CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0);
421       PartLLT = NewTy;
422     }
423 
424     if (LLTy.getScalarType() == PartLLT.getElementType()) {
425       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
426     } else {
427       unsigned I = 0;
428       LLT GCDTy = getGCDType(LLTy, PartLLT);
429 
430       // We are both splitting a vector, and bitcasting its element types. Cast
431       // the source pieces into the appropriate number of pieces with the result
432       // element type.
433       for (Register SrcReg : CastRegs)
434         CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0);
435       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
436     }
437 
438     return;
439   }
440 
441   assert(LLTy.isVector() && !PartLLT.isVector());
442 
443   LLT DstEltTy = LLTy.getElementType();
444 
445   // Pointer information was discarded. We'll need to coerce some register types
446   // to avoid violating type constraints.
447   LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType();
448 
449   assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
450 
451   if (DstEltTy == PartLLT) {
452     // Vector was trivially scalarized.
453 
454     if (RealDstEltTy.isPointer()) {
455       for (Register Reg : Regs)
456         MRI.setType(Reg, RealDstEltTy);
457     }
458 
459     B.buildBuildVector(OrigRegs[0], Regs);
460   } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
461     // Deal with vector with 64-bit elements decomposed to 32-bit
462     // registers. Need to create intermediate 64-bit elements.
463     SmallVector<Register, 8> EltMerges;
464     int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits();
465 
466     assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0);
467 
468     for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
469       auto Merge =
470           B.buildMergeLikeInstr(RealDstEltTy, Regs.take_front(PartsPerElt));
471       // Fix the type in case this is really a vector of pointers.
472       MRI.setType(Merge.getReg(0), RealDstEltTy);
473       EltMerges.push_back(Merge.getReg(0));
474       Regs = Regs.drop_front(PartsPerElt);
475     }
476 
477     B.buildBuildVector(OrigRegs[0], EltMerges);
478   } else {
479     // Vector was split, and elements promoted to a wider type.
480     // FIXME: Should handle floating point promotions.
481     unsigned NumElts = LLTy.getNumElements();
482     LLT BVType = LLT::fixed_vector(NumElts, PartLLT);
483 
484     Register BuildVec;
485     if (NumElts == Regs.size())
486       BuildVec = B.buildBuildVector(BVType, Regs).getReg(0);
487     else {
488       // Vector elements are packed in the inputs.
489       // e.g. we have a <4 x s16> but 2 x s32 in regs.
490       assert(NumElts > Regs.size());
491       LLT SrcEltTy = MRI.getType(Regs[0]);
492 
493       LLT OriginalEltTy = MRI.getType(OrigRegs[0]).getElementType();
494 
495       // Input registers contain packed elements.
496       // Determine how many elements per reg.
497       assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
498       unsigned EltPerReg =
499           (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
500 
501       SmallVector<Register, 0> BVRegs;
502       BVRegs.reserve(Regs.size() * EltPerReg);
503       for (Register R : Regs) {
504         auto Unmerge = B.buildUnmerge(OriginalEltTy, R);
505         for (unsigned K = 0; K < EltPerReg; ++K)
506           BVRegs.push_back(B.buildAnyExt(PartLLT, Unmerge.getReg(K)).getReg(0));
507       }
508 
509       // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
510       // for a <3 x s16> vector. We should have less than EltPerReg extra items.
511       if (BVRegs.size() > NumElts) {
512         assert((BVRegs.size() - NumElts) < EltPerReg);
513         BVRegs.truncate(NumElts);
514       }
515       BuildVec = B.buildBuildVector(BVType, BVRegs).getReg(0);
516     }
517     B.buildTrunc(OrigRegs[0], BuildVec);
518   }
519 }
520 
521 /// Create a sequence of instructions to expand the value in \p SrcReg (of type
522 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
523 /// contain the type of scalar value extension if necessary.
524 ///
525 /// This is used for outgoing values (vregs to physregs)
526 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
527                             Register SrcReg, LLT SrcTy, LLT PartTy,
528                             unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
529   // We could just insert a regular copy, but this is unreachable at the moment.
530   assert(SrcTy != PartTy && "identical part types shouldn't reach here");
531 
532   const unsigned PartSize = PartTy.getSizeInBits();
533 
534   if (PartTy.isVector() == SrcTy.isVector() &&
535       PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
536     assert(DstRegs.size() == 1);
537     B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});
538     return;
539   }
540 
541   if (SrcTy.isVector() && !PartTy.isVector() &&
542       PartSize > SrcTy.getElementType().getSizeInBits()) {
543     // Vector was scalarized, and the elements extended.
544     auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg);
545     for (int i = 0, e = DstRegs.size(); i != e; ++i)
546       B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));
547     return;
548   }
549 
550   if (SrcTy.isVector() && PartTy.isVector() &&
551       PartTy.getScalarSizeInBits() == SrcTy.getScalarSizeInBits() &&
552       SrcTy.getNumElements() < PartTy.getNumElements()) {
553     // A coercion like: v2f32 -> v4f32.
554     Register DstReg = DstRegs.front();
555     B.buildPadVectorWithUndefElements(DstReg, SrcReg);
556     return;
557   }
558 
559   LLT GCDTy = getGCDType(SrcTy, PartTy);
560   if (GCDTy == PartTy) {
561     // If this already evenly divisible, we can create a simple unmerge.
562     B.buildUnmerge(DstRegs, SrcReg);
563     return;
564   }
565 
566   MachineRegisterInfo &MRI = *B.getMRI();
567   LLT DstTy = MRI.getType(DstRegs[0]);
568   LLT LCMTy = getCoverTy(SrcTy, PartTy);
569 
570   if (PartTy.isVector() && LCMTy == PartTy) {
571     assert(DstRegs.size() == 1);
572     B.buildPadVectorWithUndefElements(DstRegs[0], SrcReg);
573     return;
574   }
575 
576   const unsigned DstSize = DstTy.getSizeInBits();
577   const unsigned SrcSize = SrcTy.getSizeInBits();
578   unsigned CoveringSize = LCMTy.getSizeInBits();
579 
580   Register UnmergeSrc = SrcReg;
581 
582   if (!LCMTy.isVector() && CoveringSize != SrcSize) {
583     // For scalars, it's common to be able to use a simple extension.
584     if (SrcTy.isScalar() && DstTy.isScalar()) {
585       CoveringSize = alignTo(SrcSize, DstSize);
586       LLT CoverTy = LLT::scalar(CoveringSize);
587       UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0);
588     } else {
589       // Widen to the common type.
590       // FIXME: This should respect the extend type
591       Register Undef = B.buildUndef(SrcTy).getReg(0);
592       SmallVector<Register, 8> MergeParts(1, SrcReg);
593       for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
594         MergeParts.push_back(Undef);
595       UnmergeSrc = B.buildMergeLikeInstr(LCMTy, MergeParts).getReg(0);
596     }
597   }
598 
599   if (LCMTy.isVector() && CoveringSize != SrcSize)
600     UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0);
601 
602   B.buildUnmerge(DstRegs, UnmergeSrc);
603 }
604 
605 bool CallLowering::determineAndHandleAssignments(
606     ValueHandler &Handler, ValueAssigner &Assigner,
607     SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
608     CallingConv::ID CallConv, bool IsVarArg,
609     ArrayRef<Register> ThisReturnRegs) const {
610   MachineFunction &MF = MIRBuilder.getMF();
611   const Function &F = MF.getFunction();
612   SmallVector<CCValAssign, 16> ArgLocs;
613 
614   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
615   if (!determineAssignments(Assigner, Args, CCInfo))
616     return false;
617 
618   return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder,
619                            ThisReturnRegs);
620 }
621 
622 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
623   if (Flags.isSExt())
624     return TargetOpcode::G_SEXT;
625   if (Flags.isZExt())
626     return TargetOpcode::G_ZEXT;
627   return TargetOpcode::G_ANYEXT;
628 }
629 
630 bool CallLowering::determineAssignments(ValueAssigner &Assigner,
631                                         SmallVectorImpl<ArgInfo> &Args,
632                                         CCState &CCInfo) const {
633   LLVMContext &Ctx = CCInfo.getContext();
634   const CallingConv::ID CallConv = CCInfo.getCallingConv();
635 
636   unsigned NumArgs = Args.size();
637   for (unsigned i = 0; i != NumArgs; ++i) {
638     EVT CurVT = EVT::getEVT(Args[i].Ty);
639 
640     MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT);
641 
642     // If we need to split the type over multiple regs, check it's a scenario
643     // we currently support.
644     unsigned NumParts =
645         TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT);
646 
647     if (NumParts == 1) {
648       // Try to use the register type if we couldn't assign the VT.
649       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
650                              Args[i].Flags[0], CCInfo))
651         return false;
652       continue;
653     }
654 
655     // For incoming arguments (physregs to vregs), we could have values in
656     // physregs (or memlocs) which we want to extract and copy to vregs.
657     // During this, we might have to deal with the LLT being split across
658     // multiple regs, so we have to record this information for later.
659     //
660     // If we have outgoing args, then we have the opposite case. We have a
661     // vreg with an LLT which we want to assign to a physical location, and
662     // we might have to record that the value has to be split later.
663 
664     // We're handling an incoming arg which is split over multiple regs.
665     // E.g. passing an s128 on AArch64.
666     ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
667     Args[i].Flags.clear();
668 
669     for (unsigned Part = 0; Part < NumParts; ++Part) {
670       ISD::ArgFlagsTy Flags = OrigFlags;
671       if (Part == 0) {
672         Flags.setSplit();
673       } else {
674         Flags.setOrigAlign(Align(1));
675         if (Part == NumParts - 1)
676           Flags.setSplitEnd();
677       }
678 
679       Args[i].Flags.push_back(Flags);
680       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
681                              Args[i].Flags[Part], CCInfo)) {
682         // Still couldn't assign this smaller part type for some reason.
683         return false;
684       }
685     }
686   }
687 
688   return true;
689 }
690 
691 bool CallLowering::handleAssignments(ValueHandler &Handler,
692                                      SmallVectorImpl<ArgInfo> &Args,
693                                      CCState &CCInfo,
694                                      SmallVectorImpl<CCValAssign> &ArgLocs,
695                                      MachineIRBuilder &MIRBuilder,
696                                      ArrayRef<Register> ThisReturnRegs) const {
697   MachineFunction &MF = MIRBuilder.getMF();
698   MachineRegisterInfo &MRI = MF.getRegInfo();
699   const Function &F = MF.getFunction();
700   const DataLayout &DL = F.getParent()->getDataLayout();
701 
702   const unsigned NumArgs = Args.size();
703 
704   // Stores thunks for outgoing register assignments. This is used so we delay
705   // generating register copies until mem loc assignments are done. We do this
706   // so that if the target is using the delayed stack protector feature, we can
707   // find the split point of the block accurately. E.g. if we have:
708   // G_STORE %val, %memloc
709   // $x0 = COPY %foo
710   // $x1 = COPY %bar
711   // CALL func
712   // ... then the split point for the block will correctly be at, and including,
713   // the copy to $x0. If instead the G_STORE instruction immediately precedes
714   // the CALL, then we'd prematurely choose the CALL as the split point, thus
715   // generating a split block with a CALL that uses undefined physregs.
716   SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
717 
718   for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
719     assert(j < ArgLocs.size() && "Skipped too many arg locs");
720     CCValAssign &VA = ArgLocs[j];
721     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
722 
723     if (VA.needsCustom()) {
724       std::function<void()> Thunk;
725       unsigned NumArgRegs = Handler.assignCustomValue(
726           Args[i], ArrayRef(ArgLocs).slice(j), &Thunk);
727       if (Thunk)
728         DelayedOutgoingRegAssignments.emplace_back(Thunk);
729       if (!NumArgRegs)
730         return false;
731       j += (NumArgRegs - 1);
732       continue;
733     }
734 
735     const MVT ValVT = VA.getValVT();
736     const MVT LocVT = VA.getLocVT();
737 
738     const LLT LocTy(LocVT);
739     const LLT ValTy(ValVT);
740     const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
741     const EVT OrigVT = EVT::getEVT(Args[i].Ty);
742     const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
743 
744     // Expected to be multiple regs for a single incoming arg.
745     // There should be Regs.size() ArgLocs per argument.
746     // This should be the same as getNumRegistersForCallingConv
747     const unsigned NumParts = Args[i].Flags.size();
748 
749     // Now split the registers into the assigned types.
750     Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end());
751 
752     if (NumParts != 1 || NewLLT != OrigTy) {
753       // If we can't directly assign the register, we need one or more
754       // intermediate values.
755       Args[i].Regs.resize(NumParts);
756 
757       // For each split register, create and assign a vreg that will store
758       // the incoming component of the larger value. These will later be
759       // merged to form the final vreg.
760       for (unsigned Part = 0; Part < NumParts; ++Part)
761         Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT);
762     }
763 
764     assert((j + (NumParts - 1)) < ArgLocs.size() &&
765            "Too many regs for number of args");
766 
767     // Coerce into outgoing value types before register assignment.
768     if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) {
769       assert(Args[i].OrigRegs.size() == 1);
770       buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy,
771                       ValTy, extendOpFromFlags(Args[i].Flags[0]));
772     }
773 
774     bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL);
775     for (unsigned Part = 0; Part < NumParts; ++Part) {
776       Register ArgReg = Args[i].Regs[Part];
777       // There should be Regs.size() ArgLocs per argument.
778       unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
779       CCValAssign &VA = ArgLocs[j + Idx];
780       const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
781 
782       if (VA.isMemLoc() && !Flags.isByVal()) {
783         // Individual pieces may have been spilled to the stack and others
784         // passed in registers.
785 
786         // TODO: The memory size may be larger than the value we need to
787         // store. We may need to adjust the offset for big endian targets.
788         LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
789 
790         MachinePointerInfo MPO;
791         Register StackAddr = Handler.getStackAddress(
792             MemTy.getSizeInBytes(), VA.getLocMemOffset(), MPO, Flags);
793 
794         Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO, VA);
795         continue;
796       }
797 
798       if (VA.isMemLoc() && Flags.isByVal()) {
799         assert(Args[i].Regs.size() == 1 &&
800                "didn't expect split byval pointer");
801 
802         if (Handler.isIncomingArgumentHandler()) {
803           // We just need to copy the frame index value to the pointer.
804           MachinePointerInfo MPO;
805           Register StackAddr = Handler.getStackAddress(
806               Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);
807           MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);
808         } else {
809           // For outgoing byval arguments, insert the implicit copy byval
810           // implies, such that writes in the callee do not modify the caller's
811           // value.
812           uint64_t MemSize = Flags.getByValSize();
813           int64_t Offset = VA.getLocMemOffset();
814 
815           MachinePointerInfo DstMPO;
816           Register StackAddr =
817               Handler.getStackAddress(MemSize, Offset, DstMPO, Flags);
818 
819           MachinePointerInfo SrcMPO(Args[i].OrigValue);
820           if (!Args[i].OrigValue) {
821             // We still need to accurately track the stack address space if we
822             // don't know the underlying value.
823             const LLT PtrTy = MRI.getType(StackAddr);
824             SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
825           }
826 
827           Align DstAlign = std::max(Flags.getNonZeroByValAlign(),
828                                     inferAlignFromPtrInfo(MF, DstMPO));
829 
830           Align SrcAlign = std::max(Flags.getNonZeroByValAlign(),
831                                     inferAlignFromPtrInfo(MF, SrcMPO));
832 
833           Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0],
834                                      DstMPO, DstAlign, SrcMPO, SrcAlign,
835                                      MemSize, VA);
836         }
837         continue;
838       }
839 
840       assert(!VA.needsCustom() && "custom loc should have been handled already");
841 
842       if (i == 0 && !ThisReturnRegs.empty() &&
843           Handler.isIncomingArgumentHandler() &&
844           isTypeIsValidForThisReturn(ValVT)) {
845         Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA);
846         continue;
847       }
848 
849       if (Handler.isIncomingArgumentHandler())
850         Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
851       else {
852         DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() {
853           Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
854         });
855       }
856     }
857 
858     // Now that all pieces have been assigned, re-pack the register typed values
859     // into the original value typed registers.
860     if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) {
861       // Merge the split registers into the expected larger result vregs of
862       // the original call.
863       buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy,
864                         LocTy, Args[i].Flags[0]);
865     }
866 
867     j += NumParts - 1;
868   }
869   for (auto &Fn : DelayedOutgoingRegAssignments)
870     Fn();
871 
872   return true;
873 }
874 
875 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
876                                    ArrayRef<Register> VRegs, Register DemoteReg,
877                                    int FI) const {
878   MachineFunction &MF = MIRBuilder.getMF();
879   MachineRegisterInfo &MRI = MF.getRegInfo();
880   const DataLayout &DL = MF.getDataLayout();
881 
882   SmallVector<EVT, 4> SplitVTs;
883   SmallVector<uint64_t, 4> Offsets;
884   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
885 
886   assert(VRegs.size() == SplitVTs.size());
887 
888   unsigned NumValues = SplitVTs.size();
889   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
890   Type *RetPtrTy =
891       PointerType::get(RetTy->getContext(), DL.getAllocaAddrSpace());
892   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetPtrTy), DL);
893 
894   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
895 
896   for (unsigned I = 0; I < NumValues; ++I) {
897     Register Addr;
898     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
899     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
900                                         MRI.getType(VRegs[I]),
901                                         commonAlignment(BaseAlign, Offsets[I]));
902     MIRBuilder.buildLoad(VRegs[I], Addr, *MMO);
903   }
904 }
905 
906 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
907                                     ArrayRef<Register> VRegs,
908                                     Register DemoteReg) const {
909   MachineFunction &MF = MIRBuilder.getMF();
910   MachineRegisterInfo &MRI = MF.getRegInfo();
911   const DataLayout &DL = MF.getDataLayout();
912 
913   SmallVector<EVT, 4> SplitVTs;
914   SmallVector<uint64_t, 4> Offsets;
915   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
916 
917   assert(VRegs.size() == SplitVTs.size());
918 
919   unsigned NumValues = SplitVTs.size();
920   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
921   unsigned AS = DL.getAllocaAddrSpace();
922   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getPointerTo(AS)), DL);
923 
924   MachinePointerInfo PtrInfo(AS);
925 
926   for (unsigned I = 0; I < NumValues; ++I) {
927     Register Addr;
928     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
929     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
930                                         MRI.getType(VRegs[I]),
931                                         commonAlignment(BaseAlign, Offsets[I]));
932     MIRBuilder.buildStore(VRegs[I], Addr, *MMO);
933   }
934 }
935 
936 void CallLowering::insertSRetIncomingArgument(
937     const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
938     MachineRegisterInfo &MRI, const DataLayout &DL) const {
939   unsigned AS = DL.getAllocaAddrSpace();
940   DemoteReg = MRI.createGenericVirtualRegister(
941       LLT::pointer(AS, DL.getPointerSizeInBits(AS)));
942 
943   Type *PtrTy = PointerType::get(F.getReturnType(), AS);
944 
945   SmallVector<EVT, 1> ValueVTs;
946   ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs);
947 
948   // NOTE: Assume that a pointer won't get split into more than one VT.
949   assert(ValueVTs.size() == 1);
950 
951   ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()),
952                     ArgInfo::NoArgIndex);
953   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F);
954   DemoteArg.Flags[0].setSRet();
955   SplitArgs.insert(SplitArgs.begin(), DemoteArg);
956 }
957 
958 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
959                                               const CallBase &CB,
960                                               CallLoweringInfo &Info) const {
961   const DataLayout &DL = MIRBuilder.getDataLayout();
962   Type *RetTy = CB.getType();
963   unsigned AS = DL.getAllocaAddrSpace();
964   LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
965 
966   int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
967       DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false);
968 
969   Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0);
970   ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS),
971                     ArgInfo::NoArgIndex);
972   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB);
973   DemoteArg.Flags[0].setSRet();
974 
975   Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg);
976   Info.DemoteStackIndex = FI;
977   Info.DemoteRegister = DemoteReg;
978 }
979 
980 bool CallLowering::checkReturn(CCState &CCInfo,
981                                SmallVectorImpl<BaseArgInfo> &Outs,
982                                CCAssignFn *Fn) const {
983   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
984     MVT VT = MVT::getVT(Outs[I].Ty);
985     if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
986       return false;
987   }
988   return true;
989 }
990 
991 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
992                                  AttributeList Attrs,
993                                  SmallVectorImpl<BaseArgInfo> &Outs,
994                                  const DataLayout &DL) const {
995   LLVMContext &Context = RetTy->getContext();
996   ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
997 
998   SmallVector<EVT, 4> SplitVTs;
999   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs);
1000   addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex);
1001 
1002   for (EVT VT : SplitVTs) {
1003     unsigned NumParts =
1004         TLI->getNumRegistersForCallingConv(Context, CallConv, VT);
1005     MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT);
1006     Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1007 
1008     for (unsigned I = 0; I < NumParts; ++I) {
1009       Outs.emplace_back(PartTy, Flags);
1010     }
1011   }
1012 }
1013 
1014 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1015   const auto &F = MF.getFunction();
1016   Type *ReturnType = F.getReturnType();
1017   CallingConv::ID CallConv = F.getCallingConv();
1018 
1019   SmallVector<BaseArgInfo, 4> SplitArgs;
1020   getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs,
1021                 MF.getDataLayout());
1022   return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg());
1023 }
1024 
1025 bool CallLowering::parametersInCSRMatch(
1026     const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1027     const SmallVectorImpl<CCValAssign> &OutLocs,
1028     const SmallVectorImpl<ArgInfo> &OutArgs) const {
1029   for (unsigned i = 0; i < OutLocs.size(); ++i) {
1030     const auto &ArgLoc = OutLocs[i];
1031     // If it's not a register, it's fine.
1032     if (!ArgLoc.isRegLoc())
1033       continue;
1034 
1035     MCRegister PhysReg = ArgLoc.getLocReg();
1036 
1037     // Only look at callee-saved registers.
1038     if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg))
1039       continue;
1040 
1041     LLVM_DEBUG(
1042         dbgs()
1043         << "... Call has an argument passed in a callee-saved register.\n");
1044 
1045     // Check if it was copied from.
1046     const ArgInfo &OutInfo = OutArgs[i];
1047 
1048     if (OutInfo.Regs.size() > 1) {
1049       LLVM_DEBUG(
1050           dbgs() << "... Cannot handle arguments in multiple registers.\n");
1051       return false;
1052     }
1053 
1054     // Check if we copy the register, walking through copies from virtual
1055     // registers. Note that getDefIgnoringCopies does not ignore copies from
1056     // physical registers.
1057     MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);
1058     if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1059       LLVM_DEBUG(
1060           dbgs()
1061           << "... Parameter was not copied into a VReg, cannot tail call.\n");
1062       return false;
1063     }
1064 
1065     // Got a copy. Verify that it's the same as the register we want.
1066     Register CopyRHS = RegDef->getOperand(1).getReg();
1067     if (CopyRHS != PhysReg) {
1068       LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1069                            "VReg, cannot tail call.\n");
1070       return false;
1071     }
1072   }
1073 
1074   return true;
1075 }
1076 
1077 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1078                                      MachineFunction &MF,
1079                                      SmallVectorImpl<ArgInfo> &InArgs,
1080                                      ValueAssigner &CalleeAssigner,
1081                                      ValueAssigner &CallerAssigner) const {
1082   const Function &F = MF.getFunction();
1083   CallingConv::ID CalleeCC = Info.CallConv;
1084   CallingConv::ID CallerCC = F.getCallingConv();
1085 
1086   if (CallerCC == CalleeCC)
1087     return true;
1088 
1089   SmallVector<CCValAssign, 16> ArgLocs1;
1090   CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1091   if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1))
1092     return false;
1093 
1094   SmallVector<CCValAssign, 16> ArgLocs2;
1095   CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1096   if (!determineAssignments(CallerAssigner, InArgs, CCInfo2))
1097     return false;
1098 
1099   // We need the argument locations to match up exactly. If there's more in
1100   // one than the other, then we are done.
1101   if (ArgLocs1.size() != ArgLocs2.size())
1102     return false;
1103 
1104   // Make sure that each location is passed in exactly the same way.
1105   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1106     const CCValAssign &Loc1 = ArgLocs1[i];
1107     const CCValAssign &Loc2 = ArgLocs2[i];
1108 
1109     // We need both of them to be the same. So if one is a register and one
1110     // isn't, we're done.
1111     if (Loc1.isRegLoc() != Loc2.isRegLoc())
1112       return false;
1113 
1114     if (Loc1.isRegLoc()) {
1115       // If they don't have the same register location, we're done.
1116       if (Loc1.getLocReg() != Loc2.getLocReg())
1117         return false;
1118 
1119       // They matched, so we can move to the next ArgLoc.
1120       continue;
1121     }
1122 
1123     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1124     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1125       return false;
1126   }
1127 
1128   return true;
1129 }
1130 
1131 LLT CallLowering::ValueHandler::getStackValueStoreType(
1132     const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1133   const MVT ValVT = VA.getValVT();
1134   if (ValVT != MVT::iPTR) {
1135     LLT ValTy(ValVT);
1136 
1137     // We lost the pointeriness going through CCValAssign, so try to restore it
1138     // based on the flags.
1139     if (Flags.isPointer()) {
1140       LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(),
1141                                ValTy.getScalarSizeInBits());
1142       if (ValVT.isVector())
1143         return LLT::vector(ValTy.getElementCount(), PtrTy);
1144       return PtrTy;
1145     }
1146 
1147     return ValTy;
1148   }
1149 
1150   unsigned AddrSpace = Flags.getPointerAddrSpace();
1151   return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace));
1152 }
1153 
1154 void CallLowering::ValueHandler::copyArgumentMemory(
1155     const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1156     const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1157     const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1158     CCValAssign &VA) const {
1159   MachineFunction &MF = MIRBuilder.getMF();
1160   MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1161       SrcPtrInfo,
1162       MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize,
1163       SrcAlign);
1164 
1165   MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1166       DstPtrInfo,
1167       MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1168       MemSize, DstAlign);
1169 
1170   const LLT PtrTy = MRI.getType(DstPtr);
1171   const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits());
1172 
1173   auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize);
1174   MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO);
1175 }
1176 
1177 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1178                                                     const CCValAssign &VA,
1179                                                     unsigned MaxSizeBits) {
1180   LLT LocTy{VA.getLocVT()};
1181   LLT ValTy{VA.getValVT()};
1182 
1183   if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1184     return ValReg;
1185 
1186   if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1187     if (MaxSizeBits <= ValTy.getSizeInBits())
1188       return ValReg;
1189     LocTy = LLT::scalar(MaxSizeBits);
1190   }
1191 
1192   const LLT ValRegTy = MRI.getType(ValReg);
1193   if (ValRegTy.isPointer()) {
1194     // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1195     // we have to cast to do the extension.
1196     LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits());
1197     ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0);
1198   }
1199 
1200   switch (VA.getLocInfo()) {
1201   default: break;
1202   case CCValAssign::Full:
1203   case CCValAssign::BCvt:
1204     // FIXME: bitconverting between vector types may or may not be a
1205     // nop in big-endian situations.
1206     return ValReg;
1207   case CCValAssign::AExt: {
1208     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
1209     return MIB.getReg(0);
1210   }
1211   case CCValAssign::SExt: {
1212     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1213     MIRBuilder.buildSExt(NewReg, ValReg);
1214     return NewReg;
1215   }
1216   case CCValAssign::ZExt: {
1217     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1218     MIRBuilder.buildZExt(NewReg, ValReg);
1219     return NewReg;
1220   }
1221   }
1222   llvm_unreachable("unable to extend register");
1223 }
1224 
1225 void CallLowering::ValueAssigner::anchor() {}
1226 
1227 Register CallLowering::IncomingValueHandler::buildExtensionHint(
1228     const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1229   switch (VA.getLocInfo()) {
1230   case CCValAssign::LocInfo::ZExt: {
1231     return MIRBuilder
1232         .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1233                          NarrowTy.getScalarSizeInBits())
1234         .getReg(0);
1235   }
1236   case CCValAssign::LocInfo::SExt: {
1237     return MIRBuilder
1238         .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1239                          NarrowTy.getScalarSizeInBits())
1240         .getReg(0);
1241     break;
1242   }
1243   default:
1244     return SrcReg;
1245   }
1246 }
1247 
1248 /// Check if we can use a basic COPY instruction between the two types.
1249 ///
1250 /// We're currently building on top of the infrastructure using MVT, which loses
1251 /// pointer information in the CCValAssign. We accept copies from physical
1252 /// registers that have been reported as integers if it's to an equivalent sized
1253 /// pointer LLT.
1254 static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1255   if (SrcTy == DstTy)
1256     return true;
1257 
1258   if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1259     return false;
1260 
1261   SrcTy = SrcTy.getScalarType();
1262   DstTy = DstTy.getScalarType();
1263 
1264   return (SrcTy.isPointer() && DstTy.isScalar()) ||
1265          (DstTy.isPointer() && SrcTy.isScalar());
1266 }
1267 
1268 void CallLowering::IncomingValueHandler::assignValueToReg(
1269     Register ValVReg, Register PhysReg, const CCValAssign &VA) {
1270   const MVT LocVT = VA.getLocVT();
1271   const LLT LocTy(LocVT);
1272   const LLT RegTy = MRI.getType(ValVReg);
1273 
1274   if (isCopyCompatibleType(RegTy, LocTy)) {
1275     MIRBuilder.buildCopy(ValVReg, PhysReg);
1276     return;
1277   }
1278 
1279   auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg);
1280   auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy);
1281   MIRBuilder.buildTrunc(ValVReg, Hint);
1282 }
1283