xref: /freebsd/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp (revision b1879975794772ee51f0b4865753364c7d7626c3)
1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements some simple delegations needed for call lowering.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
15 #include "llvm/CodeGen/Analysis.h"
16 #include "llvm/CodeGen/CallingConvLower.h"
17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18 #include "llvm/CodeGen/GlobalISel/Utils.h"
19 #include "llvm/CodeGen/MachineFrameInfo.h"
20 #include "llvm/CodeGen/MachineOperand.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/TargetLowering.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/Target/TargetMachine.h"
28 
29 #define DEBUG_TYPE "call-lowering"
30 
31 using namespace llvm;
32 
33 void CallLowering::anchor() {}
34 
35 /// Helper function which updates \p Flags when \p AttrFn returns true.
36 static void
37 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
38                     const std::function<bool(Attribute::AttrKind)> &AttrFn) {
39   // TODO: There are missing flags. Add them here.
40   if (AttrFn(Attribute::SExt))
41     Flags.setSExt();
42   if (AttrFn(Attribute::ZExt))
43     Flags.setZExt();
44   if (AttrFn(Attribute::InReg))
45     Flags.setInReg();
46   if (AttrFn(Attribute::StructRet))
47     Flags.setSRet();
48   if (AttrFn(Attribute::Nest))
49     Flags.setNest();
50   if (AttrFn(Attribute::ByVal))
51     Flags.setByVal();
52   if (AttrFn(Attribute::ByRef))
53     Flags.setByRef();
54   if (AttrFn(Attribute::Preallocated))
55     Flags.setPreallocated();
56   if (AttrFn(Attribute::InAlloca))
57     Flags.setInAlloca();
58   if (AttrFn(Attribute::Returned))
59     Flags.setReturned();
60   if (AttrFn(Attribute::SwiftSelf))
61     Flags.setSwiftSelf();
62   if (AttrFn(Attribute::SwiftAsync))
63     Flags.setSwiftAsync();
64   if (AttrFn(Attribute::SwiftError))
65     Flags.setSwiftError();
66 }
67 
68 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
69                                                      unsigned ArgIdx) const {
70   ISD::ArgFlagsTy Flags;
71   addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) {
72     return Call.paramHasAttr(ArgIdx, Attr);
73   });
74   return Flags;
75 }
76 
77 ISD::ArgFlagsTy
78 CallLowering::getAttributesForReturn(const CallBase &Call) const {
79   ISD::ArgFlagsTy Flags;
80   addFlagsUsingAttrFn(Flags, [&Call](Attribute::AttrKind Attr) {
81     return Call.hasRetAttr(Attr);
82   });
83   return Flags;
84 }
85 
86 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
87                                              const AttributeList &Attrs,
88                                              unsigned OpIdx) const {
89   addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
90     return Attrs.hasAttributeAtIndex(OpIdx, Attr);
91   });
92 }
93 
94 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
95                              ArrayRef<Register> ResRegs,
96                              ArrayRef<ArrayRef<Register>> ArgRegs,
97                              Register SwiftErrorVReg,
98                              std::optional<PtrAuthInfo> PAI,
99                              Register ConvergenceCtrlToken,
100                              std::function<unsigned()> GetCalleeReg) const {
101   CallLoweringInfo Info;
102   const DataLayout &DL = MIRBuilder.getDataLayout();
103   MachineFunction &MF = MIRBuilder.getMF();
104   MachineRegisterInfo &MRI = MF.getRegInfo();
105   bool CanBeTailCalled = CB.isTailCall() &&
106                          isInTailCallPosition(CB, MF.getTarget()) &&
107                          (MF.getFunction()
108                               .getFnAttribute("disable-tail-calls")
109                               .getValueAsString() != "true");
110 
111   CallingConv::ID CallConv = CB.getCallingConv();
112   Type *RetTy = CB.getType();
113   bool IsVarArg = CB.getFunctionType()->isVarArg();
114 
115   SmallVector<BaseArgInfo, 4> SplitArgs;
116   getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL);
117   Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg);
118 
119   Info.IsConvergent = CB.isConvergent();
120 
121   if (!Info.CanLowerReturn) {
122     // Callee requires sret demotion.
123     insertSRetOutgoingArgument(MIRBuilder, CB, Info);
124 
125     // The sret demotion isn't compatible with tail-calls, since the sret
126     // argument points into the caller's stack frame.
127     CanBeTailCalled = false;
128   }
129 
130   // First step is to marshall all the function's parameters into the correct
131   // physregs and memory locations. Gather the sequence of argument types that
132   // we'll pass to the assigner function.
133   unsigned i = 0;
134   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
135   for (const auto &Arg : CB.args()) {
136     ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i),
137                     i < NumFixedArgs};
138     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
139 
140     // If we have an explicit sret argument that is an Instruction, (i.e., it
141     // might point to function-local memory), we can't meaningfully tail-call.
142     if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg))
143       CanBeTailCalled = false;
144 
145     Info.OrigArgs.push_back(OrigArg);
146     ++i;
147   }
148 
149   // Try looking through a bitcast from one function type to another.
150   // Commonly happens with calls to objc_msgSend().
151   const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
152 
153   // If IRTranslator chose to drop the ptrauth info, we can turn this into
154   // a direct call.
155   if (!PAI && CB.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
156     CalleeV = cast<ConstantPtrAuth>(CalleeV)->getPointer();
157     assert(isa<Function>(CalleeV));
158   }
159 
160   if (const Function *F = dyn_cast<Function>(CalleeV)) {
161     if (F->hasFnAttribute(Attribute::NonLazyBind)) {
162       LLT Ty = getLLTForType(*F->getType(), DL);
163       Register Reg = MIRBuilder.buildGlobalValue(Ty, F).getReg(0);
164       Info.Callee = MachineOperand::CreateReg(Reg, false);
165     } else {
166       Info.Callee = MachineOperand::CreateGA(F, 0);
167     }
168   } else if (isa<GlobalIFunc>(CalleeV) || isa<GlobalAlias>(CalleeV)) {
169     // IR IFuncs and Aliases can't be forward declared (only defined), so the
170     // callee must be in the same TU and therefore we can direct-call it without
171     // worrying about it being out of range.
172     Info.Callee = MachineOperand::CreateGA(cast<GlobalValue>(CalleeV), 0);
173   } else
174     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
175 
176   Register ReturnHintAlignReg;
177   Align ReturnHintAlign;
178 
179   Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(CB)};
180 
181   if (!Info.OrigRet.Ty->isVoidTy()) {
182     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);
183 
184     if (MaybeAlign Alignment = CB.getRetAlign()) {
185       if (*Alignment > Align(1)) {
186         ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]);
187         Info.OrigRet.Regs[0] = ReturnHintAlignReg;
188         ReturnHintAlign = *Alignment;
189       }
190     }
191   }
192 
193   auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi);
194   if (Bundle && CB.isIndirectCall()) {
195     Info.CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
196     assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
197   }
198 
199   Info.CB = &CB;
200   Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
201   Info.CallConv = CallConv;
202   Info.SwiftErrorVReg = SwiftErrorVReg;
203   Info.PAI = PAI;
204   Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
205   Info.IsMustTailCall = CB.isMustTailCall();
206   Info.IsTailCall = CanBeTailCalled;
207   Info.IsVarArg = IsVarArg;
208   if (!lowerCall(MIRBuilder, Info))
209     return false;
210 
211   if (ReturnHintAlignReg && !Info.LoweredTailCall) {
212     MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg,
213                                 ReturnHintAlign);
214   }
215 
216   return true;
217 }
218 
219 template <typename FuncInfoTy>
220 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
221                                const DataLayout &DL,
222                                const FuncInfoTy &FuncInfo) const {
223   auto &Flags = Arg.Flags[0];
224   const AttributeList &Attrs = FuncInfo.getAttributes();
225   addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
226 
227   PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType());
228   if (PtrTy) {
229     Flags.setPointer();
230     Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
231   }
232 
233   Align MemAlign = DL.getABITypeAlign(Arg.Ty);
234   if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
235       Flags.isByRef()) {
236     assert(OpIdx >= AttributeList::FirstArgIndex);
237     unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
238 
239     Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
240     if (!ElementTy)
241       ElementTy = FuncInfo.getParamByRefType(ParamIdx);
242     if (!ElementTy)
243       ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
244     if (!ElementTy)
245       ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
246 
247     assert(ElementTy && "Must have byval, inalloca or preallocated type");
248 
249     uint64_t MemSize = DL.getTypeAllocSize(ElementTy);
250     if (Flags.isByRef())
251       Flags.setByRefSize(MemSize);
252     else
253       Flags.setByValSize(MemSize);
254 
255     // For ByVal, alignment should be passed from FE.  BE will guess if
256     // this info is not there but there are cases it cannot get right.
257     if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
258       MemAlign = *ParamAlign;
259     else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
260       MemAlign = *ParamAlign;
261     else
262       MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
263   } else if (OpIdx >= AttributeList::FirstArgIndex) {
264     if (auto ParamAlign =
265             FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
266       MemAlign = *ParamAlign;
267   }
268   Flags.setMemAlign(MemAlign);
269   Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
270 
271   // Don't try to use the returned attribute if the argument is marked as
272   // swiftself, since it won't be passed in x0.
273   if (Flags.isSwiftSelf())
274     Flags.setReturned(false);
275 }
276 
277 template void
278 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
279                                     const DataLayout &DL,
280                                     const Function &FuncInfo) const;
281 
282 template void
283 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
284                                     const DataLayout &DL,
285                                     const CallBase &FuncInfo) const;
286 
287 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
288                                      SmallVectorImpl<ArgInfo> &SplitArgs,
289                                      const DataLayout &DL,
290                                      CallingConv::ID CallConv,
291                                      SmallVectorImpl<uint64_t> *Offsets) const {
292   LLVMContext &Ctx = OrigArg.Ty->getContext();
293 
294   SmallVector<EVT, 4> SplitVTs;
295   ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, Offsets, 0);
296 
297   if (SplitVTs.size() == 0)
298     return;
299 
300   if (SplitVTs.size() == 1) {
301     // No splitting to do, but we want to replace the original type (e.g. [1 x
302     // double] -> double).
303     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
304                            OrigArg.OrigArgIndex, OrigArg.Flags[0],
305                            OrigArg.IsFixed, OrigArg.OrigValue);
306     return;
307   }
308 
309   // Create one ArgInfo for each virtual register in the original ArgInfo.
310   assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
311 
312   bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
313       OrigArg.Ty, CallConv, false, DL);
314   for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
315     Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
316     SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex,
317                            OrigArg.Flags[0], OrigArg.IsFixed);
318     if (NeedsRegBlock)
319       SplitArgs.back().Flags[0].setInConsecutiveRegs();
320   }
321 
322   SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
323 }
324 
325 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
326 static MachineInstrBuilder
327 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
328                             ArrayRef<Register> SrcRegs) {
329   MachineRegisterInfo &MRI = *B.getMRI();
330   LLT LLTy = MRI.getType(DstRegs[0]);
331   LLT PartLLT = MRI.getType(SrcRegs[0]);
332 
333   // Deal with v3s16 split into v2s16
334   LLT LCMTy = getCoverTy(LLTy, PartLLT);
335   if (LCMTy == LLTy) {
336     // Common case where no padding is needed.
337     assert(DstRegs.size() == 1);
338     return B.buildConcatVectors(DstRegs[0], SrcRegs);
339   }
340 
341   // We need to create an unmerge to the result registers, which may require
342   // widening the original value.
343   Register UnmergeSrcReg;
344   if (LCMTy != PartLLT) {
345     assert(DstRegs.size() == 1);
346     return B.buildDeleteTrailingVectorElements(
347         DstRegs[0], B.buildMergeLikeInstr(LCMTy, SrcRegs));
348   } else {
349     // We don't need to widen anything if we're extracting a scalar which was
350     // promoted to a vector e.g. s8 -> v4s8 -> s8
351     assert(SrcRegs.size() == 1);
352     UnmergeSrcReg = SrcRegs[0];
353   }
354 
355   int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
356 
357   SmallVector<Register, 8> PadDstRegs(NumDst);
358   std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin());
359 
360   // Create the excess dead defs for the unmerge.
361   for (int I = DstRegs.size(); I != NumDst; ++I)
362     PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);
363 
364   if (PadDstRegs.size() == 1)
365     return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg);
366   return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);
367 }
368 
369 /// Create a sequence of instructions to combine pieces split into register
370 /// typed values to the original IR value. \p OrigRegs contains the destination
371 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces
372 /// with type \p PartLLT. This is used for incoming values (physregs to vregs).
373 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
374                               ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
375                               const ISD::ArgFlagsTy Flags) {
376   MachineRegisterInfo &MRI = *B.getMRI();
377 
378   if (PartLLT == LLTy) {
379     // We should have avoided introducing a new virtual register, and just
380     // directly assigned here.
381     assert(OrigRegs[0] == Regs[0]);
382     return;
383   }
384 
385   if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
386       Regs.size() == 1) {
387     B.buildBitcast(OrigRegs[0], Regs[0]);
388     return;
389   }
390 
391   // A vector PartLLT needs extending to LLTy's element size.
392   // E.g. <2 x s64> = G_SEXT <2 x s32>.
393   if (PartLLT.isVector() == LLTy.isVector() &&
394       PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
395       (!PartLLT.isVector() ||
396        PartLLT.getElementCount() == LLTy.getElementCount()) &&
397       OrigRegs.size() == 1 && Regs.size() == 1) {
398     Register SrcReg = Regs[0];
399 
400     LLT LocTy = MRI.getType(SrcReg);
401 
402     if (Flags.isSExt()) {
403       SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
404                    .getReg(0);
405     } else if (Flags.isZExt()) {
406       SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
407                    .getReg(0);
408     }
409 
410     // Sometimes pointers are passed zero extended.
411     LLT OrigTy = MRI.getType(OrigRegs[0]);
412     if (OrigTy.isPointer()) {
413       LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits());
414       B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg));
415       return;
416     }
417 
418     B.buildTrunc(OrigRegs[0], SrcReg);
419     return;
420   }
421 
422   if (!LLTy.isVector() && !PartLLT.isVector()) {
423     assert(OrigRegs.size() == 1);
424     LLT OrigTy = MRI.getType(OrigRegs[0]);
425 
426     unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
427     if (SrcSize == OrigTy.getSizeInBits())
428       B.buildMergeValues(OrigRegs[0], Regs);
429     else {
430       auto Widened = B.buildMergeLikeInstr(LLT::scalar(SrcSize), Regs);
431       B.buildTrunc(OrigRegs[0], Widened);
432     }
433 
434     return;
435   }
436 
437   if (PartLLT.isVector()) {
438     assert(OrigRegs.size() == 1);
439     SmallVector<Register> CastRegs(Regs.begin(), Regs.end());
440 
441     // If PartLLT is a mismatched vector in both number of elements and element
442     // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
443     // have the same elt type, i.e. v4s32.
444     // TODO: Extend this coersion to element multiples other than just 2.
445     if (TypeSize::isKnownGT(PartLLT.getSizeInBits(), LLTy.getSizeInBits()) &&
446         PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
447         Regs.size() == 1) {
448       LLT NewTy = PartLLT.changeElementType(LLTy.getElementType())
449                       .changeElementCount(PartLLT.getElementCount() * 2);
450       CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0);
451       PartLLT = NewTy;
452     }
453 
454     if (LLTy.getScalarType() == PartLLT.getElementType()) {
455       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
456     } else {
457       unsigned I = 0;
458       LLT GCDTy = getGCDType(LLTy, PartLLT);
459 
460       // We are both splitting a vector, and bitcasting its element types. Cast
461       // the source pieces into the appropriate number of pieces with the result
462       // element type.
463       for (Register SrcReg : CastRegs)
464         CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0);
465       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
466     }
467 
468     return;
469   }
470 
471   assert(LLTy.isVector() && !PartLLT.isVector());
472 
473   LLT DstEltTy = LLTy.getElementType();
474 
475   // Pointer information was discarded. We'll need to coerce some register types
476   // to avoid violating type constraints.
477   LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType();
478 
479   assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
480 
481   if (DstEltTy == PartLLT) {
482     // Vector was trivially scalarized.
483 
484     if (RealDstEltTy.isPointer()) {
485       for (Register Reg : Regs)
486         MRI.setType(Reg, RealDstEltTy);
487     }
488 
489     B.buildBuildVector(OrigRegs[0], Regs);
490   } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
491     // Deal with vector with 64-bit elements decomposed to 32-bit
492     // registers. Need to create intermediate 64-bit elements.
493     SmallVector<Register, 8> EltMerges;
494     int PartsPerElt =
495         divideCeil(DstEltTy.getSizeInBits(), PartLLT.getSizeInBits());
496     LLT ExtendedPartTy = LLT::scalar(PartLLT.getSizeInBits() * PartsPerElt);
497 
498     for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
499       auto Merge =
500           B.buildMergeLikeInstr(ExtendedPartTy, Regs.take_front(PartsPerElt));
501       if (ExtendedPartTy.getSizeInBits() > RealDstEltTy.getSizeInBits())
502         Merge = B.buildTrunc(RealDstEltTy, Merge);
503       // Fix the type in case this is really a vector of pointers.
504       MRI.setType(Merge.getReg(0), RealDstEltTy);
505       EltMerges.push_back(Merge.getReg(0));
506       Regs = Regs.drop_front(PartsPerElt);
507     }
508 
509     B.buildBuildVector(OrigRegs[0], EltMerges);
510   } else {
511     // Vector was split, and elements promoted to a wider type.
512     // FIXME: Should handle floating point promotions.
513     unsigned NumElts = LLTy.getNumElements();
514     LLT BVType = LLT::fixed_vector(NumElts, PartLLT);
515 
516     Register BuildVec;
517     if (NumElts == Regs.size())
518       BuildVec = B.buildBuildVector(BVType, Regs).getReg(0);
519     else {
520       // Vector elements are packed in the inputs.
521       // e.g. we have a <4 x s16> but 2 x s32 in regs.
522       assert(NumElts > Regs.size());
523       LLT SrcEltTy = MRI.getType(Regs[0]);
524 
525       LLT OriginalEltTy = MRI.getType(OrigRegs[0]).getElementType();
526 
527       // Input registers contain packed elements.
528       // Determine how many elements per reg.
529       assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
530       unsigned EltPerReg =
531           (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
532 
533       SmallVector<Register, 0> BVRegs;
534       BVRegs.reserve(Regs.size() * EltPerReg);
535       for (Register R : Regs) {
536         auto Unmerge = B.buildUnmerge(OriginalEltTy, R);
537         for (unsigned K = 0; K < EltPerReg; ++K)
538           BVRegs.push_back(B.buildAnyExt(PartLLT, Unmerge.getReg(K)).getReg(0));
539       }
540 
541       // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
542       // for a <3 x s16> vector. We should have less than EltPerReg extra items.
543       if (BVRegs.size() > NumElts) {
544         assert((BVRegs.size() - NumElts) < EltPerReg);
545         BVRegs.truncate(NumElts);
546       }
547       BuildVec = B.buildBuildVector(BVType, BVRegs).getReg(0);
548     }
549     B.buildTrunc(OrigRegs[0], BuildVec);
550   }
551 }
552 
553 /// Create a sequence of instructions to expand the value in \p SrcReg (of type
554 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
555 /// contain the type of scalar value extension if necessary.
556 ///
557 /// This is used for outgoing values (vregs to physregs)
558 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
559                             Register SrcReg, LLT SrcTy, LLT PartTy,
560                             unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
561   // We could just insert a regular copy, but this is unreachable at the moment.
562   assert(SrcTy != PartTy && "identical part types shouldn't reach here");
563 
564   const TypeSize PartSize = PartTy.getSizeInBits();
565 
566   if (PartTy.isVector() == SrcTy.isVector() &&
567       PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
568     assert(DstRegs.size() == 1);
569     B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});
570     return;
571   }
572 
573   if (SrcTy.isVector() && !PartTy.isVector() &&
574       TypeSize::isKnownGT(PartSize, SrcTy.getElementType().getSizeInBits())) {
575     // Vector was scalarized, and the elements extended.
576     auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg);
577     for (int i = 0, e = DstRegs.size(); i != e; ++i)
578       B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));
579     return;
580   }
581 
582   if (SrcTy.isVector() && PartTy.isVector() &&
583       PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&
584       ElementCount::isKnownLT(SrcTy.getElementCount(),
585                               PartTy.getElementCount())) {
586     // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32
587     Register DstReg = DstRegs.front();
588     B.buildPadVectorWithUndefElements(DstReg, SrcReg);
589     return;
590   }
591 
592   LLT GCDTy = getGCDType(SrcTy, PartTy);
593   if (GCDTy == PartTy) {
594     // If this already evenly divisible, we can create a simple unmerge.
595     B.buildUnmerge(DstRegs, SrcReg);
596     return;
597   }
598 
599   if (SrcTy.isVector() && !PartTy.isVector() &&
600       SrcTy.getScalarSizeInBits() > PartTy.getSizeInBits()) {
601     LLT ExtTy =
602         LLT::vector(SrcTy.getElementCount(),
603                     LLT::scalar(PartTy.getScalarSizeInBits() * DstRegs.size() /
604                                 SrcTy.getNumElements()));
605     auto Ext = B.buildAnyExt(ExtTy, SrcReg);
606     B.buildUnmerge(DstRegs, Ext);
607     return;
608   }
609 
610   MachineRegisterInfo &MRI = *B.getMRI();
611   LLT DstTy = MRI.getType(DstRegs[0]);
612   LLT LCMTy = getCoverTy(SrcTy, PartTy);
613 
614   if (PartTy.isVector() && LCMTy == PartTy) {
615     assert(DstRegs.size() == 1);
616     B.buildPadVectorWithUndefElements(DstRegs[0], SrcReg);
617     return;
618   }
619 
620   const unsigned DstSize = DstTy.getSizeInBits();
621   const unsigned SrcSize = SrcTy.getSizeInBits();
622   unsigned CoveringSize = LCMTy.getSizeInBits();
623 
624   Register UnmergeSrc = SrcReg;
625 
626   if (!LCMTy.isVector() && CoveringSize != SrcSize) {
627     // For scalars, it's common to be able to use a simple extension.
628     if (SrcTy.isScalar() && DstTy.isScalar()) {
629       CoveringSize = alignTo(SrcSize, DstSize);
630       LLT CoverTy = LLT::scalar(CoveringSize);
631       UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0);
632     } else {
633       // Widen to the common type.
634       // FIXME: This should respect the extend type
635       Register Undef = B.buildUndef(SrcTy).getReg(0);
636       SmallVector<Register, 8> MergeParts(1, SrcReg);
637       for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
638         MergeParts.push_back(Undef);
639       UnmergeSrc = B.buildMergeLikeInstr(LCMTy, MergeParts).getReg(0);
640     }
641   }
642 
643   if (LCMTy.isVector() && CoveringSize != SrcSize)
644     UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0);
645 
646   B.buildUnmerge(DstRegs, UnmergeSrc);
647 }
648 
649 bool CallLowering::determineAndHandleAssignments(
650     ValueHandler &Handler, ValueAssigner &Assigner,
651     SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
652     CallingConv::ID CallConv, bool IsVarArg,
653     ArrayRef<Register> ThisReturnRegs) const {
654   MachineFunction &MF = MIRBuilder.getMF();
655   const Function &F = MF.getFunction();
656   SmallVector<CCValAssign, 16> ArgLocs;
657 
658   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
659   if (!determineAssignments(Assigner, Args, CCInfo))
660     return false;
661 
662   return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder,
663                            ThisReturnRegs);
664 }
665 
666 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
667   if (Flags.isSExt())
668     return TargetOpcode::G_SEXT;
669   if (Flags.isZExt())
670     return TargetOpcode::G_ZEXT;
671   return TargetOpcode::G_ANYEXT;
672 }
673 
674 bool CallLowering::determineAssignments(ValueAssigner &Assigner,
675                                         SmallVectorImpl<ArgInfo> &Args,
676                                         CCState &CCInfo) const {
677   LLVMContext &Ctx = CCInfo.getContext();
678   const CallingConv::ID CallConv = CCInfo.getCallingConv();
679 
680   unsigned NumArgs = Args.size();
681   for (unsigned i = 0; i != NumArgs; ++i) {
682     EVT CurVT = EVT::getEVT(Args[i].Ty);
683 
684     MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT);
685 
686     // If we need to split the type over multiple regs, check it's a scenario
687     // we currently support.
688     unsigned NumParts =
689         TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT);
690 
691     if (NumParts == 1) {
692       // Try to use the register type if we couldn't assign the VT.
693       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
694                              Args[i].Flags[0], CCInfo))
695         return false;
696       continue;
697     }
698 
699     // For incoming arguments (physregs to vregs), we could have values in
700     // physregs (or memlocs) which we want to extract and copy to vregs.
701     // During this, we might have to deal with the LLT being split across
702     // multiple regs, so we have to record this information for later.
703     //
704     // If we have outgoing args, then we have the opposite case. We have a
705     // vreg with an LLT which we want to assign to a physical location, and
706     // we might have to record that the value has to be split later.
707 
708     // We're handling an incoming arg which is split over multiple regs.
709     // E.g. passing an s128 on AArch64.
710     ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
711     Args[i].Flags.clear();
712 
713     for (unsigned Part = 0; Part < NumParts; ++Part) {
714       ISD::ArgFlagsTy Flags = OrigFlags;
715       if (Part == 0) {
716         Flags.setSplit();
717       } else {
718         Flags.setOrigAlign(Align(1));
719         if (Part == NumParts - 1)
720           Flags.setSplitEnd();
721       }
722 
723       Args[i].Flags.push_back(Flags);
724       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
725                              Args[i].Flags[Part], CCInfo)) {
726         // Still couldn't assign this smaller part type for some reason.
727         return false;
728       }
729     }
730   }
731 
732   return true;
733 }
734 
735 bool CallLowering::handleAssignments(ValueHandler &Handler,
736                                      SmallVectorImpl<ArgInfo> &Args,
737                                      CCState &CCInfo,
738                                      SmallVectorImpl<CCValAssign> &ArgLocs,
739                                      MachineIRBuilder &MIRBuilder,
740                                      ArrayRef<Register> ThisReturnRegs) const {
741   MachineFunction &MF = MIRBuilder.getMF();
742   MachineRegisterInfo &MRI = MF.getRegInfo();
743   const Function &F = MF.getFunction();
744   const DataLayout &DL = F.getDataLayout();
745 
746   const unsigned NumArgs = Args.size();
747 
748   // Stores thunks for outgoing register assignments. This is used so we delay
749   // generating register copies until mem loc assignments are done. We do this
750   // so that if the target is using the delayed stack protector feature, we can
751   // find the split point of the block accurately. E.g. if we have:
752   // G_STORE %val, %memloc
753   // $x0 = COPY %foo
754   // $x1 = COPY %bar
755   // CALL func
756   // ... then the split point for the block will correctly be at, and including,
757   // the copy to $x0. If instead the G_STORE instruction immediately precedes
758   // the CALL, then we'd prematurely choose the CALL as the split point, thus
759   // generating a split block with a CALL that uses undefined physregs.
760   SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
761 
762   for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
763     assert(j < ArgLocs.size() && "Skipped too many arg locs");
764     CCValAssign &VA = ArgLocs[j];
765     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
766 
767     if (VA.needsCustom()) {
768       std::function<void()> Thunk;
769       unsigned NumArgRegs = Handler.assignCustomValue(
770           Args[i], ArrayRef(ArgLocs).slice(j), &Thunk);
771       if (Thunk)
772         DelayedOutgoingRegAssignments.emplace_back(Thunk);
773       if (!NumArgRegs)
774         return false;
775       j += (NumArgRegs - 1);
776       continue;
777     }
778 
779     auto AllocaAddressSpace = MF.getDataLayout().getAllocaAddrSpace();
780 
781     const MVT ValVT = VA.getValVT();
782     const MVT LocVT = VA.getLocVT();
783 
784     const LLT LocTy(LocVT);
785     const LLT ValTy(ValVT);
786     const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
787     const EVT OrigVT = EVT::getEVT(Args[i].Ty);
788     const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
789     const LLT PointerTy = LLT::pointer(
790         AllocaAddressSpace, DL.getPointerSizeInBits(AllocaAddressSpace));
791 
792     // Expected to be multiple regs for a single incoming arg.
793     // There should be Regs.size() ArgLocs per argument.
794     // This should be the same as getNumRegistersForCallingConv
795     const unsigned NumParts = Args[i].Flags.size();
796 
797     // Now split the registers into the assigned types.
798     Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end());
799 
800     if (NumParts != 1 || NewLLT != OrigTy) {
801       // If we can't directly assign the register, we need one or more
802       // intermediate values.
803       Args[i].Regs.resize(NumParts);
804 
805       // When we have indirect parameter passing we are receiving a pointer,
806       // that points to the actual value, so we need one "temporary" pointer.
807       if (VA.getLocInfo() == CCValAssign::Indirect) {
808         if (Handler.isIncomingArgumentHandler())
809           Args[i].Regs[0] = MRI.createGenericVirtualRegister(PointerTy);
810       } else {
811         // For each split register, create and assign a vreg that will store
812         // the incoming component of the larger value. These will later be
813         // merged to form the final vreg.
814         for (unsigned Part = 0; Part < NumParts; ++Part)
815           Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT);
816       }
817     }
818 
819     assert((j + (NumParts - 1)) < ArgLocs.size() &&
820            "Too many regs for number of args");
821 
822     // Coerce into outgoing value types before register assignment.
823     if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy &&
824         VA.getLocInfo() != CCValAssign::Indirect) {
825       assert(Args[i].OrigRegs.size() == 1);
826       buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy,
827                       ValTy, extendOpFromFlags(Args[i].Flags[0]));
828     }
829 
830     bool IndirectParameterPassingHandled = false;
831     bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL);
832     for (unsigned Part = 0; Part < NumParts; ++Part) {
833       assert((VA.getLocInfo() != CCValAssign::Indirect || Part == 0) &&
834              "Only the first parameter should be processed when "
835              "handling indirect passing!");
836       Register ArgReg = Args[i].Regs[Part];
837       // There should be Regs.size() ArgLocs per argument.
838       unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
839       CCValAssign &VA = ArgLocs[j + Idx];
840       const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
841 
842       // We found an indirect parameter passing, and we have an
843       // OutgoingValueHandler as our handler (so we are at the call site or the
844       // return value). In this case, start the construction of the following
845       // GMIR, that is responsible for the preparation of indirect parameter
846       // passing:
847       //
848       // %1(indirectly passed type) = The value to pass
849       // %3(pointer) = G_FRAME_INDEX %stack.0
850       // G_STORE %1, %3 :: (store (s128), align 8)
851       //
852       // After this GMIR, the remaining part of the loop body will decide how
853       // to get the value to the caller and we break out of the loop.
854       if (VA.getLocInfo() == CCValAssign::Indirect &&
855           !Handler.isIncomingArgumentHandler()) {
856         Align AlignmentForStored = DL.getPrefTypeAlign(Args[i].Ty);
857         MachineFrameInfo &MFI = MF.getFrameInfo();
858         // Get some space on the stack for the value, so later we can pass it
859         // as a reference.
860         int FrameIdx = MFI.CreateStackObject(OrigTy.getScalarSizeInBits(),
861                                              AlignmentForStored, false);
862         Register PointerToStackReg =
863             MIRBuilder.buildFrameIndex(PointerTy, FrameIdx).getReg(0);
864         MachinePointerInfo StackPointerMPO =
865             MachinePointerInfo::getFixedStack(MF, FrameIdx);
866         // Store the value in the previously created stack space.
867         MIRBuilder.buildStore(Args[i].OrigRegs[Part], PointerToStackReg,
868                               StackPointerMPO,
869                               inferAlignFromPtrInfo(MF, StackPointerMPO));
870 
871         ArgReg = PointerToStackReg;
872         IndirectParameterPassingHandled = true;
873       }
874 
875       if (VA.isMemLoc() && !Flags.isByVal()) {
876         // Individual pieces may have been spilled to the stack and others
877         // passed in registers.
878 
879         // TODO: The memory size may be larger than the value we need to
880         // store. We may need to adjust the offset for big endian targets.
881         LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
882 
883         MachinePointerInfo MPO;
884         Register StackAddr =
885             Handler.getStackAddress(VA.getLocInfo() == CCValAssign::Indirect
886                                         ? PointerTy.getSizeInBytes()
887                                         : MemTy.getSizeInBytes(),
888                                     VA.getLocMemOffset(), MPO, Flags);
889 
890         // Finish the handling of indirect passing from the passers
891         // (OutgoingParameterHandler) side.
892         // This branch is needed, so the pointer to the value is loaded onto the
893         // stack.
894         if (VA.getLocInfo() == CCValAssign::Indirect)
895           Handler.assignValueToAddress(ArgReg, StackAddr, PointerTy, MPO, VA);
896         else
897           Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO,
898                                        VA);
899       } else if (VA.isMemLoc() && Flags.isByVal()) {
900         assert(Args[i].Regs.size() == 1 && "didn't expect split byval pointer");
901 
902         if (Handler.isIncomingArgumentHandler()) {
903           // We just need to copy the frame index value to the pointer.
904           MachinePointerInfo MPO;
905           Register StackAddr = Handler.getStackAddress(
906               Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);
907           MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);
908         } else {
909           // For outgoing byval arguments, insert the implicit copy byval
910           // implies, such that writes in the callee do not modify the caller's
911           // value.
912           uint64_t MemSize = Flags.getByValSize();
913           int64_t Offset = VA.getLocMemOffset();
914 
915           MachinePointerInfo DstMPO;
916           Register StackAddr =
917               Handler.getStackAddress(MemSize, Offset, DstMPO, Flags);
918 
919           MachinePointerInfo SrcMPO(Args[i].OrigValue);
920           if (!Args[i].OrigValue) {
921             // We still need to accurately track the stack address space if we
922             // don't know the underlying value.
923             const LLT PtrTy = MRI.getType(StackAddr);
924             SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
925           }
926 
927           Align DstAlign = std::max(Flags.getNonZeroByValAlign(),
928                                     inferAlignFromPtrInfo(MF, DstMPO));
929 
930           Align SrcAlign = std::max(Flags.getNonZeroByValAlign(),
931                                     inferAlignFromPtrInfo(MF, SrcMPO));
932 
933           Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0],
934                                      DstMPO, DstAlign, SrcMPO, SrcAlign,
935                                      MemSize, VA);
936         }
937       } else if (i == 0 && !ThisReturnRegs.empty() &&
938                  Handler.isIncomingArgumentHandler() &&
939                  isTypeIsValidForThisReturn(ValVT)) {
940         Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA);
941       } else if (Handler.isIncomingArgumentHandler()) {
942         Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
943       } else {
944         DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() {
945           Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
946         });
947       }
948 
949       // Finish the handling of indirect parameter passing when receiving
950       // the value (we are in the called function or the caller when receiving
951       // the return value).
952       if (VA.getLocInfo() == CCValAssign::Indirect &&
953           Handler.isIncomingArgumentHandler()) {
954         Align Alignment = DL.getABITypeAlign(Args[i].Ty);
955         MachinePointerInfo MPO = MachinePointerInfo::getUnknownStack(MF);
956 
957         // Since we are doing indirect parameter passing, we know that the value
958         // in the temporary register is not the value passed to the function,
959         // but rather a pointer to that value. Let's load that value into the
960         // virtual register where the parameter should go.
961         MIRBuilder.buildLoad(Args[i].OrigRegs[0], Args[i].Regs[0], MPO,
962                              Alignment);
963 
964         IndirectParameterPassingHandled = true;
965       }
966 
967       if (IndirectParameterPassingHandled)
968         break;
969     }
970 
971     // Now that all pieces have been assigned, re-pack the register typed values
972     // into the original value typed registers. This is only necessary, when
973     // the value was passed in multiple registers, not indirectly.
974     if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT &&
975         !IndirectParameterPassingHandled) {
976       // Merge the split registers into the expected larger result vregs of
977       // the original call.
978       buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy,
979                         LocTy, Args[i].Flags[0]);
980     }
981 
982     j += NumParts - 1;
983   }
984   for (auto &Fn : DelayedOutgoingRegAssignments)
985     Fn();
986 
987   return true;
988 }
989 
990 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
991                                    ArrayRef<Register> VRegs, Register DemoteReg,
992                                    int FI) const {
993   MachineFunction &MF = MIRBuilder.getMF();
994   MachineRegisterInfo &MRI = MF.getRegInfo();
995   const DataLayout &DL = MF.getDataLayout();
996 
997   SmallVector<EVT, 4> SplitVTs;
998   SmallVector<uint64_t, 4> Offsets;
999   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
1000 
1001   assert(VRegs.size() == SplitVTs.size());
1002 
1003   unsigned NumValues = SplitVTs.size();
1004   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
1005   Type *RetPtrTy =
1006       PointerType::get(RetTy->getContext(), DL.getAllocaAddrSpace());
1007   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetPtrTy), DL);
1008 
1009   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
1010 
1011   for (unsigned I = 0; I < NumValues; ++I) {
1012     Register Addr;
1013     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
1014     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
1015                                         MRI.getType(VRegs[I]),
1016                                         commonAlignment(BaseAlign, Offsets[I]));
1017     MIRBuilder.buildLoad(VRegs[I], Addr, *MMO);
1018   }
1019 }
1020 
1021 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
1022                                     ArrayRef<Register> VRegs,
1023                                     Register DemoteReg) const {
1024   MachineFunction &MF = MIRBuilder.getMF();
1025   MachineRegisterInfo &MRI = MF.getRegInfo();
1026   const DataLayout &DL = MF.getDataLayout();
1027 
1028   SmallVector<EVT, 4> SplitVTs;
1029   SmallVector<uint64_t, 4> Offsets;
1030   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
1031 
1032   assert(VRegs.size() == SplitVTs.size());
1033 
1034   unsigned NumValues = SplitVTs.size();
1035   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
1036   unsigned AS = DL.getAllocaAddrSpace();
1037   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getPointerTo(AS)), DL);
1038 
1039   MachinePointerInfo PtrInfo(AS);
1040 
1041   for (unsigned I = 0; I < NumValues; ++I) {
1042     Register Addr;
1043     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
1044     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
1045                                         MRI.getType(VRegs[I]),
1046                                         commonAlignment(BaseAlign, Offsets[I]));
1047     MIRBuilder.buildStore(VRegs[I], Addr, *MMO);
1048   }
1049 }
1050 
1051 void CallLowering::insertSRetIncomingArgument(
1052     const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
1053     MachineRegisterInfo &MRI, const DataLayout &DL) const {
1054   unsigned AS = DL.getAllocaAddrSpace();
1055   DemoteReg = MRI.createGenericVirtualRegister(
1056       LLT::pointer(AS, DL.getPointerSizeInBits(AS)));
1057 
1058   Type *PtrTy = PointerType::get(F.getReturnType(), AS);
1059 
1060   SmallVector<EVT, 1> ValueVTs;
1061   ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs);
1062 
1063   // NOTE: Assume that a pointer won't get split into more than one VT.
1064   assert(ValueVTs.size() == 1);
1065 
1066   ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()),
1067                     ArgInfo::NoArgIndex);
1068   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F);
1069   DemoteArg.Flags[0].setSRet();
1070   SplitArgs.insert(SplitArgs.begin(), DemoteArg);
1071 }
1072 
1073 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
1074                                               const CallBase &CB,
1075                                               CallLoweringInfo &Info) const {
1076   const DataLayout &DL = MIRBuilder.getDataLayout();
1077   Type *RetTy = CB.getType();
1078   unsigned AS = DL.getAllocaAddrSpace();
1079   LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
1080 
1081   int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
1082       DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false);
1083 
1084   Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0);
1085   ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS),
1086                     ArgInfo::NoArgIndex);
1087   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB);
1088   DemoteArg.Flags[0].setSRet();
1089 
1090   Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg);
1091   Info.DemoteStackIndex = FI;
1092   Info.DemoteRegister = DemoteReg;
1093 }
1094 
1095 bool CallLowering::checkReturn(CCState &CCInfo,
1096                                SmallVectorImpl<BaseArgInfo> &Outs,
1097                                CCAssignFn *Fn) const {
1098   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
1099     MVT VT = MVT::getVT(Outs[I].Ty);
1100     if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
1101       return false;
1102   }
1103   return true;
1104 }
1105 
1106 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
1107                                  AttributeList Attrs,
1108                                  SmallVectorImpl<BaseArgInfo> &Outs,
1109                                  const DataLayout &DL) const {
1110   LLVMContext &Context = RetTy->getContext();
1111   ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1112 
1113   SmallVector<EVT, 4> SplitVTs;
1114   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs);
1115   addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex);
1116 
1117   for (EVT VT : SplitVTs) {
1118     unsigned NumParts =
1119         TLI->getNumRegistersForCallingConv(Context, CallConv, VT);
1120     MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT);
1121     Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1122 
1123     for (unsigned I = 0; I < NumParts; ++I) {
1124       Outs.emplace_back(PartTy, Flags);
1125     }
1126   }
1127 }
1128 
1129 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1130   const auto &F = MF.getFunction();
1131   Type *ReturnType = F.getReturnType();
1132   CallingConv::ID CallConv = F.getCallingConv();
1133 
1134   SmallVector<BaseArgInfo, 4> SplitArgs;
1135   getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs,
1136                 MF.getDataLayout());
1137   return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg());
1138 }
1139 
1140 bool CallLowering::parametersInCSRMatch(
1141     const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1142     const SmallVectorImpl<CCValAssign> &OutLocs,
1143     const SmallVectorImpl<ArgInfo> &OutArgs) const {
1144   for (unsigned i = 0; i < OutLocs.size(); ++i) {
1145     const auto &ArgLoc = OutLocs[i];
1146     // If it's not a register, it's fine.
1147     if (!ArgLoc.isRegLoc())
1148       continue;
1149 
1150     MCRegister PhysReg = ArgLoc.getLocReg();
1151 
1152     // Only look at callee-saved registers.
1153     if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg))
1154       continue;
1155 
1156     LLVM_DEBUG(
1157         dbgs()
1158         << "... Call has an argument passed in a callee-saved register.\n");
1159 
1160     // Check if it was copied from.
1161     const ArgInfo &OutInfo = OutArgs[i];
1162 
1163     if (OutInfo.Regs.size() > 1) {
1164       LLVM_DEBUG(
1165           dbgs() << "... Cannot handle arguments in multiple registers.\n");
1166       return false;
1167     }
1168 
1169     // Check if we copy the register, walking through copies from virtual
1170     // registers. Note that getDefIgnoringCopies does not ignore copies from
1171     // physical registers.
1172     MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);
1173     if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1174       LLVM_DEBUG(
1175           dbgs()
1176           << "... Parameter was not copied into a VReg, cannot tail call.\n");
1177       return false;
1178     }
1179 
1180     // Got a copy. Verify that it's the same as the register we want.
1181     Register CopyRHS = RegDef->getOperand(1).getReg();
1182     if (CopyRHS != PhysReg) {
1183       LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1184                            "VReg, cannot tail call.\n");
1185       return false;
1186     }
1187   }
1188 
1189   return true;
1190 }
1191 
1192 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1193                                      MachineFunction &MF,
1194                                      SmallVectorImpl<ArgInfo> &InArgs,
1195                                      ValueAssigner &CalleeAssigner,
1196                                      ValueAssigner &CallerAssigner) const {
1197   const Function &F = MF.getFunction();
1198   CallingConv::ID CalleeCC = Info.CallConv;
1199   CallingConv::ID CallerCC = F.getCallingConv();
1200 
1201   if (CallerCC == CalleeCC)
1202     return true;
1203 
1204   SmallVector<CCValAssign, 16> ArgLocs1;
1205   CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1206   if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1))
1207     return false;
1208 
1209   SmallVector<CCValAssign, 16> ArgLocs2;
1210   CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1211   if (!determineAssignments(CallerAssigner, InArgs, CCInfo2))
1212     return false;
1213 
1214   // We need the argument locations to match up exactly. If there's more in
1215   // one than the other, then we are done.
1216   if (ArgLocs1.size() != ArgLocs2.size())
1217     return false;
1218 
1219   // Make sure that each location is passed in exactly the same way.
1220   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1221     const CCValAssign &Loc1 = ArgLocs1[i];
1222     const CCValAssign &Loc2 = ArgLocs2[i];
1223 
1224     // We need both of them to be the same. So if one is a register and one
1225     // isn't, we're done.
1226     if (Loc1.isRegLoc() != Loc2.isRegLoc())
1227       return false;
1228 
1229     if (Loc1.isRegLoc()) {
1230       // If they don't have the same register location, we're done.
1231       if (Loc1.getLocReg() != Loc2.getLocReg())
1232         return false;
1233 
1234       // They matched, so we can move to the next ArgLoc.
1235       continue;
1236     }
1237 
1238     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1239     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1240       return false;
1241   }
1242 
1243   return true;
1244 }
1245 
1246 LLT CallLowering::ValueHandler::getStackValueStoreType(
1247     const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1248   const MVT ValVT = VA.getValVT();
1249   if (ValVT != MVT::iPTR) {
1250     LLT ValTy(ValVT);
1251 
1252     // We lost the pointeriness going through CCValAssign, so try to restore it
1253     // based on the flags.
1254     if (Flags.isPointer()) {
1255       LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(),
1256                                ValTy.getScalarSizeInBits());
1257       if (ValVT.isVector())
1258         return LLT::vector(ValTy.getElementCount(), PtrTy);
1259       return PtrTy;
1260     }
1261 
1262     return ValTy;
1263   }
1264 
1265   unsigned AddrSpace = Flags.getPointerAddrSpace();
1266   return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace));
1267 }
1268 
1269 void CallLowering::ValueHandler::copyArgumentMemory(
1270     const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1271     const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1272     const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1273     CCValAssign &VA) const {
1274   MachineFunction &MF = MIRBuilder.getMF();
1275   MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1276       SrcPtrInfo,
1277       MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize,
1278       SrcAlign);
1279 
1280   MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1281       DstPtrInfo,
1282       MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1283       MemSize, DstAlign);
1284 
1285   const LLT PtrTy = MRI.getType(DstPtr);
1286   const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits());
1287 
1288   auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize);
1289   MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO);
1290 }
1291 
1292 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1293                                                     const CCValAssign &VA,
1294                                                     unsigned MaxSizeBits) {
1295   LLT LocTy{VA.getLocVT()};
1296   LLT ValTy{VA.getValVT()};
1297 
1298   if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1299     return ValReg;
1300 
1301   if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1302     if (MaxSizeBits <= ValTy.getSizeInBits())
1303       return ValReg;
1304     LocTy = LLT::scalar(MaxSizeBits);
1305   }
1306 
1307   const LLT ValRegTy = MRI.getType(ValReg);
1308   if (ValRegTy.isPointer()) {
1309     // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1310     // we have to cast to do the extension.
1311     LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits());
1312     ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0);
1313   }
1314 
1315   switch (VA.getLocInfo()) {
1316   default:
1317     break;
1318   case CCValAssign::Full:
1319   case CCValAssign::BCvt:
1320     // FIXME: bitconverting between vector types may or may not be a
1321     // nop in big-endian situations.
1322     return ValReg;
1323   case CCValAssign::AExt: {
1324     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
1325     return MIB.getReg(0);
1326   }
1327   case CCValAssign::SExt: {
1328     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1329     MIRBuilder.buildSExt(NewReg, ValReg);
1330     return NewReg;
1331   }
1332   case CCValAssign::ZExt: {
1333     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1334     MIRBuilder.buildZExt(NewReg, ValReg);
1335     return NewReg;
1336   }
1337   }
1338   llvm_unreachable("unable to extend register");
1339 }
1340 
1341 void CallLowering::ValueAssigner::anchor() {}
1342 
1343 Register CallLowering::IncomingValueHandler::buildExtensionHint(
1344     const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1345   switch (VA.getLocInfo()) {
1346   case CCValAssign::LocInfo::ZExt: {
1347     return MIRBuilder
1348         .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1349                          NarrowTy.getScalarSizeInBits())
1350         .getReg(0);
1351   }
1352   case CCValAssign::LocInfo::SExt: {
1353     return MIRBuilder
1354         .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1355                          NarrowTy.getScalarSizeInBits())
1356         .getReg(0);
1357     break;
1358   }
1359   default:
1360     return SrcReg;
1361   }
1362 }
1363 
1364 /// Check if we can use a basic COPY instruction between the two types.
1365 ///
1366 /// We're currently building on top of the infrastructure using MVT, which loses
1367 /// pointer information in the CCValAssign. We accept copies from physical
1368 /// registers that have been reported as integers if it's to an equivalent sized
1369 /// pointer LLT.
1370 static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1371   if (SrcTy == DstTy)
1372     return true;
1373 
1374   if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1375     return false;
1376 
1377   SrcTy = SrcTy.getScalarType();
1378   DstTy = DstTy.getScalarType();
1379 
1380   return (SrcTy.isPointer() && DstTy.isScalar()) ||
1381          (DstTy.isPointer() && SrcTy.isScalar());
1382 }
1383 
1384 void CallLowering::IncomingValueHandler::assignValueToReg(
1385     Register ValVReg, Register PhysReg, const CCValAssign &VA) {
1386   const MVT LocVT = VA.getLocVT();
1387   const LLT LocTy(LocVT);
1388   const LLT RegTy = MRI.getType(ValVReg);
1389 
1390   if (isCopyCompatibleType(RegTy, LocTy)) {
1391     MIRBuilder.buildCopy(ValVReg, PhysReg);
1392     return;
1393   }
1394 
1395   auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg);
1396   auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy);
1397   MIRBuilder.buildTrunc(ValVReg, Hint);
1398 }
1399