xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp (revision a8089ea5aee578e08acab2438e82fc9a9ae50ed8)
1 //===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements the lowering of LLVM calls to machine code calls for
11 /// GlobalISel.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #include "AArch64CallLowering.h"
16 #include "AArch64ISelLowering.h"
17 #include "AArch64MachineFunctionInfo.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/Analysis/ObjCARCUtil.h"
23 #include "llvm/CodeGen/Analysis.h"
24 #include "llvm/CodeGen/CallingConvLower.h"
25 #include "llvm/CodeGen/FunctionLoweringInfo.h"
26 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
27 #include "llvm/CodeGen/GlobalISel/Utils.h"
28 #include "llvm/CodeGen/LowLevelTypeUtils.h"
29 #include "llvm/CodeGen/MachineBasicBlock.h"
30 #include "llvm/CodeGen/MachineFrameInfo.h"
31 #include "llvm/CodeGen/MachineFunction.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineMemOperand.h"
34 #include "llvm/CodeGen/MachineOperand.h"
35 #include "llvm/CodeGen/MachineRegisterInfo.h"
36 #include "llvm/CodeGen/MachineValueType.h"
37 #include "llvm/CodeGen/TargetRegisterInfo.h"
38 #include "llvm/CodeGen/TargetSubtargetInfo.h"
39 #include "llvm/CodeGen/ValueTypes.h"
40 #include "llvm/IR/Argument.h"
41 #include "llvm/IR/Attributes.h"
42 #include "llvm/IR/Function.h"
43 #include "llvm/IR/Type.h"
44 #include "llvm/IR/Value.h"
45 #include <algorithm>
46 #include <cassert>
47 #include <cstdint>
48 #include <iterator>
49 
50 #define DEBUG_TYPE "aarch64-call-lowering"
51 
52 using namespace llvm;
53 
54 AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
55   : CallLowering(&TLI) {}
56 
57 static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT,
58                                              MVT &LocVT) {
59   // If ValVT is i1/i8/i16, we should set LocVT to i8/i8/i16. This is a legacy
60   // hack because the DAG calls the assignment function with pre-legalized
61   // register typed values, not the raw type.
62   //
63   // This hack is not applied to return values which are not passed on the
64   // stack.
65   if (OrigVT == MVT::i1 || OrigVT == MVT::i8)
66     ValVT = LocVT = MVT::i8;
67   else if (OrigVT == MVT::i16)
68     ValVT = LocVT = MVT::i16;
69 }
70 
71 // Account for i1/i8/i16 stack passed value hack
72 static LLT getStackValueStoreTypeHack(const CCValAssign &VA) {
73   const MVT ValVT = VA.getValVT();
74   return (ValVT == MVT::i8 || ValVT == MVT::i16) ? LLT(ValVT)
75                                                  : LLT(VA.getLocVT());
76 }
77 
78 namespace {
79 
80 struct AArch64IncomingValueAssigner
81     : public CallLowering::IncomingValueAssigner {
82   AArch64IncomingValueAssigner(CCAssignFn *AssignFn_,
83                                CCAssignFn *AssignFnVarArg_)
84       : IncomingValueAssigner(AssignFn_, AssignFnVarArg_) {}
85 
86   bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
87                  CCValAssign::LocInfo LocInfo,
88                  const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
89                  CCState &State) override {
90     applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
91     return IncomingValueAssigner::assignArg(ValNo, OrigVT, ValVT, LocVT,
92                                             LocInfo, Info, Flags, State);
93   }
94 };
95 
96 struct AArch64OutgoingValueAssigner
97     : public CallLowering::OutgoingValueAssigner {
98   const AArch64Subtarget &Subtarget;
99 
100   /// Track if this is used for a return instead of function argument
101   /// passing. We apply a hack to i1/i8/i16 stack passed values, but do not use
102   /// stack passed returns for them and cannot apply the type adjustment.
103   bool IsReturn;
104 
105   AArch64OutgoingValueAssigner(CCAssignFn *AssignFn_,
106                                CCAssignFn *AssignFnVarArg_,
107                                const AArch64Subtarget &Subtarget_,
108                                bool IsReturn)
109       : OutgoingValueAssigner(AssignFn_, AssignFnVarArg_),
110         Subtarget(Subtarget_), IsReturn(IsReturn) {}
111 
112   bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
113                  CCValAssign::LocInfo LocInfo,
114                  const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
115                  CCState &State) override {
116     bool IsCalleeWin = Subtarget.isCallingConvWin64(State.getCallingConv());
117     bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg();
118 
119     bool Res;
120     if (Info.IsFixed && !UseVarArgsCCForFixed) {
121       if (!IsReturn)
122         applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
123       Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
124     } else
125       Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, State);
126 
127     StackSize = State.getStackSize();
128     return Res;
129   }
130 };
131 
132 struct IncomingArgHandler : public CallLowering::IncomingValueHandler {
133   IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
134       : IncomingValueHandler(MIRBuilder, MRI) {}
135 
136   Register getStackAddress(uint64_t Size, int64_t Offset,
137                            MachinePointerInfo &MPO,
138                            ISD::ArgFlagsTy Flags) override {
139     auto &MFI = MIRBuilder.getMF().getFrameInfo();
140 
141     // Byval is assumed to be writable memory, but other stack passed arguments
142     // are not.
143     const bool IsImmutable = !Flags.isByVal();
144 
145     int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable);
146     MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
147     auto AddrReg = MIRBuilder.buildFrameIndex(LLT::pointer(0, 64), FI);
148     return AddrReg.getReg(0);
149   }
150 
151   LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
152                              ISD::ArgFlagsTy Flags) const override {
153     // For pointers, we just need to fixup the integer types reported in the
154     // CCValAssign.
155     if (Flags.isPointer())
156       return CallLowering::ValueHandler::getStackValueStoreType(DL, VA, Flags);
157     return getStackValueStoreTypeHack(VA);
158   }
159 
160   void assignValueToReg(Register ValVReg, Register PhysReg,
161                         const CCValAssign &VA) override {
162     markPhysRegUsed(PhysReg);
163     IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
164   }
165 
166   void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
167                             const MachinePointerInfo &MPO,
168                             const CCValAssign &VA) override {
169     MachineFunction &MF = MIRBuilder.getMF();
170 
171     LLT ValTy(VA.getValVT());
172     LLT LocTy(VA.getLocVT());
173 
174     // Fixup the types for the DAG compatibility hack.
175     if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16)
176       std::swap(ValTy, LocTy);
177     else {
178       // The calling code knows if this is a pointer or not, we're only touching
179       // the LocTy for the i8/i16 hack.
180       assert(LocTy.getSizeInBits() == MemTy.getSizeInBits());
181       LocTy = MemTy;
182     }
183 
184     auto MMO = MF.getMachineMemOperand(
185         MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, LocTy,
186         inferAlignFromPtrInfo(MF, MPO));
187 
188     switch (VA.getLocInfo()) {
189     case CCValAssign::LocInfo::ZExt:
190       MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, ValVReg, Addr, *MMO);
191       return;
192     case CCValAssign::LocInfo::SExt:
193       MIRBuilder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, ValVReg, Addr, *MMO);
194       return;
195     default:
196       MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
197       return;
198     }
199   }
200 
201   /// How the physical register gets marked varies between formal
202   /// parameters (it's a basic-block live-in), and a call instruction
203   /// (it's an implicit-def of the BL).
204   virtual void markPhysRegUsed(MCRegister PhysReg) = 0;
205 };
206 
207 struct FormalArgHandler : public IncomingArgHandler {
208   FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
209       : IncomingArgHandler(MIRBuilder, MRI) {}
210 
211   void markPhysRegUsed(MCRegister PhysReg) override {
212     MIRBuilder.getMRI()->addLiveIn(PhysReg);
213     MIRBuilder.getMBB().addLiveIn(PhysReg);
214   }
215 };
216 
217 struct CallReturnHandler : public IncomingArgHandler {
218   CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
219                     MachineInstrBuilder MIB)
220       : IncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
221 
222   void markPhysRegUsed(MCRegister PhysReg) override {
223     MIB.addDef(PhysReg, RegState::Implicit);
224   }
225 
226   MachineInstrBuilder MIB;
227 };
228 
229 /// A special return arg handler for "returned" attribute arg calls.
230 struct ReturnedArgCallReturnHandler : public CallReturnHandler {
231   ReturnedArgCallReturnHandler(MachineIRBuilder &MIRBuilder,
232                                MachineRegisterInfo &MRI,
233                                MachineInstrBuilder MIB)
234       : CallReturnHandler(MIRBuilder, MRI, MIB) {}
235 
236   void markPhysRegUsed(MCRegister PhysReg) override {}
237 };
238 
239 struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
240   OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
241                      MachineInstrBuilder MIB, bool IsTailCall = false,
242                      int FPDiff = 0)
243       : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), IsTailCall(IsTailCall),
244         FPDiff(FPDiff),
245         Subtarget(MIRBuilder.getMF().getSubtarget<AArch64Subtarget>()) {}
246 
247   Register getStackAddress(uint64_t Size, int64_t Offset,
248                            MachinePointerInfo &MPO,
249                            ISD::ArgFlagsTy Flags) override {
250     MachineFunction &MF = MIRBuilder.getMF();
251     LLT p0 = LLT::pointer(0, 64);
252     LLT s64 = LLT::scalar(64);
253 
254     if (IsTailCall) {
255       assert(!Flags.isByVal() && "byval unhandled with tail calls");
256 
257       Offset += FPDiff;
258       int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
259       auto FIReg = MIRBuilder.buildFrameIndex(p0, FI);
260       MPO = MachinePointerInfo::getFixedStack(MF, FI);
261       return FIReg.getReg(0);
262     }
263 
264     if (!SPReg)
265       SPReg = MIRBuilder.buildCopy(p0, Register(AArch64::SP)).getReg(0);
266 
267     auto OffsetReg = MIRBuilder.buildConstant(s64, Offset);
268 
269     auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg);
270 
271     MPO = MachinePointerInfo::getStack(MF, Offset);
272     return AddrReg.getReg(0);
273   }
274 
275   /// We need to fixup the reported store size for certain value types because
276   /// we invert the interpretation of ValVT and LocVT in certain cases. This is
277   /// for compatability with the DAG call lowering implementation, which we're
278   /// currently building on top of.
279   LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
280                              ISD::ArgFlagsTy Flags) const override {
281     if (Flags.isPointer())
282       return CallLowering::ValueHandler::getStackValueStoreType(DL, VA, Flags);
283     return getStackValueStoreTypeHack(VA);
284   }
285 
286   void assignValueToReg(Register ValVReg, Register PhysReg,
287                         const CCValAssign &VA) override {
288     MIB.addUse(PhysReg, RegState::Implicit);
289     Register ExtReg = extendRegister(ValVReg, VA);
290     MIRBuilder.buildCopy(PhysReg, ExtReg);
291   }
292 
293   void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
294                             const MachinePointerInfo &MPO,
295                             const CCValAssign &VA) override {
296     MachineFunction &MF = MIRBuilder.getMF();
297     auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy,
298                                        inferAlignFromPtrInfo(MF, MPO));
299     MIRBuilder.buildStore(ValVReg, Addr, *MMO);
300   }
301 
302   void assignValueToAddress(const CallLowering::ArgInfo &Arg, unsigned RegIndex,
303                             Register Addr, LLT MemTy,
304                             const MachinePointerInfo &MPO,
305                             const CCValAssign &VA) override {
306     unsigned MaxSize = MemTy.getSizeInBytes() * 8;
307     // For varargs, we always want to extend them to 8 bytes, in which case
308     // we disable setting a max.
309     if (!Arg.IsFixed)
310       MaxSize = 0;
311 
312     Register ValVReg = Arg.Regs[RegIndex];
313     if (VA.getLocInfo() != CCValAssign::LocInfo::FPExt) {
314       MVT LocVT = VA.getLocVT();
315       MVT ValVT = VA.getValVT();
316 
317       if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16) {
318         std::swap(ValVT, LocVT);
319         MemTy = LLT(VA.getValVT());
320       }
321 
322       ValVReg = extendRegister(ValVReg, VA, MaxSize);
323     } else {
324       // The store does not cover the full allocated stack slot.
325       MemTy = LLT(VA.getValVT());
326     }
327 
328     assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);
329   }
330 
331   MachineInstrBuilder MIB;
332 
333   bool IsTailCall;
334 
335   /// For tail calls, the byte offset of the call's argument area from the
336   /// callee's. Unused elsewhere.
337   int FPDiff;
338 
339   // Cache the SP register vreg if we need it more than once in this call site.
340   Register SPReg;
341 
342   const AArch64Subtarget &Subtarget;
343 };
344 } // namespace
345 
346 static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt) {
347   return (CallConv == CallingConv::Fast && TailCallOpt) ||
348          CallConv == CallingConv::Tail || CallConv == CallingConv::SwiftTail;
349 }
350 
351 bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
352                                       const Value *Val,
353                                       ArrayRef<Register> VRegs,
354                                       FunctionLoweringInfo &FLI,
355                                       Register SwiftErrorVReg) const {
356   auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR);
357   assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&
358          "Return value without a vreg");
359 
360   bool Success = true;
361   if (!FLI.CanLowerReturn) {
362     insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister);
363   } else if (!VRegs.empty()) {
364     MachineFunction &MF = MIRBuilder.getMF();
365     const Function &F = MF.getFunction();
366     const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
367 
368     MachineRegisterInfo &MRI = MF.getRegInfo();
369     const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
370     CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
371     auto &DL = F.getParent()->getDataLayout();
372     LLVMContext &Ctx = Val->getType()->getContext();
373 
374     SmallVector<EVT, 4> SplitEVTs;
375     ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
376     assert(VRegs.size() == SplitEVTs.size() &&
377            "For each split Type there should be exactly one VReg.");
378 
379     SmallVector<ArgInfo, 8> SplitArgs;
380     CallingConv::ID CC = F.getCallingConv();
381 
382     for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
383       Register CurVReg = VRegs[i];
384       ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx), 0};
385       setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F);
386 
387       // i1 is a special case because SDAG i1 true is naturally zero extended
388       // when widened using ANYEXT. We need to do it explicitly here.
389       auto &Flags = CurArgInfo.Flags[0];
390       if (MRI.getType(CurVReg).getSizeInBits() == 1 && !Flags.isSExt() &&
391           !Flags.isZExt()) {
392         CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg).getReg(0);
393       } else if (TLI.getNumRegistersForCallingConv(Ctx, CC, SplitEVTs[i]) ==
394                  1) {
395         // Some types will need extending as specified by the CC.
396         MVT NewVT = TLI.getRegisterTypeForCallingConv(Ctx, CC, SplitEVTs[i]);
397         if (EVT(NewVT) != SplitEVTs[i]) {
398           unsigned ExtendOp = TargetOpcode::G_ANYEXT;
399           if (F.getAttributes().hasRetAttr(Attribute::SExt))
400             ExtendOp = TargetOpcode::G_SEXT;
401           else if (F.getAttributes().hasRetAttr(Attribute::ZExt))
402             ExtendOp = TargetOpcode::G_ZEXT;
403 
404           LLT NewLLT(NewVT);
405           LLT OldLLT(MVT::getVT(CurArgInfo.Ty));
406           CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Ctx);
407           // Instead of an extend, we might have a vector type which needs
408           // padding with more elements, e.g. <2 x half> -> <4 x half>.
409           if (NewVT.isVector()) {
410             if (OldLLT.isVector()) {
411               if (NewLLT.getNumElements() > OldLLT.getNumElements()) {
412 
413                 CurVReg =
414                     MIRBuilder.buildPadVectorWithUndefElements(NewLLT, CurVReg)
415                         .getReg(0);
416               } else {
417                 // Just do a vector extend.
418                 CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
419                               .getReg(0);
420               }
421             } else if (NewLLT.getNumElements() >= 2 &&
422                        NewLLT.getNumElements() <= 8) {
423               // We need to pad a <1 x S> type to <2/4/8 x S>. Since we don't
424               // have <1 x S> vector types in GISel we use a build_vector
425               // instead of a vector merge/concat.
426               CurVReg =
427                   MIRBuilder.buildPadVectorWithUndefElements(NewLLT, CurVReg)
428                       .getReg(0);
429             } else {
430               LLVM_DEBUG(dbgs() << "Could not handle ret ty\n");
431               return false;
432             }
433           } else {
434             // If the split EVT was a <1 x T> vector, and NewVT is T, then we
435             // don't have to do anything since we don't distinguish between the
436             // two.
437             if (NewLLT != MRI.getType(CurVReg)) {
438               // A scalar extend.
439               CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
440                             .getReg(0);
441             }
442           }
443         }
444       }
445       if (CurVReg != CurArgInfo.Regs[0]) {
446         CurArgInfo.Regs[0] = CurVReg;
447         // Reset the arg flags after modifying CurVReg.
448         setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F);
449       }
450       splitToValueTypes(CurArgInfo, SplitArgs, DL, CC);
451     }
452 
453     AArch64OutgoingValueAssigner Assigner(AssignFn, AssignFn, Subtarget,
454                                           /*IsReturn*/ true);
455     OutgoingArgHandler Handler(MIRBuilder, MRI, MIB);
456     Success = determineAndHandleAssignments(Handler, Assigner, SplitArgs,
457                                             MIRBuilder, CC, F.isVarArg());
458   }
459 
460   if (SwiftErrorVReg) {
461     MIB.addUse(AArch64::X21, RegState::Implicit);
462     MIRBuilder.buildCopy(AArch64::X21, SwiftErrorVReg);
463   }
464 
465   MIRBuilder.insertInstr(MIB);
466   return Success;
467 }
468 
469 bool AArch64CallLowering::canLowerReturn(MachineFunction &MF,
470                                          CallingConv::ID CallConv,
471                                          SmallVectorImpl<BaseArgInfo> &Outs,
472                                          bool IsVarArg) const {
473   SmallVector<CCValAssign, 16> ArgLocs;
474   const auto &TLI = *getTLI<AArch64TargetLowering>();
475   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
476                  MF.getFunction().getContext());
477 
478   return checkReturn(CCInfo, Outs, TLI.CCAssignFnForReturn(CallConv));
479 }
480 
481 /// Helper function to compute forwarded registers for musttail calls. Computes
482 /// the forwarded registers, sets MBB liveness, and emits COPY instructions that
483 /// can be used to save + restore registers later.
484 static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
485                                              CCAssignFn *AssignFn) {
486   MachineBasicBlock &MBB = MIRBuilder.getMBB();
487   MachineFunction &MF = MIRBuilder.getMF();
488   MachineFrameInfo &MFI = MF.getFrameInfo();
489 
490   if (!MFI.hasMustTailInVarArgFunc())
491     return;
492 
493   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
494   const Function &F = MF.getFunction();
495   assert(F.isVarArg() && "Expected F to be vararg?");
496 
497   // Compute the set of forwarded registers. The rest are scratch.
498   SmallVector<CCValAssign, 16> ArgLocs;
499   CCState CCInfo(F.getCallingConv(), /*IsVarArg=*/true, MF, ArgLocs,
500                  F.getContext());
501   SmallVector<MVT, 2> RegParmTypes;
502   RegParmTypes.push_back(MVT::i64);
503   RegParmTypes.push_back(MVT::f128);
504 
505   // Later on, we can use this vector to restore the registers if necessary.
506   SmallVectorImpl<ForwardedRegister> &Forwards =
507       FuncInfo->getForwardedMustTailRegParms();
508   CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, AssignFn);
509 
510   // Conservatively forward X8, since it might be used for an aggregate
511   // return.
512   if (!CCInfo.isAllocated(AArch64::X8)) {
513     Register X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
514     Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
515   }
516 
517   // Add the forwards to the MachineBasicBlock and MachineFunction.
518   for (const auto &F : Forwards) {
519     MBB.addLiveIn(F.PReg);
520     MIRBuilder.buildCopy(Register(F.VReg), Register(F.PReg));
521   }
522 }
523 
524 bool AArch64CallLowering::fallBackToDAGISel(const MachineFunction &MF) const {
525   auto &F = MF.getFunction();
526   if (F.getReturnType()->isScalableTy() ||
527       llvm::any_of(F.args(), [](const Argument &A) {
528         return A.getType()->isScalableTy();
529       }))
530     return true;
531   const auto &ST = MF.getSubtarget<AArch64Subtarget>();
532   if (!ST.hasNEON() || !ST.hasFPARMv8()) {
533     LLVM_DEBUG(dbgs() << "Falling back to SDAG because we don't support no-NEON\n");
534     return true;
535   }
536 
537   SMEAttrs Attrs(F);
538   if (Attrs.hasZAState() || Attrs.hasStreamingInterfaceOrBody() ||
539       Attrs.hasStreamingCompatibleInterface())
540     return true;
541 
542   return false;
543 }
544 
545 void AArch64CallLowering::saveVarArgRegisters(
546     MachineIRBuilder &MIRBuilder, CallLowering::IncomingValueHandler &Handler,
547     CCState &CCInfo) const {
548   auto GPRArgRegs = AArch64::getGPRArgRegs();
549   auto FPRArgRegs = AArch64::getFPRArgRegs();
550 
551   MachineFunction &MF = MIRBuilder.getMF();
552   MachineRegisterInfo &MRI = MF.getRegInfo();
553   MachineFrameInfo &MFI = MF.getFrameInfo();
554   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
555   auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
556   bool IsWin64CC =
557       Subtarget.isCallingConvWin64(CCInfo.getCallingConv());
558   const LLT p0 = LLT::pointer(0, 64);
559   const LLT s64 = LLT::scalar(64);
560 
561   unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
562   unsigned NumVariadicGPRArgRegs = GPRArgRegs.size() - FirstVariadicGPR + 1;
563 
564   unsigned GPRSaveSize = 8 * (GPRArgRegs.size() - FirstVariadicGPR);
565   int GPRIdx = 0;
566   if (GPRSaveSize != 0) {
567     if (IsWin64CC) {
568       GPRIdx = MFI.CreateFixedObject(GPRSaveSize,
569                                      -static_cast<int>(GPRSaveSize), false);
570       if (GPRSaveSize & 15)
571         // The extra size here, if triggered, will always be 8.
572         MFI.CreateFixedObject(16 - (GPRSaveSize & 15),
573                               -static_cast<int>(alignTo(GPRSaveSize, 16)),
574                               false);
575     } else
576       GPRIdx = MFI.CreateStackObject(GPRSaveSize, Align(8), false);
577 
578     auto FIN = MIRBuilder.buildFrameIndex(p0, GPRIdx);
579     auto Offset =
580         MIRBuilder.buildConstant(MRI.createGenericVirtualRegister(s64), 8);
581 
582     for (unsigned i = FirstVariadicGPR; i < GPRArgRegs.size(); ++i) {
583       Register Val = MRI.createGenericVirtualRegister(s64);
584       Handler.assignValueToReg(
585           Val, GPRArgRegs[i],
586           CCValAssign::getReg(i + MF.getFunction().getNumOperands(), MVT::i64,
587                               GPRArgRegs[i], MVT::i64, CCValAssign::Full));
588       auto MPO = IsWin64CC ? MachinePointerInfo::getFixedStack(
589                                MF, GPRIdx, (i - FirstVariadicGPR) * 8)
590                          : MachinePointerInfo::getStack(MF, i * 8);
591       MIRBuilder.buildStore(Val, FIN, MPO, inferAlignFromPtrInfo(MF, MPO));
592 
593       FIN = MIRBuilder.buildPtrAdd(MRI.createGenericVirtualRegister(p0),
594                                    FIN.getReg(0), Offset);
595     }
596   }
597   FuncInfo->setVarArgsGPRIndex(GPRIdx);
598   FuncInfo->setVarArgsGPRSize(GPRSaveSize);
599 
600   if (Subtarget.hasFPARMv8() && !IsWin64CC) {
601     unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
602 
603     unsigned FPRSaveSize = 16 * (FPRArgRegs.size() - FirstVariadicFPR);
604     int FPRIdx = 0;
605     if (FPRSaveSize != 0) {
606       FPRIdx = MFI.CreateStackObject(FPRSaveSize, Align(16), false);
607 
608       auto FIN = MIRBuilder.buildFrameIndex(p0, FPRIdx);
609       auto Offset =
610           MIRBuilder.buildConstant(MRI.createGenericVirtualRegister(s64), 16);
611 
612       for (unsigned i = FirstVariadicFPR; i < FPRArgRegs.size(); ++i) {
613         Register Val = MRI.createGenericVirtualRegister(LLT::scalar(128));
614         Handler.assignValueToReg(
615             Val, FPRArgRegs[i],
616             CCValAssign::getReg(
617                 i + MF.getFunction().getNumOperands() + NumVariadicGPRArgRegs,
618                 MVT::f128, FPRArgRegs[i], MVT::f128, CCValAssign::Full));
619 
620         auto MPO = MachinePointerInfo::getStack(MF, i * 16);
621         MIRBuilder.buildStore(Val, FIN, MPO, inferAlignFromPtrInfo(MF, MPO));
622 
623         FIN = MIRBuilder.buildPtrAdd(MRI.createGenericVirtualRegister(p0),
624                                      FIN.getReg(0), Offset);
625       }
626     }
627     FuncInfo->setVarArgsFPRIndex(FPRIdx);
628     FuncInfo->setVarArgsFPRSize(FPRSaveSize);
629   }
630 }
631 
632 bool AArch64CallLowering::lowerFormalArguments(
633     MachineIRBuilder &MIRBuilder, const Function &F,
634     ArrayRef<ArrayRef<Register>> VRegs, FunctionLoweringInfo &FLI) const {
635   MachineFunction &MF = MIRBuilder.getMF();
636   MachineBasicBlock &MBB = MIRBuilder.getMBB();
637   MachineRegisterInfo &MRI = MF.getRegInfo();
638   auto &DL = F.getParent()->getDataLayout();
639   auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
640 
641   // Arm64EC has extra requirements for varargs calls which are only implemented
642   // in SelectionDAG; bail out for now.
643   if (F.isVarArg() && Subtarget.isWindowsArm64EC())
644     return false;
645 
646   // Arm64EC thunks have a special calling convention which is only implemented
647   // in SelectionDAG; bail out for now.
648   if (F.getCallingConv() == CallingConv::ARM64EC_Thunk_Native ||
649       F.getCallingConv() == CallingConv::ARM64EC_Thunk_X64)
650     return false;
651 
652   bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv()) && !Subtarget.isWindowsArm64EC();
653 
654   SmallVector<ArgInfo, 8> SplitArgs;
655   SmallVector<std::pair<Register, Register>> BoolArgs;
656 
657   // Insert the hidden sret parameter if the return value won't fit in the
658   // return registers.
659   if (!FLI.CanLowerReturn)
660     insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL);
661 
662   unsigned i = 0;
663   for (auto &Arg : F.args()) {
664     if (DL.getTypeStoreSize(Arg.getType()).isZero())
665       continue;
666 
667     ArgInfo OrigArg{VRegs[i], Arg, i};
668     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
669 
670     // i1 arguments are zero-extended to i8 by the caller. Emit a
671     // hint to reflect this.
672     if (OrigArg.Ty->isIntegerTy(1)) {
673       assert(OrigArg.Regs.size() == 1 &&
674              MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
675              "Unexpected registers used for i1 arg");
676 
677       auto &Flags = OrigArg.Flags[0];
678       if (!Flags.isZExt() && !Flags.isSExt()) {
679         // Lower i1 argument as i8, and insert AssertZExt + Trunc later.
680         Register OrigReg = OrigArg.Regs[0];
681         Register WideReg = MRI.createGenericVirtualRegister(LLT::scalar(8));
682         OrigArg.Regs[0] = WideReg;
683         BoolArgs.push_back({OrigReg, WideReg});
684       }
685     }
686 
687     if (Arg.hasAttribute(Attribute::SwiftAsync))
688       MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
689 
690     splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv());
691     ++i;
692   }
693 
694   if (!MBB.empty())
695     MIRBuilder.setInstr(*MBB.begin());
696 
697   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
698   CCAssignFn *AssignFn = TLI.CCAssignFnForCall(F.getCallingConv(), IsWin64 && F.isVarArg());
699 
700   AArch64IncomingValueAssigner Assigner(AssignFn, AssignFn);
701   FormalArgHandler Handler(MIRBuilder, MRI);
702   SmallVector<CCValAssign, 16> ArgLocs;
703   CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
704   if (!determineAssignments(Assigner, SplitArgs, CCInfo) ||
705       !handleAssignments(Handler, SplitArgs, CCInfo, ArgLocs, MIRBuilder))
706     return false;
707 
708   if (!BoolArgs.empty()) {
709     for (auto &KV : BoolArgs) {
710       Register OrigReg = KV.first;
711       Register WideReg = KV.second;
712       LLT WideTy = MRI.getType(WideReg);
713       assert(MRI.getType(OrigReg).getScalarSizeInBits() == 1 &&
714              "Unexpected bit size of a bool arg");
715       MIRBuilder.buildTrunc(
716           OrigReg, MIRBuilder.buildAssertZExt(WideTy, WideReg, 1).getReg(0));
717     }
718   }
719 
720   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
721   uint64_t StackSize = Assigner.StackSize;
722   if (F.isVarArg()) {
723     if ((!Subtarget.isTargetDarwin() && !Subtarget.isWindowsArm64EC()) || IsWin64) {
724       // The AAPCS variadic function ABI is identical to the non-variadic
725       // one. As a result there may be more arguments in registers and we should
726       // save them for future reference.
727       // Win64 variadic functions also pass arguments in registers, but all
728       // float arguments are passed in integer registers.
729       saveVarArgRegisters(MIRBuilder, Handler, CCInfo);
730     } else if (Subtarget.isWindowsArm64EC()) {
731       return false;
732     }
733 
734     // We currently pass all varargs at 8-byte alignment, or 4 in ILP32.
735     StackSize = alignTo(Assigner.StackSize, Subtarget.isTargetILP32() ? 4 : 8);
736 
737     auto &MFI = MIRBuilder.getMF().getFrameInfo();
738     FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackSize, true));
739   }
740 
741   if (doesCalleeRestoreStack(F.getCallingConv(),
742                              MF.getTarget().Options.GuaranteedTailCallOpt)) {
743     // We have a non-standard ABI, so why not make full use of the stack that
744     // we're going to pop? It must be aligned to 16 B in any case.
745     StackSize = alignTo(StackSize, 16);
746 
747     // If we're expected to restore the stack (e.g. fastcc), then we'll be
748     // adding a multiple of 16.
749     FuncInfo->setArgumentStackToRestore(StackSize);
750 
751     // Our own callers will guarantee that the space is free by giving an
752     // aligned value to CALLSEQ_START.
753   }
754 
755   // When we tail call, we need to check if the callee's arguments
756   // will fit on the caller's stack. So, whenever we lower formal arguments,
757   // we should keep track of this information, since we might lower a tail call
758   // in this function later.
759   FuncInfo->setBytesInStackArgArea(StackSize);
760 
761   if (Subtarget.hasCustomCallingConv())
762     Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
763 
764   handleMustTailForwardedRegisters(MIRBuilder, AssignFn);
765 
766   // Move back to the end of the basic block.
767   MIRBuilder.setMBB(MBB);
768 
769   return true;
770 }
771 
772 /// Return true if the calling convention is one that we can guarantee TCO for.
773 static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
774   return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
775          CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
776 }
777 
778 /// Return true if we might ever do TCO for calls with this calling convention.
779 static bool mayTailCallThisCC(CallingConv::ID CC) {
780   switch (CC) {
781   case CallingConv::C:
782   case CallingConv::PreserveMost:
783   case CallingConv::PreserveAll:
784   case CallingConv::Swift:
785   case CallingConv::SwiftTail:
786   case CallingConv::Tail:
787   case CallingConv::Fast:
788     return true;
789   default:
790     return false;
791   }
792 }
793 
794 /// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
795 /// CC.
796 static std::pair<CCAssignFn *, CCAssignFn *>
797 getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI) {
798   return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
799 }
800 
801 bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
802     CallLoweringInfo &Info, MachineFunction &MF,
803     SmallVectorImpl<ArgInfo> &InArgs) const {
804   const Function &CallerF = MF.getFunction();
805   CallingConv::ID CalleeCC = Info.CallConv;
806   CallingConv::ID CallerCC = CallerF.getCallingConv();
807 
808   // If the calling conventions match, then everything must be the same.
809   if (CalleeCC == CallerCC)
810     return true;
811 
812   // Check if the caller and callee will handle arguments in the same way.
813   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
814   CCAssignFn *CalleeAssignFnFixed;
815   CCAssignFn *CalleeAssignFnVarArg;
816   std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
817       getAssignFnsForCC(CalleeCC, TLI);
818 
819   CCAssignFn *CallerAssignFnFixed;
820   CCAssignFn *CallerAssignFnVarArg;
821   std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
822       getAssignFnsForCC(CallerCC, TLI);
823 
824   AArch64IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
825                                               CalleeAssignFnVarArg);
826   AArch64IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
827                                               CallerAssignFnVarArg);
828 
829   if (!resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner))
830     return false;
831 
832   // Make sure that the caller and callee preserve all of the same registers.
833   auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
834   const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
835   const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
836   if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv()) {
837     TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
838     TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
839   }
840 
841   return TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved);
842 }
843 
844 bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
845     CallLoweringInfo &Info, MachineFunction &MF,
846     SmallVectorImpl<ArgInfo> &OrigOutArgs) const {
847   // If there are no outgoing arguments, then we are done.
848   if (OrigOutArgs.empty())
849     return true;
850 
851   const Function &CallerF = MF.getFunction();
852   LLVMContext &Ctx = CallerF.getContext();
853   CallingConv::ID CalleeCC = Info.CallConv;
854   CallingConv::ID CallerCC = CallerF.getCallingConv();
855   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
856   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
857 
858   CCAssignFn *AssignFnFixed;
859   CCAssignFn *AssignFnVarArg;
860   std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
861 
862   // We have outgoing arguments. Make sure that we can tail call with them.
863   SmallVector<CCValAssign, 16> OutLocs;
864   CCState OutInfo(CalleeCC, false, MF, OutLocs, Ctx);
865 
866   AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
867                                               Subtarget, /*IsReturn*/ false);
868   // determineAssignments() may modify argument flags, so make a copy.
869   SmallVector<ArgInfo, 8> OutArgs;
870   append_range(OutArgs, OrigOutArgs);
871   if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo)) {
872     LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
873     return false;
874   }
875 
876   // Make sure that they can fit on the caller's stack.
877   const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
878   if (OutInfo.getStackSize() > FuncInfo->getBytesInStackArgArea()) {
879     LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");
880     return false;
881   }
882 
883   // Verify that the parameters in callee-saved registers match.
884   // TODO: Port this over to CallLowering as general code once swiftself is
885   // supported.
886   auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
887   const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
888   MachineRegisterInfo &MRI = MF.getRegInfo();
889 
890   if (Info.IsVarArg) {
891     // Be conservative and disallow variadic memory operands to match SDAG's
892     // behaviour.
893     // FIXME: If the caller's calling convention is C, then we can
894     // potentially use its argument area. However, for cases like fastcc,
895     // we can't do anything.
896     for (unsigned i = 0; i < OutLocs.size(); ++i) {
897       auto &ArgLoc = OutLocs[i];
898       if (ArgLoc.isRegLoc())
899         continue;
900 
901       LLVM_DEBUG(
902           dbgs()
903           << "... Cannot tail call vararg function with stack arguments\n");
904       return false;
905     }
906   }
907 
908   return parametersInCSRMatch(MRI, CallerPreservedMask, OutLocs, OutArgs);
909 }
910 
911 bool AArch64CallLowering::isEligibleForTailCallOptimization(
912     MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
913     SmallVectorImpl<ArgInfo> &InArgs,
914     SmallVectorImpl<ArgInfo> &OutArgs) const {
915 
916   // Must pass all target-independent checks in order to tail call optimize.
917   if (!Info.IsTailCall)
918     return false;
919 
920   CallingConv::ID CalleeCC = Info.CallConv;
921   MachineFunction &MF = MIRBuilder.getMF();
922   const Function &CallerF = MF.getFunction();
923 
924   LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n");
925 
926   if (Info.SwiftErrorVReg) {
927     // TODO: We should handle this.
928     // Note that this is also handled by the check for no outgoing arguments.
929     // Proactively disabling this though, because the swifterror handling in
930     // lowerCall inserts a COPY *after* the location of the call.
931     LLVM_DEBUG(dbgs() << "... Cannot handle tail calls with swifterror yet.\n");
932     return false;
933   }
934 
935   if (!mayTailCallThisCC(CalleeCC)) {
936     LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
937     return false;
938   }
939 
940   // Byval parameters hand the function a pointer directly into the stack area
941   // we want to reuse during a tail call. Working around this *is* possible (see
942   // X86).
943   //
944   // FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
945   // it?
946   //
947   // On Windows, "inreg" attributes signify non-aggregate indirect returns.
948   // In this case, it is necessary to save/restore X0 in the callee. Tail
949   // call opt interferes with this. So we disable tail call opt when the
950   // caller has an argument with "inreg" attribute.
951   //
952   // FIXME: Check whether the callee also has an "inreg" argument.
953   //
954   // When the caller has a swifterror argument, we don't want to tail call
955   // because would have to move into the swifterror register before the
956   // tail call.
957   if (any_of(CallerF.args(), [](const Argument &A) {
958         return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
959       })) {
960     LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "
961                          "inreg, or swifterror arguments\n");
962     return false;
963   }
964 
965   // Externally-defined functions with weak linkage should not be
966   // tail-called on AArch64 when the OS does not support dynamic
967   // pre-emption of symbols, as the AAELF spec requires normal calls
968   // to undefined weak functions to be replaced with a NOP or jump to the
969   // next instruction. The behaviour of branch instructions in this
970   // situation (as used for tail calls) is implementation-defined, so we
971   // cannot rely on the linker replacing the tail call with a return.
972   if (Info.Callee.isGlobal()) {
973     const GlobalValue *GV = Info.Callee.getGlobal();
974     const Triple &TT = MF.getTarget().getTargetTriple();
975     if (GV->hasExternalWeakLinkage() &&
976         (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
977          TT.isOSBinFormatMachO())) {
978       LLVM_DEBUG(dbgs() << "... Cannot tail call externally-defined function "
979                            "with weak linkage for this OS.\n");
980       return false;
981     }
982   }
983 
984   // If we have -tailcallopt, then we're done.
985   if (canGuaranteeTCO(CalleeCC, MF.getTarget().Options.GuaranteedTailCallOpt))
986     return CalleeCC == CallerF.getCallingConv();
987 
988   // We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
989   // Try to find cases where we can do that.
990 
991   // I want anyone implementing a new calling convention to think long and hard
992   // about this assert.
993   assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&
994          "Unexpected variadic calling convention");
995 
996   // Verify that the incoming and outgoing arguments from the callee are
997   // safe to tail call.
998   if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
999     LLVM_DEBUG(
1000         dbgs()
1001         << "... Caller and callee have incompatible calling conventions.\n");
1002     return false;
1003   }
1004 
1005   if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
1006     return false;
1007 
1008   LLVM_DEBUG(
1009       dbgs() << "... Call is eligible for tail call optimization.\n");
1010   return true;
1011 }
1012 
1013 static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
1014                               bool IsTailCall) {
1015   if (!IsTailCall)
1016     return IsIndirect ? getBLRCallOpcode(CallerF) : (unsigned)AArch64::BL;
1017 
1018   if (!IsIndirect)
1019     return AArch64::TCRETURNdi;
1020 
1021   // When BTI is enabled, we need to use TCRETURNriBTI to make sure that we use
1022   // x16 or x17.
1023   if (CallerF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement())
1024     return AArch64::TCRETURNriBTI;
1025 
1026   return AArch64::TCRETURNri;
1027 }
1028 
1029 static const uint32_t *
1030 getMaskForArgs(SmallVectorImpl<AArch64CallLowering::ArgInfo> &OutArgs,
1031                AArch64CallLowering::CallLoweringInfo &Info,
1032                const AArch64RegisterInfo &TRI, MachineFunction &MF) {
1033   const uint32_t *Mask;
1034   if (!OutArgs.empty() && OutArgs[0].Flags[0].isReturned()) {
1035     // For 'this' returns, use the X0-preserving mask if applicable
1036     Mask = TRI.getThisReturnPreservedMask(MF, Info.CallConv);
1037     if (!Mask) {
1038       OutArgs[0].Flags[0].setReturned(false);
1039       Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1040     }
1041   } else {
1042     Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1043   }
1044   return Mask;
1045 }
1046 
1047 bool AArch64CallLowering::lowerTailCall(
1048     MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1049     SmallVectorImpl<ArgInfo> &OutArgs) const {
1050   MachineFunction &MF = MIRBuilder.getMF();
1051   const Function &F = MF.getFunction();
1052   MachineRegisterInfo &MRI = MF.getRegInfo();
1053   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1054   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
1055 
1056   // True when we're tail calling, but without -tailcallopt.
1057   bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt &&
1058                    Info.CallConv != CallingConv::Tail &&
1059                    Info.CallConv != CallingConv::SwiftTail;
1060 
1061   // TODO: Right now, regbankselect doesn't know how to handle the rtcGPR64
1062   // register class. Until we can do that, we should fall back here.
1063   if (MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement()) {
1064     LLVM_DEBUG(
1065         dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n");
1066     return false;
1067   }
1068 
1069   // Find out which ABI gets to decide where things go.
1070   CallingConv::ID CalleeCC = Info.CallConv;
1071   CCAssignFn *AssignFnFixed;
1072   CCAssignFn *AssignFnVarArg;
1073   std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
1074 
1075   MachineInstrBuilder CallSeqStart;
1076   if (!IsSibCall)
1077     CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
1078 
1079   unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true);
1080   auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1081   MIB.add(Info.Callee);
1082 
1083   // Byte offset for the tail call. When we are sibcalling, this will always
1084   // be 0.
1085   MIB.addImm(0);
1086 
1087   // Tell the call which registers are clobbered.
1088   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1089   auto TRI = Subtarget.getRegisterInfo();
1090   const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1091   if (Subtarget.hasCustomCallingConv())
1092     TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1093   MIB.addRegMask(Mask);
1094 
1095   if (Info.CFIType)
1096     MIB->setCFIType(MF, Info.CFIType->getZExtValue());
1097 
1098   if (TRI->isAnyArgRegReserved(MF))
1099     TRI->emitReservedArgRegCallError(MF);
1100 
1101   // FPDiff is the byte offset of the call's argument area from the callee's.
1102   // Stores to callee stack arguments will be placed in FixedStackSlots offset
1103   // by this amount for a tail call. In a sibling call it must be 0 because the
1104   // caller will deallocate the entire stack and the callee still expects its
1105   // arguments to begin at SP+0.
1106   int FPDiff = 0;
1107 
1108   // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1109   // by -tailcallopt. For sibcalls, the memory operands for the call are
1110   // already available in the caller's incoming argument space.
1111   unsigned NumBytes = 0;
1112   if (!IsSibCall) {
1113     // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1114     // before handling assignments, because FPDiff must be known for memory
1115     // arguments.
1116     unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1117     SmallVector<CCValAssign, 16> OutLocs;
1118     CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1119 
1120     AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
1121                                                 Subtarget, /*IsReturn*/ false);
1122     if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo))
1123       return false;
1124 
1125     // The callee will pop the argument stack as a tail call. Thus, we must
1126     // keep it 16-byte aligned.
1127     NumBytes = alignTo(OutInfo.getStackSize(), 16);
1128 
1129     // FPDiff will be negative if this tail call requires more space than we
1130     // would automatically have in our incoming argument space. Positive if we
1131     // actually shrink the stack.
1132     FPDiff = NumReusableBytes - NumBytes;
1133 
1134     // Update the required reserved area if this is the tail call requiring the
1135     // most argument stack space.
1136     if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (unsigned)-FPDiff)
1137       FuncInfo->setTailCallReservedStack(-FPDiff);
1138 
1139     // The stack pointer must be 16-byte aligned at all times it's used for a
1140     // memory operation, which in practice means at *all* times and in
1141     // particular across call boundaries. Therefore our own arguments started at
1142     // a 16-byte aligned SP and the delta applied for the tail call should
1143     // satisfy the same constraint.
1144     assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
1145   }
1146 
1147   const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
1148 
1149   AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1150                                         Subtarget, /*IsReturn*/ false);
1151 
1152   // Do the actual argument marshalling.
1153   OutgoingArgHandler Handler(MIRBuilder, MRI, MIB,
1154                              /*IsTailCall*/ true, FPDiff);
1155   if (!determineAndHandleAssignments(Handler, Assigner, OutArgs, MIRBuilder,
1156                                      CalleeCC, Info.IsVarArg))
1157     return false;
1158 
1159   Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
1160 
1161   if (Info.IsVarArg && Info.IsMustTailCall) {
1162     // Now we know what's being passed to the function. Add uses to the call for
1163     // the forwarded registers that we *aren't* passing as parameters. This will
1164     // preserve the copies we build earlier.
1165     for (const auto &F : Forwards) {
1166       Register ForwardedReg = F.PReg;
1167       // If the register is already passed, or aliases a register which is
1168       // already being passed, then skip it.
1169       if (any_of(MIB->uses(), [&ForwardedReg, &TRI](const MachineOperand &Use) {
1170             if (!Use.isReg())
1171               return false;
1172             return TRI->regsOverlap(Use.getReg(), ForwardedReg);
1173           }))
1174         continue;
1175 
1176       // We aren't passing it already, so we should add it to the call.
1177       MIRBuilder.buildCopy(ForwardedReg, Register(F.VReg));
1178       MIB.addReg(ForwardedReg, RegState::Implicit);
1179     }
1180   }
1181 
1182   // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1183   // sequence start and end here.
1184   if (!IsSibCall) {
1185     MIB->getOperand(1).setImm(FPDiff);
1186     CallSeqStart.addImm(0).addImm(0);
1187     // End the call sequence *before* emitting the call. Normally, we would
1188     // tidy the frame up after the call. However, here, we've laid out the
1189     // parameters so that when SP is reset, they will be in the correct
1190     // location.
1191     MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP).addImm(0).addImm(0);
1192   }
1193 
1194   // Now we can add the actual call instruction to the correct basic block.
1195   MIRBuilder.insertInstr(MIB);
1196 
1197   // If Callee is a reg, since it is used by a target specific instruction,
1198   // it must have a register class matching the constraint of that instruction.
1199   if (MIB->getOperand(0).isReg())
1200     constrainOperandRegClass(MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
1201                              *MF.getSubtarget().getRegBankInfo(), *MIB,
1202                              MIB->getDesc(), MIB->getOperand(0), 0);
1203 
1204   MF.getFrameInfo().setHasTailCall();
1205   Info.LoweredTailCall = true;
1206   return true;
1207 }
1208 
1209 bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
1210                                     CallLoweringInfo &Info) const {
1211   MachineFunction &MF = MIRBuilder.getMF();
1212   const Function &F = MF.getFunction();
1213   MachineRegisterInfo &MRI = MF.getRegInfo();
1214   auto &DL = F.getParent()->getDataLayout();
1215   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1216   const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1217 
1218   // Arm64EC has extra requirements for varargs calls; bail out for now.
1219   //
1220   // Arm64EC has special mangling rules for calls; bail out on all calls for
1221   // now.
1222   if (Subtarget.isWindowsArm64EC())
1223     return false;
1224 
1225   // Arm64EC thunks have a special calling convention which is only implemented
1226   // in SelectionDAG; bail out for now.
1227   if (Info.CallConv == CallingConv::ARM64EC_Thunk_Native ||
1228       Info.CallConv == CallingConv::ARM64EC_Thunk_X64)
1229     return false;
1230 
1231   SmallVector<ArgInfo, 8> OutArgs;
1232   for (auto &OrigArg : Info.OrigArgs) {
1233     splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv);
1234     // AAPCS requires that we zero-extend i1 to 8 bits by the caller.
1235     auto &Flags = OrigArg.Flags[0];
1236     if (OrigArg.Ty->isIntegerTy(1) && !Flags.isSExt() && !Flags.isZExt()) {
1237       ArgInfo &OutArg = OutArgs.back();
1238       assert(OutArg.Regs.size() == 1 &&
1239              MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 &&
1240              "Unexpected registers used for i1 arg");
1241 
1242       // We cannot use a ZExt ArgInfo flag here, because it will
1243       // zero-extend the argument to i32 instead of just i8.
1244       OutArg.Regs[0] =
1245           MIRBuilder.buildZExt(LLT::scalar(8), OutArg.Regs[0]).getReg(0);
1246       LLVMContext &Ctx = MF.getFunction().getContext();
1247       OutArg.Ty = Type::getInt8Ty(Ctx);
1248     }
1249   }
1250 
1251   SmallVector<ArgInfo, 8> InArgs;
1252   if (!Info.OrigRet.Ty->isVoidTy())
1253     splitToValueTypes(Info.OrigRet, InArgs, DL, Info.CallConv);
1254 
1255   // If we can lower as a tail call, do that instead.
1256   bool CanTailCallOpt =
1257       isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
1258 
1259   // We must emit a tail call if we have musttail.
1260   if (Info.IsMustTailCall && !CanTailCallOpt) {
1261     // There are types of incoming/outgoing arguments we can't handle yet, so
1262     // it doesn't make sense to actually die here like in ISelLowering. Instead,
1263     // fall back to SelectionDAG and let it try to handle this.
1264     LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
1265     return false;
1266   }
1267 
1268   Info.IsTailCall = CanTailCallOpt;
1269   if (CanTailCallOpt)
1270     return lowerTailCall(MIRBuilder, Info, OutArgs);
1271 
1272   // Find out which ABI gets to decide where things go.
1273   CCAssignFn *AssignFnFixed;
1274   CCAssignFn *AssignFnVarArg;
1275   std::tie(AssignFnFixed, AssignFnVarArg) =
1276       getAssignFnsForCC(Info.CallConv, TLI);
1277 
1278   MachineInstrBuilder CallSeqStart;
1279   CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
1280 
1281   // Create a temporarily-floating call instruction so we can add the implicit
1282   // uses of arg registers.
1283 
1284   unsigned Opc = 0;
1285   // Calls with operand bundle "clang.arc.attachedcall" are special. They should
1286   // be expanded to the call, directly followed by a special marker sequence and
1287   // a call to an ObjC library function.
1288   if (Info.CB && objcarc::hasAttachedCallOpBundle(Info.CB))
1289     Opc = AArch64::BLR_RVMARKER;
1290   // A call to a returns twice function like setjmp must be followed by a bti
1291   // instruction.
1292   else if (Info.CB && Info.CB->hasFnAttr(Attribute::ReturnsTwice) &&
1293            !Subtarget.noBTIAtReturnTwice() &&
1294            MF.getInfo<AArch64FunctionInfo>()->branchTargetEnforcement())
1295     Opc = AArch64::BLR_BTI;
1296   else
1297     Opc = getCallOpcode(MF, Info.Callee.isReg(), false);
1298 
1299   auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1300   unsigned CalleeOpNo = 0;
1301 
1302   if (Opc == AArch64::BLR_RVMARKER) {
1303     // Add a target global address for the retainRV/claimRV runtime function
1304     // just before the call target.
1305     Function *ARCFn = *objcarc::getAttachedARCFunction(Info.CB);
1306     MIB.addGlobalAddress(ARCFn);
1307     ++CalleeOpNo;
1308   } else if (Info.CFIType) {
1309     MIB->setCFIType(MF, Info.CFIType->getZExtValue());
1310   }
1311 
1312   MIB.add(Info.Callee);
1313 
1314   // Tell the call which registers are clobbered.
1315   const uint32_t *Mask;
1316   const auto *TRI = Subtarget.getRegisterInfo();
1317 
1318   AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1319                                         Subtarget, /*IsReturn*/ false);
1320   // Do the actual argument marshalling.
1321   OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, /*IsReturn*/ false);
1322   if (!determineAndHandleAssignments(Handler, Assigner, OutArgs, MIRBuilder,
1323                                      Info.CallConv, Info.IsVarArg))
1324     return false;
1325 
1326   Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
1327 
1328   if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
1329     TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1330   MIB.addRegMask(Mask);
1331 
1332   if (TRI->isAnyArgRegReserved(MF))
1333     TRI->emitReservedArgRegCallError(MF);
1334 
1335   // Now we can add the actual call instruction to the correct basic block.
1336   MIRBuilder.insertInstr(MIB);
1337 
1338   uint64_t CalleePopBytes =
1339       doesCalleeRestoreStack(Info.CallConv,
1340                              MF.getTarget().Options.GuaranteedTailCallOpt)
1341           ? alignTo(Assigner.StackSize, 16)
1342           : 0;
1343 
1344   CallSeqStart.addImm(Assigner.StackSize).addImm(0);
1345   MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
1346       .addImm(Assigner.StackSize)
1347       .addImm(CalleePopBytes);
1348 
1349   // If Callee is a reg, since it is used by a target specific
1350   // instruction, it must have a register class matching the
1351   // constraint of that instruction.
1352   if (MIB->getOperand(CalleeOpNo).isReg())
1353     constrainOperandRegClass(MF, *TRI, MRI, *Subtarget.getInstrInfo(),
1354                              *Subtarget.getRegBankInfo(), *MIB, MIB->getDesc(),
1355                              MIB->getOperand(CalleeOpNo), CalleeOpNo);
1356 
1357   // Finally we can copy the returned value back into its virtual-register. In
1358   // symmetry with the arguments, the physical register must be an
1359   // implicit-define of the call instruction.
1360   if (Info.CanLowerReturn  && !Info.OrigRet.Ty->isVoidTy()) {
1361     CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(Info.CallConv);
1362     CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1363     bool UsingReturnedArg =
1364         !OutArgs.empty() && OutArgs[0].Flags[0].isReturned();
1365 
1366     AArch64OutgoingValueAssigner Assigner(RetAssignFn, RetAssignFn, Subtarget,
1367                                           /*IsReturn*/ false);
1368     ReturnedArgCallReturnHandler ReturnedArgHandler(MIRBuilder, MRI, MIB);
1369     if (!determineAndHandleAssignments(
1370             UsingReturnedArg ? ReturnedArgHandler : Handler, Assigner, InArgs,
1371             MIRBuilder, Info.CallConv, Info.IsVarArg,
1372             UsingReturnedArg ? ArrayRef(OutArgs[0].Regs) : std::nullopt))
1373       return false;
1374   }
1375 
1376   if (Info.SwiftErrorVReg) {
1377     MIB.addDef(AArch64::X21, RegState::Implicit);
1378     MIRBuilder.buildCopy(Info.SwiftErrorVReg, Register(AArch64::X21));
1379   }
1380 
1381   if (!Info.CanLowerReturn) {
1382     insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs,
1383                     Info.DemoteRegister, Info.DemoteStackIndex);
1384   }
1385   return true;
1386 }
1387 
1388 bool AArch64CallLowering::isTypeIsValidForThisReturn(EVT Ty) const {
1389   return Ty.getSizeInBits() == 64;
1390 }
1391