xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGCall.cpp (revision cb14a3fe5122c879eae1fb480ed7ce82a699ddb6)
1 //===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // These classes wrap the information about a call or function
10 // definition used to handle ABI compliancy.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCall.h"
15 #include "ABIInfo.h"
16 #include "ABIInfoImpl.h"
17 #include "CGBlocks.h"
18 #include "CGCXXABI.h"
19 #include "CGCleanup.h"
20 #include "CGRecordLayout.h"
21 #include "CodeGenFunction.h"
22 #include "CodeGenModule.h"
23 #include "TargetInfo.h"
24 #include "clang/AST/Attr.h"
25 #include "clang/AST/Decl.h"
26 #include "clang/AST/DeclCXX.h"
27 #include "clang/AST/DeclObjC.h"
28 #include "clang/Basic/CodeGenOptions.h"
29 #include "clang/Basic/TargetInfo.h"
30 #include "clang/CodeGen/CGFunctionInfo.h"
31 #include "clang/CodeGen/SwiftCallingConv.h"
32 #include "llvm/ADT/StringExtras.h"
33 #include "llvm/Analysis/ValueTracking.h"
34 #include "llvm/IR/Assumptions.h"
35 #include "llvm/IR/AttributeMask.h"
36 #include "llvm/IR/Attributes.h"
37 #include "llvm/IR/CallingConv.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/IR/InlineAsm.h"
40 #include "llvm/IR/IntrinsicInst.h"
41 #include "llvm/IR/Intrinsics.h"
42 #include "llvm/IR/Type.h"
43 #include "llvm/Transforms/Utils/Local.h"
44 #include <optional>
45 using namespace clang;
46 using namespace CodeGen;
47 
48 /***/
49 
50 unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
51   switch (CC) {
52   default: return llvm::CallingConv::C;
53   case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
54   case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
55   case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
56   case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
57   case CC_Win64: return llvm::CallingConv::Win64;
58   case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
59   case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
60   case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
61   case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
62   // TODO: Add support for __pascal to LLVM.
63   case CC_X86Pascal: return llvm::CallingConv::C;
64   // TODO: Add support for __vectorcall to LLVM.
65   case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
66   case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
67   case CC_AArch64SVEPCS: return llvm::CallingConv::AArch64_SVE_VectorCall;
68   case CC_AMDGPUKernelCall: return llvm::CallingConv::AMDGPU_KERNEL;
69   case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
70   case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
71   case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
72   case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
73   case CC_Swift: return llvm::CallingConv::Swift;
74   case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
75   case CC_M68kRTD: return llvm::CallingConv::M68k_RTD;
76   }
77 }
78 
79 /// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
80 /// qualification. Either or both of RD and MD may be null. A null RD indicates
81 /// that there is no meaningful 'this' type, and a null MD can occur when
82 /// calling a method pointer.
83 CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
84                                          const CXXMethodDecl *MD) {
85   QualType RecTy;
86   if (RD)
87     RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
88   else
89     RecTy = Context.VoidTy;
90 
91   if (MD)
92     RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
93   return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
94 }
95 
96 /// Returns the canonical formal type of the given C++ method.
97 static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
98   return MD->getType()->getCanonicalTypeUnqualified()
99            .getAs<FunctionProtoType>();
100 }
101 
102 /// Returns the "extra-canonicalized" return type, which discards
103 /// qualifiers on the return type.  Codegen doesn't care about them,
104 /// and it makes ABI code a little easier to be able to assume that
105 /// all parameter and return types are top-level unqualified.
106 static CanQualType GetReturnType(QualType RetTy) {
107   return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
108 }
109 
110 /// Arrange the argument and result information for a value of the given
111 /// unprototyped freestanding function type.
112 const CGFunctionInfo &
113 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
114   // When translating an unprototyped function type, always use a
115   // variadic type.
116   return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
117                                  FnInfoOpts::None, std::nullopt,
118                                  FTNP->getExtInfo(), {}, RequiredArgs(0));
119 }
120 
121 static void addExtParameterInfosForCall(
122          llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
123                                         const FunctionProtoType *proto,
124                                         unsigned prefixArgs,
125                                         unsigned totalArgs) {
126   assert(proto->hasExtParameterInfos());
127   assert(paramInfos.size() <= prefixArgs);
128   assert(proto->getNumParams() + prefixArgs <= totalArgs);
129 
130   paramInfos.reserve(totalArgs);
131 
132   // Add default infos for any prefix args that don't already have infos.
133   paramInfos.resize(prefixArgs);
134 
135   // Add infos for the prototype.
136   for (const auto &ParamInfo : proto->getExtParameterInfos()) {
137     paramInfos.push_back(ParamInfo);
138     // pass_object_size params have no parameter info.
139     if (ParamInfo.hasPassObjectSize())
140       paramInfos.emplace_back();
141   }
142 
143   assert(paramInfos.size() <= totalArgs &&
144          "Did we forget to insert pass_object_size args?");
145   // Add default infos for the variadic and/or suffix arguments.
146   paramInfos.resize(totalArgs);
147 }
148 
149 /// Adds the formal parameters in FPT to the given prefix. If any parameter in
150 /// FPT has pass_object_size attrs, then we'll add parameters for those, too.
151 static void appendParameterTypes(const CodeGenTypes &CGT,
152                                  SmallVectorImpl<CanQualType> &prefix,
153               SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
154                                  CanQual<FunctionProtoType> FPT) {
155   // Fast path: don't touch param info if we don't need to.
156   if (!FPT->hasExtParameterInfos()) {
157     assert(paramInfos.empty() &&
158            "We have paramInfos, but the prototype doesn't?");
159     prefix.append(FPT->param_type_begin(), FPT->param_type_end());
160     return;
161   }
162 
163   unsigned PrefixSize = prefix.size();
164   // In the vast majority of cases, we'll have precisely FPT->getNumParams()
165   // parameters; the only thing that can change this is the presence of
166   // pass_object_size. So, we preallocate for the common case.
167   prefix.reserve(prefix.size() + FPT->getNumParams());
168 
169   auto ExtInfos = FPT->getExtParameterInfos();
170   assert(ExtInfos.size() == FPT->getNumParams());
171   for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
172     prefix.push_back(FPT->getParamType(I));
173     if (ExtInfos[I].hasPassObjectSize())
174       prefix.push_back(CGT.getContext().getSizeType());
175   }
176 
177   addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
178                               prefix.size());
179 }
180 
181 /// Arrange the LLVM function layout for a value of the given function
182 /// type, on top of any implicit parameters already stored.
183 static const CGFunctionInfo &
184 arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
185                         SmallVectorImpl<CanQualType> &prefix,
186                         CanQual<FunctionProtoType> FTP) {
187   SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
188   RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
189   // FIXME: Kill copy.
190   appendParameterTypes(CGT, prefix, paramInfos, FTP);
191   CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
192 
193   FnInfoOpts opts =
194       instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;
195   return CGT.arrangeLLVMFunctionInfo(resultType, opts, prefix,
196                                      FTP->getExtInfo(), paramInfos, Required);
197 }
198 
199 /// Arrange the argument and result information for a value of the
200 /// given freestanding function type.
201 const CGFunctionInfo &
202 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
203   SmallVector<CanQualType, 16> argTypes;
204   return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
205                                    FTP);
206 }
207 
208 static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
209                                                bool IsWindows) {
210   // Set the appropriate calling convention for the Function.
211   if (D->hasAttr<StdCallAttr>())
212     return CC_X86StdCall;
213 
214   if (D->hasAttr<FastCallAttr>())
215     return CC_X86FastCall;
216 
217   if (D->hasAttr<RegCallAttr>())
218     return CC_X86RegCall;
219 
220   if (D->hasAttr<ThisCallAttr>())
221     return CC_X86ThisCall;
222 
223   if (D->hasAttr<VectorCallAttr>())
224     return CC_X86VectorCall;
225 
226   if (D->hasAttr<PascalAttr>())
227     return CC_X86Pascal;
228 
229   if (PcsAttr *PCS = D->getAttr<PcsAttr>())
230     return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
231 
232   if (D->hasAttr<AArch64VectorPcsAttr>())
233     return CC_AArch64VectorCall;
234 
235   if (D->hasAttr<AArch64SVEPcsAttr>())
236     return CC_AArch64SVEPCS;
237 
238   if (D->hasAttr<AMDGPUKernelCallAttr>())
239     return CC_AMDGPUKernelCall;
240 
241   if (D->hasAttr<IntelOclBiccAttr>())
242     return CC_IntelOclBicc;
243 
244   if (D->hasAttr<MSABIAttr>())
245     return IsWindows ? CC_C : CC_Win64;
246 
247   if (D->hasAttr<SysVABIAttr>())
248     return IsWindows ? CC_X86_64SysV : CC_C;
249 
250   if (D->hasAttr<PreserveMostAttr>())
251     return CC_PreserveMost;
252 
253   if (D->hasAttr<PreserveAllAttr>())
254     return CC_PreserveAll;
255 
256   if (D->hasAttr<M68kRTDAttr>())
257     return CC_M68kRTD;
258 
259   return CC_C;
260 }
261 
262 /// Arrange the argument and result information for a call to an
263 /// unknown C++ non-static member function of the given abstract type.
264 /// (A null RD means we don't have any meaningful "this" argument type,
265 ///  so fall back to a generic pointer type).
266 /// The member function must be an ordinary function, i.e. not a
267 /// constructor or destructor.
268 const CGFunctionInfo &
269 CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
270                                    const FunctionProtoType *FTP,
271                                    const CXXMethodDecl *MD) {
272   SmallVector<CanQualType, 16> argTypes;
273 
274   // Add the 'this' pointer.
275   argTypes.push_back(DeriveThisType(RD, MD));
276 
277   return ::arrangeLLVMFunctionInfo(
278       *this, /*instanceMethod=*/true, argTypes,
279       FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
280 }
281 
282 /// Set calling convention for CUDA/HIP kernel.
283 static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
284                                            const FunctionDecl *FD) {
285   if (FD->hasAttr<CUDAGlobalAttr>()) {
286     const FunctionType *FT = FTy->getAs<FunctionType>();
287     CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
288     FTy = FT->getCanonicalTypeUnqualified();
289   }
290 }
291 
292 /// Arrange the argument and result information for a declaration or
293 /// definition of the given C++ non-static member function.  The
294 /// member function must be an ordinary function, i.e. not a
295 /// constructor or destructor.
296 const CGFunctionInfo &
297 CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
298   assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
299   assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
300 
301   CanQualType FT = GetFormalType(MD).getAs<Type>();
302   setCUDAKernelCallingConvention(FT, CGM, MD);
303   auto prototype = FT.getAs<FunctionProtoType>();
304 
305   if (MD->isImplicitObjectMemberFunction()) {
306     // The abstract case is perfectly fine.
307     const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
308     return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
309   }
310 
311   return arrangeFreeFunctionType(prototype);
312 }
313 
314 bool CodeGenTypes::inheritingCtorHasParams(
315     const InheritedConstructor &Inherited, CXXCtorType Type) {
316   // Parameters are unnecessary if we're constructing a base class subobject
317   // and the inherited constructor lives in a virtual base.
318   return Type == Ctor_Complete ||
319          !Inherited.getShadowDecl()->constructsVirtualBase() ||
320          !Target.getCXXABI().hasConstructorVariants();
321 }
322 
323 const CGFunctionInfo &
324 CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
325   auto *MD = cast<CXXMethodDecl>(GD.getDecl());
326 
327   SmallVector<CanQualType, 16> argTypes;
328   SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
329 
330   const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(GD);
331   argTypes.push_back(DeriveThisType(ThisType, MD));
332 
333   bool PassParams = true;
334 
335   if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
336     // A base class inheriting constructor doesn't get forwarded arguments
337     // needed to construct a virtual base (or base class thereof).
338     if (auto Inherited = CD->getInheritedConstructor())
339       PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
340   }
341 
342   CanQual<FunctionProtoType> FTP = GetFormalType(MD);
343 
344   // Add the formal parameters.
345   if (PassParams)
346     appendParameterTypes(*this, argTypes, paramInfos, FTP);
347 
348   CGCXXABI::AddedStructorArgCounts AddedArgs =
349       TheCXXABI.buildStructorSignature(GD, argTypes);
350   if (!paramInfos.empty()) {
351     // Note: prefix implies after the first param.
352     if (AddedArgs.Prefix)
353       paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
354                         FunctionProtoType::ExtParameterInfo{});
355     if (AddedArgs.Suffix)
356       paramInfos.append(AddedArgs.Suffix,
357                         FunctionProtoType::ExtParameterInfo{});
358   }
359 
360   RequiredArgs required =
361       (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
362                                       : RequiredArgs::All);
363 
364   FunctionType::ExtInfo extInfo = FTP->getExtInfo();
365   CanQualType resultType = TheCXXABI.HasThisReturn(GD)
366                                ? argTypes.front()
367                                : TheCXXABI.hasMostDerivedReturn(GD)
368                                      ? CGM.getContext().VoidPtrTy
369                                      : Context.VoidTy;
370   return arrangeLLVMFunctionInfo(resultType, FnInfoOpts::IsInstanceMethod,
371                                  argTypes, extInfo, paramInfos, required);
372 }
373 
374 static SmallVector<CanQualType, 16>
375 getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
376   SmallVector<CanQualType, 16> argTypes;
377   for (auto &arg : args)
378     argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
379   return argTypes;
380 }
381 
382 static SmallVector<CanQualType, 16>
383 getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
384   SmallVector<CanQualType, 16> argTypes;
385   for (auto &arg : args)
386     argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
387   return argTypes;
388 }
389 
390 static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
391 getExtParameterInfosForCall(const FunctionProtoType *proto,
392                             unsigned prefixArgs, unsigned totalArgs) {
393   llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
394   if (proto->hasExtParameterInfos()) {
395     addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
396   }
397   return result;
398 }
399 
400 /// Arrange a call to a C++ method, passing the given arguments.
401 ///
402 /// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
403 /// parameter.
404 /// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
405 /// args.
406 /// PassProtoArgs indicates whether `args` has args for the parameters in the
407 /// given CXXConstructorDecl.
408 const CGFunctionInfo &
409 CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
410                                         const CXXConstructorDecl *D,
411                                         CXXCtorType CtorKind,
412                                         unsigned ExtraPrefixArgs,
413                                         unsigned ExtraSuffixArgs,
414                                         bool PassProtoArgs) {
415   // FIXME: Kill copy.
416   SmallVector<CanQualType, 16> ArgTypes;
417   for (const auto &Arg : args)
418     ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
419 
420   // +1 for implicit this, which should always be args[0].
421   unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
422 
423   CanQual<FunctionProtoType> FPT = GetFormalType(D);
424   RequiredArgs Required = PassProtoArgs
425                               ? RequiredArgs::forPrototypePlus(
426                                     FPT, TotalPrefixArgs + ExtraSuffixArgs)
427                               : RequiredArgs::All;
428 
429   GlobalDecl GD(D, CtorKind);
430   CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
431                                ? ArgTypes.front()
432                                : TheCXXABI.hasMostDerivedReturn(GD)
433                                      ? CGM.getContext().VoidPtrTy
434                                      : Context.VoidTy;
435 
436   FunctionType::ExtInfo Info = FPT->getExtInfo();
437   llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
438   // If the prototype args are elided, we should only have ABI-specific args,
439   // which never have param info.
440   if (PassProtoArgs && FPT->hasExtParameterInfos()) {
441     // ABI-specific suffix arguments are treated the same as variadic arguments.
442     addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
443                                 ArgTypes.size());
444   }
445 
446   return arrangeLLVMFunctionInfo(ResultType, FnInfoOpts::IsInstanceMethod,
447                                  ArgTypes, Info, ParamInfos, Required);
448 }
449 
450 /// Arrange the argument and result information for the declaration or
451 /// definition of the given function.
452 const CGFunctionInfo &
453 CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
454   if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
455     if (MD->isImplicitObjectMemberFunction())
456       return arrangeCXXMethodDeclaration(MD);
457 
458   CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
459 
460   assert(isa<FunctionType>(FTy));
461   setCUDAKernelCallingConvention(FTy, CGM, FD);
462 
463   // When declaring a function without a prototype, always use a
464   // non-variadic type.
465   if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
466     return arrangeLLVMFunctionInfo(noProto->getReturnType(), FnInfoOpts::None,
467                                    std::nullopt, noProto->getExtInfo(), {},
468                                    RequiredArgs::All);
469   }
470 
471   return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
472 }
473 
474 /// Arrange the argument and result information for the declaration or
475 /// definition of an Objective-C method.
476 const CGFunctionInfo &
477 CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
478   // It happens that this is the same as a call with no optional
479   // arguments, except also using the formal 'self' type.
480   return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
481 }
482 
483 /// Arrange the argument and result information for the function type
484 /// through which to perform a send to the given Objective-C method,
485 /// using the given receiver type.  The receiver type is not always
486 /// the 'self' type of the method or even an Objective-C pointer type.
487 /// This is *not* the right method for actually performing such a
488 /// message send, due to the possibility of optional arguments.
489 const CGFunctionInfo &
490 CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
491                                               QualType receiverType) {
492   SmallVector<CanQualType, 16> argTys;
493   SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(
494       MD->isDirectMethod() ? 1 : 2);
495   argTys.push_back(Context.getCanonicalParamType(receiverType));
496   if (!MD->isDirectMethod())
497     argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
498   // FIXME: Kill copy?
499   for (const auto *I : MD->parameters()) {
500     argTys.push_back(Context.getCanonicalParamType(I->getType()));
501     auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
502         I->hasAttr<NoEscapeAttr>());
503     extParamInfos.push_back(extParamInfo);
504   }
505 
506   FunctionType::ExtInfo einfo;
507   bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
508   einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
509 
510   if (getContext().getLangOpts().ObjCAutoRefCount &&
511       MD->hasAttr<NSReturnsRetainedAttr>())
512     einfo = einfo.withProducesResult(true);
513 
514   RequiredArgs required =
515     (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
516 
517   return arrangeLLVMFunctionInfo(GetReturnType(MD->getReturnType()),
518                                  FnInfoOpts::None, argTys, einfo, extParamInfos,
519                                  required);
520 }
521 
522 const CGFunctionInfo &
523 CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
524                                                  const CallArgList &args) {
525   auto argTypes = getArgTypesForCall(Context, args);
526   FunctionType::ExtInfo einfo;
527 
528   return arrangeLLVMFunctionInfo(GetReturnType(returnType), FnInfoOpts::None,
529                                  argTypes, einfo, {}, RequiredArgs::All);
530 }
531 
532 const CGFunctionInfo &
533 CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
534   // FIXME: Do we need to handle ObjCMethodDecl?
535   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
536 
537   if (isa<CXXConstructorDecl>(GD.getDecl()) ||
538       isa<CXXDestructorDecl>(GD.getDecl()))
539     return arrangeCXXStructorDeclaration(GD);
540 
541   return arrangeFunctionDeclaration(FD);
542 }
543 
544 /// Arrange a thunk that takes 'this' as the first parameter followed by
545 /// varargs.  Return a void pointer, regardless of the actual return type.
546 /// The body of the thunk will end in a musttail call to a function of the
547 /// correct type, and the caller will bitcast the function to the correct
548 /// prototype.
549 const CGFunctionInfo &
550 CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
551   assert(MD->isVirtual() && "only methods have thunks");
552   CanQual<FunctionProtoType> FTP = GetFormalType(MD);
553   CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
554   return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::None, ArgTys,
555                                  FTP->getExtInfo(), {}, RequiredArgs(1));
556 }
557 
558 const CGFunctionInfo &
559 CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
560                                    CXXCtorType CT) {
561   assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
562 
563   CanQual<FunctionProtoType> FTP = GetFormalType(CD);
564   SmallVector<CanQualType, 2> ArgTys;
565   const CXXRecordDecl *RD = CD->getParent();
566   ArgTys.push_back(DeriveThisType(RD, CD));
567   if (CT == Ctor_CopyingClosure)
568     ArgTys.push_back(*FTP->param_type_begin());
569   if (RD->getNumVBases() > 0)
570     ArgTys.push_back(Context.IntTy);
571   CallingConv CC = Context.getDefaultCallingConvention(
572       /*IsVariadic=*/false, /*IsCXXMethod=*/true);
573   return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::IsInstanceMethod,
574                                  ArgTys, FunctionType::ExtInfo(CC), {},
575                                  RequiredArgs::All);
576 }
577 
578 /// Arrange a call as unto a free function, except possibly with an
579 /// additional number of formal parameters considered required.
580 static const CGFunctionInfo &
581 arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
582                             CodeGenModule &CGM,
583                             const CallArgList &args,
584                             const FunctionType *fnType,
585                             unsigned numExtraRequiredArgs,
586                             bool chainCall) {
587   assert(args.size() >= numExtraRequiredArgs);
588 
589   llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
590 
591   // In most cases, there are no optional arguments.
592   RequiredArgs required = RequiredArgs::All;
593 
594   // If we have a variadic prototype, the required arguments are the
595   // extra prefix plus the arguments in the prototype.
596   if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
597     if (proto->isVariadic())
598       required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
599 
600     if (proto->hasExtParameterInfos())
601       addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
602                                   args.size());
603 
604   // If we don't have a prototype at all, but we're supposed to
605   // explicitly use the variadic convention for unprototyped calls,
606   // treat all of the arguments as required but preserve the nominal
607   // possibility of variadics.
608   } else if (CGM.getTargetCodeGenInfo()
609                 .isNoProtoCallVariadic(args,
610                                        cast<FunctionNoProtoType>(fnType))) {
611     required = RequiredArgs(args.size());
612   }
613 
614   // FIXME: Kill copy.
615   SmallVector<CanQualType, 16> argTypes;
616   for (const auto &arg : args)
617     argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
618   FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;
619   return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
620                                      opts, argTypes, fnType->getExtInfo(),
621                                      paramInfos, required);
622 }
623 
624 /// Figure out the rules for calling a function with the given formal
625 /// type using the given arguments.  The arguments are necessary
626 /// because the function might be unprototyped, in which case it's
627 /// target-dependent in crazy ways.
628 const CGFunctionInfo &
629 CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
630                                       const FunctionType *fnType,
631                                       bool chainCall) {
632   return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
633                                      chainCall ? 1 : 0, chainCall);
634 }
635 
636 /// A block function is essentially a free function with an
637 /// extra implicit argument.
638 const CGFunctionInfo &
639 CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
640                                        const FunctionType *fnType) {
641   return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
642                                      /*chainCall=*/false);
643 }
644 
645 const CGFunctionInfo &
646 CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
647                                               const FunctionArgList &params) {
648   auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
649   auto argTypes = getArgTypesForDeclaration(Context, params);
650 
651   return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
652                                  FnInfoOpts::None, argTypes,
653                                  proto->getExtInfo(), paramInfos,
654                                  RequiredArgs::forPrototypePlus(proto, 1));
655 }
656 
657 const CGFunctionInfo &
658 CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
659                                          const CallArgList &args) {
660   // FIXME: Kill copy.
661   SmallVector<CanQualType, 16> argTypes;
662   for (const auto &Arg : args)
663     argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
664   return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,
665                                  argTypes, FunctionType::ExtInfo(),
666                                  /*paramInfos=*/{}, RequiredArgs::All);
667 }
668 
669 const CGFunctionInfo &
670 CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
671                                                 const FunctionArgList &args) {
672   auto argTypes = getArgTypesForDeclaration(Context, args);
673 
674   return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,
675                                  argTypes, FunctionType::ExtInfo(), {},
676                                  RequiredArgs::All);
677 }
678 
679 const CGFunctionInfo &
680 CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
681                                               ArrayRef<CanQualType> argTypes) {
682   return arrangeLLVMFunctionInfo(resultType, FnInfoOpts::None, argTypes,
683                                  FunctionType::ExtInfo(), {},
684                                  RequiredArgs::All);
685 }
686 
687 /// Arrange a call to a C++ method, passing the given arguments.
688 ///
689 /// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
690 /// does not count `this`.
691 const CGFunctionInfo &
692 CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
693                                    const FunctionProtoType *proto,
694                                    RequiredArgs required,
695                                    unsigned numPrefixArgs) {
696   assert(numPrefixArgs + 1 <= args.size() &&
697          "Emitting a call with less args than the required prefix?");
698   // Add one to account for `this`. It's a bit awkward here, but we don't count
699   // `this` in similar places elsewhere.
700   auto paramInfos =
701     getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
702 
703   // FIXME: Kill copy.
704   auto argTypes = getArgTypesForCall(Context, args);
705 
706   FunctionType::ExtInfo info = proto->getExtInfo();
707   return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
708                                  FnInfoOpts::IsInstanceMethod, argTypes, info,
709                                  paramInfos, required);
710 }
711 
712 const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
713   return arrangeLLVMFunctionInfo(getContext().VoidTy, FnInfoOpts::None,
714                                  std::nullopt, FunctionType::ExtInfo(), {},
715                                  RequiredArgs::All);
716 }
717 
718 const CGFunctionInfo &
719 CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
720                           const CallArgList &args) {
721   assert(signature.arg_size() <= args.size());
722   if (signature.arg_size() == args.size())
723     return signature;
724 
725   SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
726   auto sigParamInfos = signature.getExtParameterInfos();
727   if (!sigParamInfos.empty()) {
728     paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
729     paramInfos.resize(args.size());
730   }
731 
732   auto argTypes = getArgTypesForCall(Context, args);
733 
734   assert(signature.getRequiredArgs().allowsOptionalArgs());
735   FnInfoOpts opts = FnInfoOpts::None;
736   if (signature.isInstanceMethod())
737     opts |= FnInfoOpts::IsInstanceMethod;
738   if (signature.isChainCall())
739     opts |= FnInfoOpts::IsChainCall;
740   if (signature.isDelegateCall())
741     opts |= FnInfoOpts::IsDelegateCall;
742   return arrangeLLVMFunctionInfo(signature.getReturnType(), opts, argTypes,
743                                  signature.getExtInfo(), paramInfos,
744                                  signature.getRequiredArgs());
745 }
746 
747 namespace clang {
748 namespace CodeGen {
749 void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
750 }
751 }
752 
753 /// Arrange the argument and result information for an abstract value
754 /// of a given function type.  This is the method which all of the
755 /// above functions ultimately defer to.
756 const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(
757     CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,
758     FunctionType::ExtInfo info,
759     ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
760     RequiredArgs required) {
761   assert(llvm::all_of(argTypes,
762                       [](CanQualType T) { return T.isCanonicalAsParam(); }));
763 
764   // Lookup or create unique function info.
765   llvm::FoldingSetNodeID ID;
766   bool isInstanceMethod =
767       (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;
768   bool isChainCall =
769       (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;
770   bool isDelegateCall =
771       (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;
772   CGFunctionInfo::Profile(ID, isInstanceMethod, isChainCall, isDelegateCall,
773                           info, paramInfos, required, resultType, argTypes);
774 
775   void *insertPos = nullptr;
776   CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
777   if (FI)
778     return *FI;
779 
780   unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
781 
782   // Construct the function info.  We co-allocate the ArgInfos.
783   FI = CGFunctionInfo::create(CC, isInstanceMethod, isChainCall, isDelegateCall,
784                               info, paramInfos, resultType, argTypes, required);
785   FunctionInfos.InsertNode(FI, insertPos);
786 
787   bool inserted = FunctionsBeingProcessed.insert(FI).second;
788   (void)inserted;
789   assert(inserted && "Recursively being processed?");
790 
791   // Compute ABI information.
792   if (CC == llvm::CallingConv::SPIR_KERNEL) {
793     // Force target independent argument handling for the host visible
794     // kernel functions.
795     computeSPIRKernelABIInfo(CGM, *FI);
796   } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {
797     swiftcall::computeABIInfo(CGM, *FI);
798   } else {
799     getABIInfo().computeInfo(*FI);
800   }
801 
802   // Loop over all of the computed argument and return value info.  If any of
803   // them are direct or extend without a specified coerce type, specify the
804   // default now.
805   ABIArgInfo &retInfo = FI->getReturnInfo();
806   if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
807     retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
808 
809   for (auto &I : FI->arguments())
810     if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
811       I.info.setCoerceToType(ConvertType(I.type));
812 
813   bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
814   assert(erased && "Not in set?");
815 
816   return *FI;
817 }
818 
819 CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,
820                                        bool chainCall, bool delegateCall,
821                                        const FunctionType::ExtInfo &info,
822                                        ArrayRef<ExtParameterInfo> paramInfos,
823                                        CanQualType resultType,
824                                        ArrayRef<CanQualType> argTypes,
825                                        RequiredArgs required) {
826   assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
827   assert(!required.allowsOptionalArgs() ||
828          required.getNumRequiredArgs() <= argTypes.size());
829 
830   void *buffer =
831     operator new(totalSizeToAlloc<ArgInfo,             ExtParameterInfo>(
832                                   argTypes.size() + 1, paramInfos.size()));
833 
834   CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
835   FI->CallingConvention = llvmCC;
836   FI->EffectiveCallingConvention = llvmCC;
837   FI->ASTCallingConvention = info.getCC();
838   FI->InstanceMethod = instanceMethod;
839   FI->ChainCall = chainCall;
840   FI->DelegateCall = delegateCall;
841   FI->CmseNSCall = info.getCmseNSCall();
842   FI->NoReturn = info.getNoReturn();
843   FI->ReturnsRetained = info.getProducesResult();
844   FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
845   FI->NoCfCheck = info.getNoCfCheck();
846   FI->Required = required;
847   FI->HasRegParm = info.getHasRegParm();
848   FI->RegParm = info.getRegParm();
849   FI->ArgStruct = nullptr;
850   FI->ArgStructAlign = 0;
851   FI->NumArgs = argTypes.size();
852   FI->HasExtParameterInfos = !paramInfos.empty();
853   FI->getArgsBuffer()[0].type = resultType;
854   FI->MaxVectorWidth = 0;
855   for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
856     FI->getArgsBuffer()[i + 1].type = argTypes[i];
857   for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
858     FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
859   return FI;
860 }
861 
862 /***/
863 
864 namespace {
865 // ABIArgInfo::Expand implementation.
866 
867 // Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
868 struct TypeExpansion {
869   enum TypeExpansionKind {
870     // Elements of constant arrays are expanded recursively.
871     TEK_ConstantArray,
872     // Record fields are expanded recursively (but if record is a union, only
873     // the field with the largest size is expanded).
874     TEK_Record,
875     // For complex types, real and imaginary parts are expanded recursively.
876     TEK_Complex,
877     // All other types are not expandable.
878     TEK_None
879   };
880 
881   const TypeExpansionKind Kind;
882 
883   TypeExpansion(TypeExpansionKind K) : Kind(K) {}
884   virtual ~TypeExpansion() {}
885 };
886 
887 struct ConstantArrayExpansion : TypeExpansion {
888   QualType EltTy;
889   uint64_t NumElts;
890 
891   ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
892       : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
893   static bool classof(const TypeExpansion *TE) {
894     return TE->Kind == TEK_ConstantArray;
895   }
896 };
897 
898 struct RecordExpansion : TypeExpansion {
899   SmallVector<const CXXBaseSpecifier *, 1> Bases;
900 
901   SmallVector<const FieldDecl *, 1> Fields;
902 
903   RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
904                   SmallVector<const FieldDecl *, 1> &&Fields)
905       : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
906         Fields(std::move(Fields)) {}
907   static bool classof(const TypeExpansion *TE) {
908     return TE->Kind == TEK_Record;
909   }
910 };
911 
912 struct ComplexExpansion : TypeExpansion {
913   QualType EltTy;
914 
915   ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
916   static bool classof(const TypeExpansion *TE) {
917     return TE->Kind == TEK_Complex;
918   }
919 };
920 
921 struct NoExpansion : TypeExpansion {
922   NoExpansion() : TypeExpansion(TEK_None) {}
923   static bool classof(const TypeExpansion *TE) {
924     return TE->Kind == TEK_None;
925   }
926 };
927 }  // namespace
928 
929 static std::unique_ptr<TypeExpansion>
930 getTypeExpansion(QualType Ty, const ASTContext &Context) {
931   if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
932     return std::make_unique<ConstantArrayExpansion>(
933         AT->getElementType(), AT->getSize().getZExtValue());
934   }
935   if (const RecordType *RT = Ty->getAs<RecordType>()) {
936     SmallVector<const CXXBaseSpecifier *, 1> Bases;
937     SmallVector<const FieldDecl *, 1> Fields;
938     const RecordDecl *RD = RT->getDecl();
939     assert(!RD->hasFlexibleArrayMember() &&
940            "Cannot expand structure with flexible array.");
941     if (RD->isUnion()) {
942       // Unions can be here only in degenerative cases - all the fields are same
943       // after flattening. Thus we have to use the "largest" field.
944       const FieldDecl *LargestFD = nullptr;
945       CharUnits UnionSize = CharUnits::Zero();
946 
947       for (const auto *FD : RD->fields()) {
948         if (FD->isZeroLengthBitField(Context))
949           continue;
950         assert(!FD->isBitField() &&
951                "Cannot expand structure with bit-field members.");
952         CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
953         if (UnionSize < FieldSize) {
954           UnionSize = FieldSize;
955           LargestFD = FD;
956         }
957       }
958       if (LargestFD)
959         Fields.push_back(LargestFD);
960     } else {
961       if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
962         assert(!CXXRD->isDynamicClass() &&
963                "cannot expand vtable pointers in dynamic classes");
964         llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));
965       }
966 
967       for (const auto *FD : RD->fields()) {
968         if (FD->isZeroLengthBitField(Context))
969           continue;
970         assert(!FD->isBitField() &&
971                "Cannot expand structure with bit-field members.");
972         Fields.push_back(FD);
973       }
974     }
975     return std::make_unique<RecordExpansion>(std::move(Bases),
976                                               std::move(Fields));
977   }
978   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
979     return std::make_unique<ComplexExpansion>(CT->getElementType());
980   }
981   return std::make_unique<NoExpansion>();
982 }
983 
984 static int getExpansionSize(QualType Ty, const ASTContext &Context) {
985   auto Exp = getTypeExpansion(Ty, Context);
986   if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
987     return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
988   }
989   if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
990     int Res = 0;
991     for (auto BS : RExp->Bases)
992       Res += getExpansionSize(BS->getType(), Context);
993     for (auto FD : RExp->Fields)
994       Res += getExpansionSize(FD->getType(), Context);
995     return Res;
996   }
997   if (isa<ComplexExpansion>(Exp.get()))
998     return 2;
999   assert(isa<NoExpansion>(Exp.get()));
1000   return 1;
1001 }
1002 
1003 void
1004 CodeGenTypes::getExpandedTypes(QualType Ty,
1005                                SmallVectorImpl<llvm::Type *>::iterator &TI) {
1006   auto Exp = getTypeExpansion(Ty, Context);
1007   if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1008     for (int i = 0, n = CAExp->NumElts; i < n; i++) {
1009       getExpandedTypes(CAExp->EltTy, TI);
1010     }
1011   } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1012     for (auto BS : RExp->Bases)
1013       getExpandedTypes(BS->getType(), TI);
1014     for (auto FD : RExp->Fields)
1015       getExpandedTypes(FD->getType(), TI);
1016   } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
1017     llvm::Type *EltTy = ConvertType(CExp->EltTy);
1018     *TI++ = EltTy;
1019     *TI++ = EltTy;
1020   } else {
1021     assert(isa<NoExpansion>(Exp.get()));
1022     *TI++ = ConvertType(Ty);
1023   }
1024 }
1025 
1026 static void forConstantArrayExpansion(CodeGenFunction &CGF,
1027                                       ConstantArrayExpansion *CAE,
1028                                       Address BaseAddr,
1029                                       llvm::function_ref<void(Address)> Fn) {
1030   CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1031   CharUnits EltAlign =
1032     BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1033   llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy);
1034 
1035   for (int i = 0, n = CAE->NumElts; i < n; i++) {
1036     llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
1037         BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
1038     Fn(Address(EltAddr, EltTy, EltAlign));
1039   }
1040 }
1041 
1042 void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1043                                          llvm::Function::arg_iterator &AI) {
1044   assert(LV.isSimple() &&
1045          "Unexpected non-simple lvalue during struct expansion.");
1046 
1047   auto Exp = getTypeExpansion(Ty, getContext());
1048   if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1049     forConstantArrayExpansion(
1050         *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1051           LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1052           ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1053         });
1054   } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1055     Address This = LV.getAddress(*this);
1056     for (const CXXBaseSpecifier *BS : RExp->Bases) {
1057       // Perform a single step derived-to-base conversion.
1058       Address Base =
1059           GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1060                                 /*NullCheckValue=*/false, SourceLocation());
1061       LValue SubLV = MakeAddrLValue(Base, BS->getType());
1062 
1063       // Recurse onto bases.
1064       ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1065     }
1066     for (auto FD : RExp->Fields) {
1067       // FIXME: What are the right qualifiers here?
1068       LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1069       ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1070     }
1071   } else if (isa<ComplexExpansion>(Exp.get())) {
1072     auto realValue = &*AI++;
1073     auto imagValue = &*AI++;
1074     EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1075   } else {
1076     // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1077     // primitive store.
1078     assert(isa<NoExpansion>(Exp.get()));
1079     llvm::Value *Arg = &*AI++;
1080     if (LV.isBitField()) {
1081       EmitStoreThroughLValue(RValue::get(Arg), LV);
1082     } else {
1083       // TODO: currently there are some places are inconsistent in what LLVM
1084       // pointer type they use (see D118744). Once clang uses opaque pointers
1085       // all LLVM pointer types will be the same and we can remove this check.
1086       if (Arg->getType()->isPointerTy()) {
1087         Address Addr = LV.getAddress(*this);
1088         Arg = Builder.CreateBitCast(Arg, Addr.getElementType());
1089       }
1090       EmitStoreOfScalar(Arg, LV);
1091     }
1092   }
1093 }
1094 
1095 void CodeGenFunction::ExpandTypeToArgs(
1096     QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1097     SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1098   auto Exp = getTypeExpansion(Ty, getContext());
1099   if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1100     Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1101                                    : Arg.getKnownRValue().getAggregateAddress();
1102     forConstantArrayExpansion(
1103         *this, CAExp, Addr, [&](Address EltAddr) {
1104           CallArg EltArg = CallArg(
1105               convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1106               CAExp->EltTy);
1107           ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1108                            IRCallArgPos);
1109         });
1110   } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1111     Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1112                                    : Arg.getKnownRValue().getAggregateAddress();
1113     for (const CXXBaseSpecifier *BS : RExp->Bases) {
1114       // Perform a single step derived-to-base conversion.
1115       Address Base =
1116           GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1117                                 /*NullCheckValue=*/false, SourceLocation());
1118       CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1119 
1120       // Recurse onto bases.
1121       ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1122                        IRCallArgPos);
1123     }
1124 
1125     LValue LV = MakeAddrLValue(This, Ty);
1126     for (auto FD : RExp->Fields) {
1127       CallArg FldArg =
1128           CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1129       ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1130                        IRCallArgPos);
1131     }
1132   } else if (isa<ComplexExpansion>(Exp.get())) {
1133     ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1134     IRCallArgs[IRCallArgPos++] = CV.first;
1135     IRCallArgs[IRCallArgPos++] = CV.second;
1136   } else {
1137     assert(isa<NoExpansion>(Exp.get()));
1138     auto RV = Arg.getKnownRValue();
1139     assert(RV.isScalar() &&
1140            "Unexpected non-scalar rvalue during struct expansion.");
1141 
1142     // Insert a bitcast as needed.
1143     llvm::Value *V = RV.getScalarVal();
1144     if (IRCallArgPos < IRFuncTy->getNumParams() &&
1145         V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1146       V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1147 
1148     IRCallArgs[IRCallArgPos++] = V;
1149   }
1150 }
1151 
1152 /// Create a temporary allocation for the purposes of coercion.
1153 static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1154                                            CharUnits MinAlign,
1155                                            const Twine &Name = "tmp") {
1156   // Don't use an alignment that's worse than what LLVM would prefer.
1157   auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
1158   CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1159 
1160   return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1161 }
1162 
1163 /// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1164 /// accessing some number of bytes out of it, try to gep into the struct to get
1165 /// at its inner goodness.  Dive as deep as possible without entering an element
1166 /// with an in-memory size smaller than DstSize.
1167 static Address
1168 EnterStructPointerForCoercedAccess(Address SrcPtr,
1169                                    llvm::StructType *SrcSTy,
1170                                    uint64_t DstSize, CodeGenFunction &CGF) {
1171   // We can't dive into a zero-element struct.
1172   if (SrcSTy->getNumElements() == 0) return SrcPtr;
1173 
1174   llvm::Type *FirstElt = SrcSTy->getElementType(0);
1175 
1176   // If the first elt is at least as large as what we're looking for, or if the
1177   // first element is the same size as the whole struct, we can enter it. The
1178   // comparison must be made on the store size and not the alloca size. Using
1179   // the alloca size may overstate the size of the load.
1180   uint64_t FirstEltSize =
1181     CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1182   if (FirstEltSize < DstSize &&
1183       FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy))
1184     return SrcPtr;
1185 
1186   // GEP into the first element.
1187   SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1188 
1189   // If the first element is a struct, recurse.
1190   llvm::Type *SrcTy = SrcPtr.getElementType();
1191   if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1192     return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1193 
1194   return SrcPtr;
1195 }
1196 
1197 /// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1198 /// are either integers or pointers.  This does a truncation of the value if it
1199 /// is too large or a zero extension if it is too small.
1200 ///
1201 /// This behaves as if the value were coerced through memory, so on big-endian
1202 /// targets the high bits are preserved in a truncation, while little-endian
1203 /// targets preserve the low bits.
1204 static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1205                                              llvm::Type *Ty,
1206                                              CodeGenFunction &CGF) {
1207   if (Val->getType() == Ty)
1208     return Val;
1209 
1210   if (isa<llvm::PointerType>(Val->getType())) {
1211     // If this is Pointer->Pointer avoid conversion to and from int.
1212     if (isa<llvm::PointerType>(Ty))
1213       return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1214 
1215     // Convert the pointer to an integer so we can play with its width.
1216     Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1217   }
1218 
1219   llvm::Type *DestIntTy = Ty;
1220   if (isa<llvm::PointerType>(DestIntTy))
1221     DestIntTy = CGF.IntPtrTy;
1222 
1223   if (Val->getType() != DestIntTy) {
1224     const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1225     if (DL.isBigEndian()) {
1226       // Preserve the high bits on big-endian targets.
1227       // That is what memory coercion does.
1228       uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1229       uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1230 
1231       if (SrcSize > DstSize) {
1232         Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1233         Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1234       } else {
1235         Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1236         Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1237       }
1238     } else {
1239       // Little-endian targets preserve the low bits. No shifts required.
1240       Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1241     }
1242   }
1243 
1244   if (isa<llvm::PointerType>(Ty))
1245     Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1246   return Val;
1247 }
1248 
1249 
1250 
1251 /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1252 /// a pointer to an object of type \arg Ty, known to be aligned to
1253 /// \arg SrcAlign bytes.
1254 ///
1255 /// This safely handles the case when the src type is smaller than the
1256 /// destination type; in this situation the values of bits which not
1257 /// present in the src are undefined.
1258 static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1259                                       CodeGenFunction &CGF) {
1260   llvm::Type *SrcTy = Src.getElementType();
1261 
1262   // If SrcTy and Ty are the same, just do a load.
1263   if (SrcTy == Ty)
1264     return CGF.Builder.CreateLoad(Src);
1265 
1266   llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1267 
1268   if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1269     Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1270                                              DstSize.getFixedValue(), CGF);
1271     SrcTy = Src.getElementType();
1272   }
1273 
1274   llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1275 
1276   // If the source and destination are integer or pointer types, just do an
1277   // extension or truncation to the desired type.
1278   if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
1279       (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
1280     llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1281     return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1282   }
1283 
1284   // If load is legal, just bitcast the src pointer.
1285   if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1286       SrcSize.getFixedValue() >= DstSize.getFixedValue()) {
1287     // Generally SrcSize is never greater than DstSize, since this means we are
1288     // losing bits. However, this can happen in cases where the structure has
1289     // additional padding, for example due to a user specified alignment.
1290     //
1291     // FIXME: Assert that we aren't truncating non-padding bits when have access
1292     // to that information.
1293     Src = Src.withElementType(Ty);
1294     return CGF.Builder.CreateLoad(Src);
1295   }
1296 
1297   // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1298   // the types match, use the llvm.vector.insert intrinsic to perform the
1299   // conversion.
1300   if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1301     if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1302       // If we are casting a fixed i8 vector to a scalable 16 x i1 predicate
1303       // vector, use a vector insert and bitcast the result.
1304       bool NeedsBitcast = false;
1305       auto PredType =
1306           llvm::ScalableVectorType::get(CGF.Builder.getInt1Ty(), 16);
1307       llvm::Type *OrigType = Ty;
1308       if (ScalableDst == PredType &&
1309           FixedSrc->getElementType() == CGF.Builder.getInt8Ty()) {
1310         ScalableDst = llvm::ScalableVectorType::get(CGF.Builder.getInt8Ty(), 2);
1311         NeedsBitcast = true;
1312       }
1313       if (ScalableDst->getElementType() == FixedSrc->getElementType()) {
1314         auto *Load = CGF.Builder.CreateLoad(Src);
1315         auto *UndefVec = llvm::UndefValue::get(ScalableDst);
1316         auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
1317         llvm::Value *Result = CGF.Builder.CreateInsertVector(
1318             ScalableDst, UndefVec, Load, Zero, "cast.scalable");
1319         if (NeedsBitcast)
1320           Result = CGF.Builder.CreateBitCast(Result, OrigType);
1321         return Result;
1322       }
1323     }
1324   }
1325 
1326   // Otherwise do coercion through memory. This is stupid, but simple.
1327   Address Tmp =
1328       CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1329   CGF.Builder.CreateMemCpy(
1330       Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
1331       Src.getAlignment().getAsAlign(),
1332       llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue()));
1333   return CGF.Builder.CreateLoad(Tmp);
1334 }
1335 
1336 // Function to store a first-class aggregate into memory.  We prefer to
1337 // store the elements rather than the aggregate to be more friendly to
1338 // fast-isel.
1339 // FIXME: Do we need to recurse here?
1340 void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1341                                          bool DestIsVolatile) {
1342   // Prefer scalar stores to first-class aggregate stores.
1343   if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1344     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1345       Address EltPtr = Builder.CreateStructGEP(Dest, i);
1346       llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
1347       Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1348     }
1349   } else {
1350     Builder.CreateStore(Val, Dest, DestIsVolatile);
1351   }
1352 }
1353 
1354 /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1355 /// where the source and destination may have different types.  The
1356 /// destination is known to be aligned to \arg DstAlign bytes.
1357 ///
1358 /// This safely handles the case when the src type is larger than the
1359 /// destination type; the upper bits of the src will be lost.
1360 static void CreateCoercedStore(llvm::Value *Src,
1361                                Address Dst,
1362                                bool DstIsVolatile,
1363                                CodeGenFunction &CGF) {
1364   llvm::Type *SrcTy = Src->getType();
1365   llvm::Type *DstTy = Dst.getElementType();
1366   if (SrcTy == DstTy) {
1367     CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1368     return;
1369   }
1370 
1371   llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1372 
1373   if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1374     Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1375                                              SrcSize.getFixedValue(), CGF);
1376     DstTy = Dst.getElementType();
1377   }
1378 
1379   llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1380   llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1381   if (SrcPtrTy && DstPtrTy &&
1382       SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) {
1383     Src = CGF.Builder.CreateAddrSpaceCast(Src, DstTy);
1384     CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1385     return;
1386   }
1387 
1388   // If the source and destination are integer or pointer types, just do an
1389   // extension or truncation to the desired type.
1390   if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
1391       (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
1392     Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1393     CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1394     return;
1395   }
1396 
1397   llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1398 
1399   // If store is legal, just bitcast the src pointer.
1400   if (isa<llvm::ScalableVectorType>(SrcTy) ||
1401       isa<llvm::ScalableVectorType>(DstTy) ||
1402       SrcSize.getFixedValue() <= DstSize.getFixedValue()) {
1403     Dst = Dst.withElementType(SrcTy);
1404     CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
1405   } else {
1406     // Otherwise do coercion through memory. This is stupid, but
1407     // simple.
1408 
1409     // Generally SrcSize is never greater than DstSize, since this means we are
1410     // losing bits. However, this can happen in cases where the structure has
1411     // additional padding, for example due to a user specified alignment.
1412     //
1413     // FIXME: Assert that we aren't truncating non-padding bits when have access
1414     // to that information.
1415     Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1416     CGF.Builder.CreateStore(Src, Tmp);
1417     CGF.Builder.CreateMemCpy(
1418         Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
1419         Tmp.getAlignment().getAsAlign(),
1420         llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedValue()));
1421   }
1422 }
1423 
1424 static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1425                                    const ABIArgInfo &info) {
1426   if (unsigned offset = info.getDirectOffset()) {
1427     addr = addr.withElementType(CGF.Int8Ty);
1428     addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1429                                              CharUnits::fromQuantity(offset));
1430     addr = addr.withElementType(info.getCoerceToType());
1431   }
1432   return addr;
1433 }
1434 
1435 namespace {
1436 
1437 /// Encapsulates information about the way function arguments from
1438 /// CGFunctionInfo should be passed to actual LLVM IR function.
1439 class ClangToLLVMArgMapping {
1440   static const unsigned InvalidIndex = ~0U;
1441   unsigned InallocaArgNo;
1442   unsigned SRetArgNo;
1443   unsigned TotalIRArgs;
1444 
1445   /// Arguments of LLVM IR function corresponding to single Clang argument.
1446   struct IRArgs {
1447     unsigned PaddingArgIndex;
1448     // Argument is expanded to IR arguments at positions
1449     // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1450     unsigned FirstArgIndex;
1451     unsigned NumberOfArgs;
1452 
1453     IRArgs()
1454         : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1455           NumberOfArgs(0) {}
1456   };
1457 
1458   SmallVector<IRArgs, 8> ArgInfo;
1459 
1460 public:
1461   ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1462                         bool OnlyRequiredArgs = false)
1463       : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1464         ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1465     construct(Context, FI, OnlyRequiredArgs);
1466   }
1467 
1468   bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1469   unsigned getInallocaArgNo() const {
1470     assert(hasInallocaArg());
1471     return InallocaArgNo;
1472   }
1473 
1474   bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1475   unsigned getSRetArgNo() const {
1476     assert(hasSRetArg());
1477     return SRetArgNo;
1478   }
1479 
1480   unsigned totalIRArgs() const { return TotalIRArgs; }
1481 
1482   bool hasPaddingArg(unsigned ArgNo) const {
1483     assert(ArgNo < ArgInfo.size());
1484     return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1485   }
1486   unsigned getPaddingArgNo(unsigned ArgNo) const {
1487     assert(hasPaddingArg(ArgNo));
1488     return ArgInfo[ArgNo].PaddingArgIndex;
1489   }
1490 
1491   /// Returns index of first IR argument corresponding to ArgNo, and their
1492   /// quantity.
1493   std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1494     assert(ArgNo < ArgInfo.size());
1495     return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1496                           ArgInfo[ArgNo].NumberOfArgs);
1497   }
1498 
1499 private:
1500   void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1501                  bool OnlyRequiredArgs);
1502 };
1503 
1504 void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1505                                       const CGFunctionInfo &FI,
1506                                       bool OnlyRequiredArgs) {
1507   unsigned IRArgNo = 0;
1508   bool SwapThisWithSRet = false;
1509   const ABIArgInfo &RetAI = FI.getReturnInfo();
1510 
1511   if (RetAI.getKind() == ABIArgInfo::Indirect) {
1512     SwapThisWithSRet = RetAI.isSRetAfterThis();
1513     SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1514   }
1515 
1516   unsigned ArgNo = 0;
1517   unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1518   for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1519        ++I, ++ArgNo) {
1520     assert(I != FI.arg_end());
1521     QualType ArgType = I->type;
1522     const ABIArgInfo &AI = I->info;
1523     // Collect data about IR arguments corresponding to Clang argument ArgNo.
1524     auto &IRArgs = ArgInfo[ArgNo];
1525 
1526     if (AI.getPaddingType())
1527       IRArgs.PaddingArgIndex = IRArgNo++;
1528 
1529     switch (AI.getKind()) {
1530     case ABIArgInfo::Extend:
1531     case ABIArgInfo::Direct: {
1532       // FIXME: handle sseregparm someday...
1533       llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1534       if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1535         IRArgs.NumberOfArgs = STy->getNumElements();
1536       } else {
1537         IRArgs.NumberOfArgs = 1;
1538       }
1539       break;
1540     }
1541     case ABIArgInfo::Indirect:
1542     case ABIArgInfo::IndirectAliased:
1543       IRArgs.NumberOfArgs = 1;
1544       break;
1545     case ABIArgInfo::Ignore:
1546     case ABIArgInfo::InAlloca:
1547       // ignore and inalloca doesn't have matching LLVM parameters.
1548       IRArgs.NumberOfArgs = 0;
1549       break;
1550     case ABIArgInfo::CoerceAndExpand:
1551       IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1552       break;
1553     case ABIArgInfo::Expand:
1554       IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1555       break;
1556     }
1557 
1558     if (IRArgs.NumberOfArgs > 0) {
1559       IRArgs.FirstArgIndex = IRArgNo;
1560       IRArgNo += IRArgs.NumberOfArgs;
1561     }
1562 
1563     // Skip over the sret parameter when it comes second.  We already handled it
1564     // above.
1565     if (IRArgNo == 1 && SwapThisWithSRet)
1566       IRArgNo++;
1567   }
1568   assert(ArgNo == ArgInfo.size());
1569 
1570   if (FI.usesInAlloca())
1571     InallocaArgNo = IRArgNo++;
1572 
1573   TotalIRArgs = IRArgNo;
1574 }
1575 }  // namespace
1576 
1577 /***/
1578 
1579 bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1580   const auto &RI = FI.getReturnInfo();
1581   return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1582 }
1583 
1584 bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1585   return ReturnTypeUsesSRet(FI) &&
1586          getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1587 }
1588 
1589 bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1590   if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1591     switch (BT->getKind()) {
1592     default:
1593       return false;
1594     case BuiltinType::Float:
1595       return getTarget().useObjCFPRetForRealType(FloatModeKind::Float);
1596     case BuiltinType::Double:
1597       return getTarget().useObjCFPRetForRealType(FloatModeKind::Double);
1598     case BuiltinType::LongDouble:
1599       return getTarget().useObjCFPRetForRealType(FloatModeKind::LongDouble);
1600     }
1601   }
1602 
1603   return false;
1604 }
1605 
1606 bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1607   if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1608     if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1609       if (BT->getKind() == BuiltinType::LongDouble)
1610         return getTarget().useObjCFP2RetForComplexLongDouble();
1611     }
1612   }
1613 
1614   return false;
1615 }
1616 
1617 llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1618   const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1619   return GetFunctionType(FI);
1620 }
1621 
1622 llvm::FunctionType *
1623 CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1624 
1625   bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1626   (void)Inserted;
1627   assert(Inserted && "Recursively being processed?");
1628 
1629   llvm::Type *resultType = nullptr;
1630   const ABIArgInfo &retAI = FI.getReturnInfo();
1631   switch (retAI.getKind()) {
1632   case ABIArgInfo::Expand:
1633   case ABIArgInfo::IndirectAliased:
1634     llvm_unreachable("Invalid ABI kind for return argument");
1635 
1636   case ABIArgInfo::Extend:
1637   case ABIArgInfo::Direct:
1638     resultType = retAI.getCoerceToType();
1639     break;
1640 
1641   case ABIArgInfo::InAlloca:
1642     if (retAI.getInAllocaSRet()) {
1643       // sret things on win32 aren't void, they return the sret pointer.
1644       QualType ret = FI.getReturnType();
1645       unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(ret);
1646       resultType = llvm::PointerType::get(getLLVMContext(), addressSpace);
1647     } else {
1648       resultType = llvm::Type::getVoidTy(getLLVMContext());
1649     }
1650     break;
1651 
1652   case ABIArgInfo::Indirect:
1653   case ABIArgInfo::Ignore:
1654     resultType = llvm::Type::getVoidTy(getLLVMContext());
1655     break;
1656 
1657   case ABIArgInfo::CoerceAndExpand:
1658     resultType = retAI.getUnpaddedCoerceAndExpandType();
1659     break;
1660   }
1661 
1662   ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1663   SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1664 
1665   // Add type for sret argument.
1666   if (IRFunctionArgs.hasSRetArg()) {
1667     QualType Ret = FI.getReturnType();
1668     unsigned AddressSpace = CGM.getTypes().getTargetAddressSpace(Ret);
1669     ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1670         llvm::PointerType::get(getLLVMContext(), AddressSpace);
1671   }
1672 
1673   // Add type for inalloca argument.
1674   if (IRFunctionArgs.hasInallocaArg())
1675     ArgTypes[IRFunctionArgs.getInallocaArgNo()] =
1676         llvm::PointerType::getUnqual(getLLVMContext());
1677 
1678   // Add in all of the required arguments.
1679   unsigned ArgNo = 0;
1680   CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1681                                      ie = it + FI.getNumRequiredArgs();
1682   for (; it != ie; ++it, ++ArgNo) {
1683     const ABIArgInfo &ArgInfo = it->info;
1684 
1685     // Insert a padding type to ensure proper alignment.
1686     if (IRFunctionArgs.hasPaddingArg(ArgNo))
1687       ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1688           ArgInfo.getPaddingType();
1689 
1690     unsigned FirstIRArg, NumIRArgs;
1691     std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1692 
1693     switch (ArgInfo.getKind()) {
1694     case ABIArgInfo::Ignore:
1695     case ABIArgInfo::InAlloca:
1696       assert(NumIRArgs == 0);
1697       break;
1698 
1699     case ABIArgInfo::Indirect:
1700       assert(NumIRArgs == 1);
1701       // indirect arguments are always on the stack, which is alloca addr space.
1702       ArgTypes[FirstIRArg] = llvm::PointerType::get(
1703           getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace());
1704       break;
1705     case ABIArgInfo::IndirectAliased:
1706       assert(NumIRArgs == 1);
1707       ArgTypes[FirstIRArg] = llvm::PointerType::get(
1708           getLLVMContext(), ArgInfo.getIndirectAddrSpace());
1709       break;
1710     case ABIArgInfo::Extend:
1711     case ABIArgInfo::Direct: {
1712       // Fast-isel and the optimizer generally like scalar values better than
1713       // FCAs, so we flatten them if this is safe to do for this argument.
1714       llvm::Type *argType = ArgInfo.getCoerceToType();
1715       llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1716       if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1717         assert(NumIRArgs == st->getNumElements());
1718         for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1719           ArgTypes[FirstIRArg + i] = st->getElementType(i);
1720       } else {
1721         assert(NumIRArgs == 1);
1722         ArgTypes[FirstIRArg] = argType;
1723       }
1724       break;
1725     }
1726 
1727     case ABIArgInfo::CoerceAndExpand: {
1728       auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1729       for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1730         *ArgTypesIter++ = EltTy;
1731       }
1732       assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1733       break;
1734     }
1735 
1736     case ABIArgInfo::Expand:
1737       auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1738       getExpandedTypes(it->type, ArgTypesIter);
1739       assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1740       break;
1741     }
1742   }
1743 
1744   bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1745   assert(Erased && "Not in set?");
1746 
1747   return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1748 }
1749 
1750 llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1751   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1752   const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
1753 
1754   if (!isFuncTypeConvertible(FPT))
1755     return llvm::StructType::get(getLLVMContext());
1756 
1757   return GetFunctionType(GD);
1758 }
1759 
1760 static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1761                                                llvm::AttrBuilder &FuncAttrs,
1762                                                const FunctionProtoType *FPT) {
1763   if (!FPT)
1764     return;
1765 
1766   if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1767       FPT->isNothrow())
1768     FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1769 
1770   if (FPT->getAArch64SMEAttributes() & FunctionType::SME_PStateSMEnabledMask)
1771     FuncAttrs.addAttribute("aarch64_pstate_sm_enabled");
1772   if (FPT->getAArch64SMEAttributes() & FunctionType::SME_PStateSMCompatibleMask)
1773     FuncAttrs.addAttribute("aarch64_pstate_sm_compatible");
1774   if (FPT->getAArch64SMEAttributes() & FunctionType::SME_PStateZASharedMask)
1775     FuncAttrs.addAttribute("aarch64_pstate_za_shared");
1776   if (FPT->getAArch64SMEAttributes() & FunctionType::SME_PStateZAPreservedMask)
1777     FuncAttrs.addAttribute("aarch64_pstate_za_preserved");
1778 }
1779 
1780 static void AddAttributesFromAssumes(llvm::AttrBuilder &FuncAttrs,
1781                                      const Decl *Callee) {
1782   if (!Callee)
1783     return;
1784 
1785   SmallVector<StringRef, 4> Attrs;
1786 
1787   for (const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>())
1788     AA->getAssumption().split(Attrs, ",");
1789 
1790   if (!Attrs.empty())
1791     FuncAttrs.addAttribute(llvm::AssumptionAttrKey,
1792                            llvm::join(Attrs.begin(), Attrs.end(), ","));
1793 }
1794 
1795 bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1796                                           QualType ReturnType) const {
1797   // We can't just discard the return value for a record type with a
1798   // complex destructor or a non-trivially copyable type.
1799   if (const RecordType *RT =
1800           ReturnType.getCanonicalType()->getAs<RecordType>()) {
1801     if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1802       return ClassDecl->hasTrivialDestructor();
1803   }
1804   return ReturnType.isTriviallyCopyableType(Context);
1805 }
1806 
1807 static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,
1808                             const Decl *TargetDecl) {
1809   // As-is msan can not tolerate noundef mismatch between caller and
1810   // implementation. Mismatch is possible for e.g. indirect calls from C-caller
1811   // into C++. Such mismatches lead to confusing false reports. To avoid
1812   // expensive workaround on msan we enforce initialization event in uncommon
1813   // cases where it's allowed.
1814   if (Module.getLangOpts().Sanitize.has(SanitizerKind::Memory))
1815     return true;
1816   // C++ explicitly makes returning undefined values UB. C's rule only applies
1817   // to used values, so we never mark them noundef for now.
1818   if (!Module.getLangOpts().CPlusPlus)
1819     return false;
1820   if (TargetDecl) {
1821     if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl)) {
1822       if (FDecl->isExternC())
1823         return false;
1824     } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl)) {
1825       // Function pointer.
1826       if (VDecl->isExternC())
1827         return false;
1828     }
1829   }
1830 
1831   // We don't want to be too aggressive with the return checking, unless
1832   // it's explicit in the code opts or we're using an appropriate sanitizer.
1833   // Try to respect what the programmer intended.
1834   return Module.getCodeGenOpts().StrictReturn ||
1835          !Module.MayDropFunctionReturn(Module.getContext(), RetTy) ||
1836          Module.getLangOpts().Sanitize.has(SanitizerKind::Return);
1837 }
1838 
1839 /// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the
1840 /// requested denormal behavior, accounting for the overriding behavior of the
1841 /// -f32 case.
1842 static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,
1843                                  llvm::DenormalMode FP32DenormalMode,
1844                                  llvm::AttrBuilder &FuncAttrs) {
1845   if (FPDenormalMode != llvm::DenormalMode::getDefault())
1846     FuncAttrs.addAttribute("denormal-fp-math", FPDenormalMode.str());
1847 
1848   if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())
1849     FuncAttrs.addAttribute("denormal-fp-math-f32", FP32DenormalMode.str());
1850 }
1851 
1852 /// Add default attributes to a function, which have merge semantics under
1853 /// -mlink-builtin-bitcode and should not simply overwrite any existing
1854 /// attributes in the linked library.
1855 static void
1856 addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
1857                                      llvm::AttrBuilder &FuncAttrs) {
1858   addDenormalModeAttrs(CodeGenOpts.FPDenormalMode, CodeGenOpts.FP32DenormalMode,
1859                        FuncAttrs);
1860 }
1861 
1862 static void getTrivialDefaultFunctionAttributes(
1863     StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
1864     const LangOptions &LangOpts, bool AttrOnCallSite,
1865     llvm::AttrBuilder &FuncAttrs) {
1866   // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1867   if (!HasOptnone) {
1868     if (CodeGenOpts.OptimizeSize)
1869       FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1870     if (CodeGenOpts.OptimizeSize == 2)
1871       FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1872   }
1873 
1874   if (CodeGenOpts.DisableRedZone)
1875     FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1876   if (CodeGenOpts.IndirectTlsSegRefs)
1877     FuncAttrs.addAttribute("indirect-tls-seg-refs");
1878   if (CodeGenOpts.NoImplicitFloat)
1879     FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1880 
1881   if (AttrOnCallSite) {
1882     // Attributes that should go on the call site only.
1883     // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1884     // the -fno-builtin-foo list.
1885     if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))
1886       FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1887     if (!CodeGenOpts.TrapFuncName.empty())
1888       FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1889   } else {
1890     switch (CodeGenOpts.getFramePointer()) {
1891     case CodeGenOptions::FramePointerKind::None:
1892       // This is the default behavior.
1893       break;
1894     case CodeGenOptions::FramePointerKind::NonLeaf:
1895     case CodeGenOptions::FramePointerKind::All:
1896       FuncAttrs.addAttribute("frame-pointer",
1897                              CodeGenOptions::getFramePointerKindName(
1898                                  CodeGenOpts.getFramePointer()));
1899     }
1900 
1901     if (CodeGenOpts.LessPreciseFPMAD)
1902       FuncAttrs.addAttribute("less-precise-fpmad", "true");
1903 
1904     if (CodeGenOpts.NullPointerIsValid)
1905       FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1906 
1907     if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
1908       FuncAttrs.addAttribute("no-trapping-math", "true");
1909 
1910     // TODO: Are these all needed?
1911     // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1912     if (LangOpts.NoHonorInfs)
1913       FuncAttrs.addAttribute("no-infs-fp-math", "true");
1914     if (LangOpts.NoHonorNaNs)
1915       FuncAttrs.addAttribute("no-nans-fp-math", "true");
1916     if (LangOpts.ApproxFunc)
1917       FuncAttrs.addAttribute("approx-func-fp-math", "true");
1918     if (LangOpts.AllowFPReassoc && LangOpts.AllowRecip &&
1919         LangOpts.NoSignedZero && LangOpts.ApproxFunc &&
1920         (LangOpts.getDefaultFPContractMode() ==
1921              LangOptions::FPModeKind::FPM_Fast ||
1922          LangOpts.getDefaultFPContractMode() ==
1923              LangOptions::FPModeKind::FPM_FastHonorPragmas))
1924       FuncAttrs.addAttribute("unsafe-fp-math", "true");
1925     if (CodeGenOpts.SoftFloat)
1926       FuncAttrs.addAttribute("use-soft-float", "true");
1927     FuncAttrs.addAttribute("stack-protector-buffer-size",
1928                            llvm::utostr(CodeGenOpts.SSPBufferSize));
1929     if (LangOpts.NoSignedZero)
1930       FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true");
1931 
1932     // TODO: Reciprocal estimate codegen options should apply to instructions?
1933     const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1934     if (!Recips.empty())
1935       FuncAttrs.addAttribute("reciprocal-estimates",
1936                              llvm::join(Recips, ","));
1937 
1938     if (!CodeGenOpts.PreferVectorWidth.empty() &&
1939         CodeGenOpts.PreferVectorWidth != "none")
1940       FuncAttrs.addAttribute("prefer-vector-width",
1941                              CodeGenOpts.PreferVectorWidth);
1942 
1943     if (CodeGenOpts.StackRealignment)
1944       FuncAttrs.addAttribute("stackrealign");
1945     if (CodeGenOpts.Backchain)
1946       FuncAttrs.addAttribute("backchain");
1947     if (CodeGenOpts.EnableSegmentedStacks)
1948       FuncAttrs.addAttribute("split-stack");
1949 
1950     if (CodeGenOpts.SpeculativeLoadHardening)
1951       FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1952 
1953     // Add zero-call-used-regs attribute.
1954     switch (CodeGenOpts.getZeroCallUsedRegs()) {
1955     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
1956       FuncAttrs.removeAttribute("zero-call-used-regs");
1957       break;
1958     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
1959       FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr-arg");
1960       break;
1961     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
1962       FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr");
1963       break;
1964     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
1965       FuncAttrs.addAttribute("zero-call-used-regs", "used-arg");
1966       break;
1967     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
1968       FuncAttrs.addAttribute("zero-call-used-regs", "used");
1969       break;
1970     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
1971       FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr-arg");
1972       break;
1973     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
1974       FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr");
1975       break;
1976     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
1977       FuncAttrs.addAttribute("zero-call-used-regs", "all-arg");
1978       break;
1979     case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
1980       FuncAttrs.addAttribute("zero-call-used-regs", "all");
1981       break;
1982     }
1983   }
1984 
1985   if (LangOpts.assumeFunctionsAreConvergent()) {
1986     // Conservatively, mark all functions and calls in CUDA and OpenCL as
1987     // convergent (meaning, they may call an intrinsically convergent op, such
1988     // as __syncthreads() / barrier(), and so can't have certain optimizations
1989     // applied around them).  LLVM will remove this attribute where it safely
1990     // can.
1991     FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1992   }
1993 
1994   // TODO: NoUnwind attribute should be added for other GPU modes HIP,
1995   // OpenMP offload. AFAIK, neither of them support exceptions in device code.
1996   if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
1997       LangOpts.SYCLIsDevice) {
1998     FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1999   }
2000 
2001   for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
2002     StringRef Var, Value;
2003     std::tie(Var, Value) = Attr.split('=');
2004     FuncAttrs.addAttribute(Var, Value);
2005   }
2006 }
2007 
2008 /// Merges `target-features` from \TargetOpts and \F, and sets the result in
2009 /// \FuncAttr
2010 /// * features from \F are always kept
2011 /// * a feature from \TargetOpts is kept if itself and its opposite are absent
2012 /// from \F
2013 static void
2014 overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,
2015                                            const llvm::Function &F,
2016                                            const TargetOptions &TargetOpts) {
2017   auto FFeatures = F.getFnAttribute("target-features");
2018 
2019   llvm::StringSet<> MergedNames;
2020   SmallVector<StringRef> MergedFeatures;
2021   MergedFeatures.reserve(TargetOpts.Features.size());
2022 
2023   auto AddUnmergedFeatures = [&](auto &&FeatureRange) {
2024     for (StringRef Feature : FeatureRange) {
2025       if (Feature.empty())
2026         continue;
2027       assert(Feature[0] == '+' || Feature[0] == '-');
2028       StringRef Name = Feature.drop_front(1);
2029       bool Merged = !MergedNames.insert(Name).second;
2030       if (!Merged)
2031         MergedFeatures.push_back(Feature);
2032     }
2033   };
2034 
2035   if (FFeatures.isValid())
2036     AddUnmergedFeatures(llvm::split(FFeatures.getValueAsString(), ','));
2037   AddUnmergedFeatures(TargetOpts.Features);
2038 
2039   if (!MergedFeatures.empty()) {
2040     llvm::sort(MergedFeatures);
2041     FuncAttr.addAttribute("target-features", llvm::join(MergedFeatures, ","));
2042   }
2043 }
2044 
2045 void CodeGen::mergeDefaultFunctionDefinitionAttributes(
2046     llvm::Function &F, const CodeGenOptions &CodeGenOpts,
2047     const LangOptions &LangOpts, const TargetOptions &TargetOpts,
2048     bool WillInternalize) {
2049 
2050   llvm::AttrBuilder FuncAttrs(F.getContext());
2051   // Here we only extract the options that are relevant compared to the version
2052   // from GetCPUAndFeaturesAttributes.
2053   if (!TargetOpts.CPU.empty())
2054     FuncAttrs.addAttribute("target-cpu", TargetOpts.CPU);
2055   if (!TargetOpts.TuneCPU.empty())
2056     FuncAttrs.addAttribute("tune-cpu", TargetOpts.TuneCPU);
2057 
2058   ::getTrivialDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
2059                                         CodeGenOpts, LangOpts,
2060                                         /*AttrOnCallSite=*/false, FuncAttrs);
2061 
2062   if (!WillInternalize && F.isInterposable()) {
2063     // Do not promote "dynamic" denormal-fp-math to this translation unit's
2064     // setting for weak functions that won't be internalized. The user has no
2065     // real control for how builtin bitcode is linked, so we shouldn't assume
2066     // later copies will use a consistent mode.
2067     F.addFnAttrs(FuncAttrs);
2068     return;
2069   }
2070 
2071   llvm::AttributeMask AttrsToRemove;
2072 
2073   llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();
2074   llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();
2075   llvm::DenormalMode Merged =
2076       CodeGenOpts.FPDenormalMode.mergeCalleeMode(DenormModeToMerge);
2077   llvm::DenormalMode MergedF32 = CodeGenOpts.FP32DenormalMode;
2078 
2079   if (DenormModeToMergeF32.isValid()) {
2080     MergedF32 =
2081         CodeGenOpts.FP32DenormalMode.mergeCalleeMode(DenormModeToMergeF32);
2082   }
2083 
2084   if (Merged == llvm::DenormalMode::getDefault()) {
2085     AttrsToRemove.addAttribute("denormal-fp-math");
2086   } else if (Merged != DenormModeToMerge) {
2087     // Overwrite existing attribute
2088     FuncAttrs.addAttribute("denormal-fp-math",
2089                            CodeGenOpts.FPDenormalMode.str());
2090   }
2091 
2092   if (MergedF32 == llvm::DenormalMode::getDefault()) {
2093     AttrsToRemove.addAttribute("denormal-fp-math-f32");
2094   } else if (MergedF32 != DenormModeToMergeF32) {
2095     // Overwrite existing attribute
2096     FuncAttrs.addAttribute("denormal-fp-math-f32",
2097                            CodeGenOpts.FP32DenormalMode.str());
2098   }
2099 
2100   F.removeFnAttrs(AttrsToRemove);
2101   addDenormalModeAttrs(Merged, MergedF32, FuncAttrs);
2102 
2103   overrideFunctionFeaturesWithTargetFeatures(FuncAttrs, F, TargetOpts);
2104 
2105   F.addFnAttrs(FuncAttrs);
2106 }
2107 
2108 void CodeGenModule::getTrivialDefaultFunctionAttributes(
2109     StringRef Name, bool HasOptnone, bool AttrOnCallSite,
2110     llvm::AttrBuilder &FuncAttrs) {
2111   ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, getCodeGenOpts(),
2112                                         getLangOpts(), AttrOnCallSite,
2113                                         FuncAttrs);
2114 }
2115 
2116 void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
2117                                                  bool HasOptnone,
2118                                                  bool AttrOnCallSite,
2119                                                  llvm::AttrBuilder &FuncAttrs) {
2120   getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
2121                                       FuncAttrs);
2122   // If we're just getting the default, get the default values for mergeable
2123   // attributes.
2124   if (!AttrOnCallSite)
2125     addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
2126 }
2127 
2128 void CodeGenModule::addDefaultFunctionDefinitionAttributes(
2129     llvm::AttrBuilder &attrs) {
2130   getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
2131                                /*for call*/ false, attrs);
2132   GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
2133 }
2134 
2135 static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
2136                                    const LangOptions &LangOpts,
2137                                    const NoBuiltinAttr *NBA = nullptr) {
2138   auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
2139     SmallString<32> AttributeName;
2140     AttributeName += "no-builtin-";
2141     AttributeName += BuiltinName;
2142     FuncAttrs.addAttribute(AttributeName);
2143   };
2144 
2145   // First, handle the language options passed through -fno-builtin.
2146   if (LangOpts.NoBuiltin) {
2147     // -fno-builtin disables them all.
2148     FuncAttrs.addAttribute("no-builtins");
2149     return;
2150   }
2151 
2152   // Then, add attributes for builtins specified through -fno-builtin-<name>.
2153   llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
2154 
2155   // Now, let's check the __attribute__((no_builtin("...")) attribute added to
2156   // the source.
2157   if (!NBA)
2158     return;
2159 
2160   // If there is a wildcard in the builtin names specified through the
2161   // attribute, disable them all.
2162   if (llvm::is_contained(NBA->builtinNames(), "*")) {
2163     FuncAttrs.addAttribute("no-builtins");
2164     return;
2165   }
2166 
2167   // And last, add the rest of the builtin names.
2168   llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
2169 }
2170 
2171 static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2172                              const llvm::DataLayout &DL, const ABIArgInfo &AI,
2173                              bool CheckCoerce = true) {
2174   llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
2175   if (AI.getKind() == ABIArgInfo::Indirect ||
2176       AI.getKind() == ABIArgInfo::IndirectAliased)
2177     return true;
2178   if (AI.getKind() == ABIArgInfo::Extend)
2179     return true;
2180   if (!DL.typeSizeEqualsStoreSize(Ty))
2181     // TODO: This will result in a modest amount of values not marked noundef
2182     // when they could be. We care about values that *invisibly* contain undef
2183     // bits from the perspective of LLVM IR.
2184     return false;
2185   if (CheckCoerce && AI.canHaveCoerceToType()) {
2186     llvm::Type *CoerceTy = AI.getCoerceToType();
2187     if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
2188                                   DL.getTypeSizeInBits(Ty)))
2189       // If we're coercing to a type with a greater size than the canonical one,
2190       // we're introducing new undef bits.
2191       // Coercing to a type of smaller or equal size is ok, as we know that
2192       // there's no internal padding (typeSizeEqualsStoreSize).
2193       return false;
2194   }
2195   if (QTy->isBitIntType())
2196     return true;
2197   if (QTy->isReferenceType())
2198     return true;
2199   if (QTy->isNullPtrType())
2200     return false;
2201   if (QTy->isMemberPointerType())
2202     // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2203     // now, never mark them.
2204     return false;
2205   if (QTy->isScalarType()) {
2206     if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy))
2207       return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false);
2208     return true;
2209   }
2210   if (const VectorType *Vector = dyn_cast<VectorType>(QTy))
2211     return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false);
2212   if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
2213     return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false);
2214   if (const ArrayType *Array = dyn_cast<ArrayType>(QTy))
2215     return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false);
2216 
2217   // TODO: Some structs may be `noundef`, in specific situations.
2218   return false;
2219 }
2220 
2221 /// Check if the argument of a function has maybe_undef attribute.
2222 static bool IsArgumentMaybeUndef(const Decl *TargetDecl,
2223                                  unsigned NumRequiredArgs, unsigned ArgNo) {
2224   const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl);
2225   if (!FD)
2226     return false;
2227 
2228   // Assume variadic arguments do not have maybe_undef attribute.
2229   if (ArgNo >= NumRequiredArgs)
2230     return false;
2231 
2232   // Check if argument has maybe_undef attribute.
2233   if (ArgNo < FD->getNumParams()) {
2234     const ParmVarDecl *Param = FD->getParamDecl(ArgNo);
2235     if (Param && Param->hasAttr<MaybeUndefAttr>())
2236       return true;
2237   }
2238 
2239   return false;
2240 }
2241 
2242 /// Test if it's legal to apply nofpclass for the given parameter type and it's
2243 /// lowered IR type.
2244 static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,
2245                               bool IsReturn) {
2246   // Should only apply to FP types in the source, not ABI promoted.
2247   if (!ParamType->hasFloatingRepresentation())
2248     return false;
2249 
2250   // The promoted-to IR type also needs to support nofpclass.
2251   llvm::Type *IRTy = AI.getCoerceToType();
2252   if (llvm::AttributeFuncs::isNoFPClassCompatibleType(IRTy))
2253     return true;
2254 
2255   if (llvm::StructType *ST = dyn_cast<llvm::StructType>(IRTy)) {
2256     return !IsReturn && AI.getCanBeFlattened() &&
2257            llvm::all_of(ST->elements(), [](llvm::Type *Ty) {
2258              return llvm::AttributeFuncs::isNoFPClassCompatibleType(Ty);
2259            });
2260   }
2261 
2262   return false;
2263 }
2264 
2265 /// Return the nofpclass mask that can be applied to floating-point parameters.
2266 static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {
2267   llvm::FPClassTest Mask = llvm::fcNone;
2268   if (LangOpts.NoHonorInfs)
2269     Mask |= llvm::fcInf;
2270   if (LangOpts.NoHonorNaNs)
2271     Mask |= llvm::fcNan;
2272   return Mask;
2273 }
2274 
2275 void CodeGenModule::AdjustMemoryAttribute(StringRef Name,
2276                                           CGCalleeInfo CalleeInfo,
2277                                           llvm::AttributeList &Attrs) {
2278   if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {
2279     Attrs = Attrs.removeFnAttribute(getLLVMContext(), llvm::Attribute::Memory);
2280     llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(
2281         getLLVMContext(), llvm::MemoryEffects::writeOnly());
2282     Attrs = Attrs.addFnAttribute(getLLVMContext(), MemoryAttr);
2283   }
2284 }
2285 
2286 /// Construct the IR attribute list of a function or call.
2287 ///
2288 /// When adding an attribute, please consider where it should be handled:
2289 ///
2290 ///   - getDefaultFunctionAttributes is for attributes that are essentially
2291 ///     part of the global target configuration (but perhaps can be
2292 ///     overridden on a per-function basis).  Adding attributes there
2293 ///     will cause them to also be set in frontends that build on Clang's
2294 ///     target-configuration logic, as well as for code defined in library
2295 ///     modules such as CUDA's libdevice.
2296 ///
2297 ///   - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2298 ///     and adds declaration-specific, convention-specific, and
2299 ///     frontend-specific logic.  The last is of particular importance:
2300 ///     attributes that restrict how the frontend generates code must be
2301 ///     added here rather than getDefaultFunctionAttributes.
2302 ///
2303 void CodeGenModule::ConstructAttributeList(StringRef Name,
2304                                            const CGFunctionInfo &FI,
2305                                            CGCalleeInfo CalleeInfo,
2306                                            llvm::AttributeList &AttrList,
2307                                            unsigned &CallingConv,
2308                                            bool AttrOnCallSite, bool IsThunk) {
2309   llvm::AttrBuilder FuncAttrs(getLLVMContext());
2310   llvm::AttrBuilder RetAttrs(getLLVMContext());
2311 
2312   // Collect function IR attributes from the CC lowering.
2313   // We'll collect the paramete and result attributes later.
2314   CallingConv = FI.getEffectiveCallingConvention();
2315   if (FI.isNoReturn())
2316     FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2317   if (FI.isCmseNSCall())
2318     FuncAttrs.addAttribute("cmse_nonsecure_call");
2319 
2320   // Collect function IR attributes from the callee prototype if we have one.
2321   AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
2322                                      CalleeInfo.getCalleeFunctionProtoType());
2323 
2324   const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2325 
2326   // Attach assumption attributes to the declaration. If this is a call
2327   // site, attach assumptions from the caller to the call as well.
2328   AddAttributesFromAssumes(FuncAttrs, TargetDecl);
2329 
2330   bool HasOptnone = false;
2331   // The NoBuiltinAttr attached to the target FunctionDecl.
2332   const NoBuiltinAttr *NBA = nullptr;
2333 
2334   // Some ABIs may result in additional accesses to arguments that may
2335   // otherwise not be present.
2336   auto AddPotentialArgAccess = [&]() {
2337     llvm::Attribute A = FuncAttrs.getAttribute(llvm::Attribute::Memory);
2338     if (A.isValid())
2339       FuncAttrs.addMemoryAttr(A.getMemoryEffects() |
2340                               llvm::MemoryEffects::argMemOnly());
2341   };
2342 
2343   // Collect function IR attributes based on declaration-specific
2344   // information.
2345   // FIXME: handle sseregparm someday...
2346   if (TargetDecl) {
2347     if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2348       FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2349     if (TargetDecl->hasAttr<NoThrowAttr>())
2350       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2351     if (TargetDecl->hasAttr<NoReturnAttr>())
2352       FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2353     if (TargetDecl->hasAttr<ColdAttr>())
2354       FuncAttrs.addAttribute(llvm::Attribute::Cold);
2355     if (TargetDecl->hasAttr<HotAttr>())
2356       FuncAttrs.addAttribute(llvm::Attribute::Hot);
2357     if (TargetDecl->hasAttr<NoDuplicateAttr>())
2358       FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2359     if (TargetDecl->hasAttr<ConvergentAttr>())
2360       FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2361 
2362     if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2363       AddAttributesFromFunctionProtoType(
2364           getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2365       if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
2366         // A sane operator new returns a non-aliasing pointer.
2367         auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2368         if (getCodeGenOpts().AssumeSaneOperatorNew &&
2369             (Kind == OO_New || Kind == OO_Array_New))
2370           RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2371       }
2372       const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
2373       const bool IsVirtualCall = MD && MD->isVirtual();
2374       // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2375       // virtual function. These attributes are not inherited by overloads.
2376       if (!(AttrOnCallSite && IsVirtualCall)) {
2377         if (Fn->isNoReturn())
2378           FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2379         NBA = Fn->getAttr<NoBuiltinAttr>();
2380       }
2381     }
2382 
2383     if (isa<FunctionDecl>(TargetDecl) || isa<VarDecl>(TargetDecl)) {
2384       // Only place nomerge attribute on call sites, never functions. This
2385       // allows it to work on indirect virtual function calls.
2386       if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())
2387         FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2388     }
2389 
2390     // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2391     if (TargetDecl->hasAttr<ConstAttr>()) {
2392       FuncAttrs.addMemoryAttr(llvm::MemoryEffects::none());
2393       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2394       // gcc specifies that 'const' functions have greater restrictions than
2395       // 'pure' functions, so they also cannot have infinite loops.
2396       FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2397     } else if (TargetDecl->hasAttr<PureAttr>()) {
2398       FuncAttrs.addMemoryAttr(llvm::MemoryEffects::readOnly());
2399       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2400       // gcc specifies that 'pure' functions cannot have infinite loops.
2401       FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2402     } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2403       FuncAttrs.addMemoryAttr(llvm::MemoryEffects::inaccessibleOrArgMemOnly());
2404       FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2405     }
2406     if (TargetDecl->hasAttr<RestrictAttr>())
2407       RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2408     if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2409         !CodeGenOpts.NullPointerIsValid)
2410       RetAttrs.addAttribute(llvm::Attribute::NonNull);
2411     if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2412       FuncAttrs.addAttribute("no_caller_saved_registers");
2413     if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2414       FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2415     if (TargetDecl->hasAttr<LeafAttr>())
2416       FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2417 
2418     HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2419     if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2420       std::optional<unsigned> NumElemsParam;
2421       if (AllocSize->getNumElemsParam().isValid())
2422         NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2423       FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2424                                  NumElemsParam);
2425     }
2426 
2427     if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2428       if (getLangOpts().OpenCLVersion <= 120) {
2429         // OpenCL v1.2 Work groups are always uniform
2430         FuncAttrs.addAttribute("uniform-work-group-size", "true");
2431       } else {
2432         // OpenCL v2.0 Work groups may be whether uniform or not.
2433         // '-cl-uniform-work-group-size' compile option gets a hint
2434         // to the compiler that the global work-size be a multiple of
2435         // the work-group size specified to clEnqueueNDRangeKernel
2436         // (i.e. work groups are uniform).
2437         FuncAttrs.addAttribute(
2438             "uniform-work-group-size",
2439             llvm::toStringRef(getLangOpts().OffloadUniformBlock));
2440       }
2441     }
2442 
2443     if (TargetDecl->hasAttr<CUDAGlobalAttr>() &&
2444         getLangOpts().OffloadUniformBlock)
2445       FuncAttrs.addAttribute("uniform-work-group-size", "true");
2446 
2447     if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())
2448       FuncAttrs.addAttribute("aarch64_pstate_sm_body");
2449 
2450     if (TargetDecl->hasAttr<ArmNewZAAttr>())
2451       FuncAttrs.addAttribute("aarch64_pstate_za_new");
2452   }
2453 
2454   // Attach "no-builtins" attributes to:
2455   // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2456   // * definitions: "no-builtins" or "no-builtin-<name>" only.
2457   // The attributes can come from:
2458   // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2459   // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2460   addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2461 
2462   // Collect function IR attributes based on global settiings.
2463   getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2464 
2465   // Override some default IR attributes based on declaration-specific
2466   // information.
2467   if (TargetDecl) {
2468     if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2469       FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2470     if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2471       FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2472     if (TargetDecl->hasAttr<NoSplitStackAttr>())
2473       FuncAttrs.removeAttribute("split-stack");
2474     if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2475       // A function "__attribute__((...))" overrides the command-line flag.
2476       auto Kind =
2477           TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2478       FuncAttrs.removeAttribute("zero-call-used-regs");
2479       FuncAttrs.addAttribute(
2480           "zero-call-used-regs",
2481           ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2482     }
2483 
2484     // Add NonLazyBind attribute to function declarations when -fno-plt
2485     // is used.
2486     // FIXME: what if we just haven't processed the function definition
2487     // yet, or if it's an external definition like C99 inline?
2488     if (CodeGenOpts.NoPLT) {
2489       if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2490         if (!Fn->isDefined() && !AttrOnCallSite) {
2491           FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2492         }
2493       }
2494     }
2495   }
2496 
2497   // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2498   // functions with -funique-internal-linkage-names.
2499   if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {
2500     if (const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
2501       if (!FD->isExternallyVisible())
2502         FuncAttrs.addAttribute("sample-profile-suffix-elision-policy",
2503                                "selected");
2504     }
2505   }
2506 
2507   // Collect non-call-site function IR attributes from declaration-specific
2508   // information.
2509   if (!AttrOnCallSite) {
2510     if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2511       FuncAttrs.addAttribute("cmse_nonsecure_entry");
2512 
2513     // Whether tail calls are enabled.
2514     auto shouldDisableTailCalls = [&] {
2515       // Should this be honored in getDefaultFunctionAttributes?
2516       if (CodeGenOpts.DisableTailCalls)
2517         return true;
2518 
2519       if (!TargetDecl)
2520         return false;
2521 
2522       if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2523           TargetDecl->hasAttr<AnyX86InterruptAttr>())
2524         return true;
2525 
2526       if (CodeGenOpts.NoEscapingBlockTailCalls) {
2527         if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2528           if (!BD->doesNotEscape())
2529             return true;
2530       }
2531 
2532       return false;
2533     };
2534     if (shouldDisableTailCalls())
2535       FuncAttrs.addAttribute("disable-tail-calls", "true");
2536 
2537     // CPU/feature overrides.  addDefaultFunctionDefinitionAttributes
2538     // handles these separately to set them based on the global defaults.
2539     GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2540   }
2541 
2542   // Collect attributes from arguments and return values.
2543   ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2544 
2545   QualType RetTy = FI.getReturnType();
2546   const ABIArgInfo &RetAI = FI.getReturnInfo();
2547   const llvm::DataLayout &DL = getDataLayout();
2548 
2549   // Determine if the return type could be partially undef
2550   if (CodeGenOpts.EnableNoundefAttrs &&
2551       HasStrictReturn(*this, RetTy, TargetDecl)) {
2552     if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&
2553         DetermineNoUndef(RetTy, getTypes(), DL, RetAI))
2554       RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2555   }
2556 
2557   switch (RetAI.getKind()) {
2558   case ABIArgInfo::Extend:
2559     if (RetAI.isSignExt())
2560       RetAttrs.addAttribute(llvm::Attribute::SExt);
2561     else
2562       RetAttrs.addAttribute(llvm::Attribute::ZExt);
2563     [[fallthrough]];
2564   case ABIArgInfo::Direct:
2565     if (RetAI.getInReg())
2566       RetAttrs.addAttribute(llvm::Attribute::InReg);
2567 
2568     if (canApplyNoFPClass(RetAI, RetTy, true))
2569       RetAttrs.addNoFPClassAttr(getNoFPClassTestMask(getLangOpts()));
2570 
2571     break;
2572   case ABIArgInfo::Ignore:
2573     break;
2574 
2575   case ABIArgInfo::InAlloca:
2576   case ABIArgInfo::Indirect: {
2577     // inalloca and sret disable readnone and readonly
2578     AddPotentialArgAccess();
2579     break;
2580   }
2581 
2582   case ABIArgInfo::CoerceAndExpand:
2583     break;
2584 
2585   case ABIArgInfo::Expand:
2586   case ABIArgInfo::IndirectAliased:
2587     llvm_unreachable("Invalid ABI kind for return argument");
2588   }
2589 
2590   if (!IsThunk) {
2591     // FIXME: fix this properly, https://reviews.llvm.org/D100388
2592     if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2593       QualType PTy = RefTy->getPointeeType();
2594       if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2595         RetAttrs.addDereferenceableAttr(
2596             getMinimumObjectSize(PTy).getQuantity());
2597       if (getTypes().getTargetAddressSpace(PTy) == 0 &&
2598           !CodeGenOpts.NullPointerIsValid)
2599         RetAttrs.addAttribute(llvm::Attribute::NonNull);
2600       if (PTy->isObjectType()) {
2601         llvm::Align Alignment =
2602             getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
2603         RetAttrs.addAlignmentAttr(Alignment);
2604       }
2605     }
2606   }
2607 
2608   bool hasUsedSRet = false;
2609   SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2610 
2611   // Attach attributes to sret.
2612   if (IRFunctionArgs.hasSRetArg()) {
2613     llvm::AttrBuilder SRETAttrs(getLLVMContext());
2614     SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2615     hasUsedSRet = true;
2616     if (RetAI.getInReg())
2617       SRETAttrs.addAttribute(llvm::Attribute::InReg);
2618     SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2619     ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2620         llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2621   }
2622 
2623   // Attach attributes to inalloca argument.
2624   if (IRFunctionArgs.hasInallocaArg()) {
2625     llvm::AttrBuilder Attrs(getLLVMContext());
2626     Attrs.addInAllocaAttr(FI.getArgStruct());
2627     ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2628         llvm::AttributeSet::get(getLLVMContext(), Attrs);
2629   }
2630 
2631   // Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument,
2632   // unless this is a thunk function.
2633   // FIXME: fix this properly, https://reviews.llvm.org/D100388
2634   if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2635       !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {
2636     auto IRArgs = IRFunctionArgs.getIRArgs(0);
2637 
2638     assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2639 
2640     llvm::AttrBuilder Attrs(getLLVMContext());
2641 
2642     QualType ThisTy =
2643         FI.arg_begin()->type.getTypePtr()->getPointeeType();
2644 
2645     if (!CodeGenOpts.NullPointerIsValid &&
2646         getTypes().getTargetAddressSpace(FI.arg_begin()->type) == 0) {
2647       Attrs.addAttribute(llvm::Attribute::NonNull);
2648       Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity());
2649     } else {
2650       // FIXME dereferenceable should be correct here, regardless of
2651       // NullPointerIsValid. However, dereferenceable currently does not always
2652       // respect NullPointerIsValid and may imply nonnull and break the program.
2653       // See https://reviews.llvm.org/D66618 for discussions.
2654       Attrs.addDereferenceableOrNullAttr(
2655           getMinimumObjectSize(
2656               FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2657               .getQuantity());
2658     }
2659 
2660     llvm::Align Alignment =
2661         getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr,
2662                                 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2663             .getAsAlign();
2664     Attrs.addAlignmentAttr(Alignment);
2665 
2666     ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2667   }
2668 
2669   unsigned ArgNo = 0;
2670   for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2671                                           E = FI.arg_end();
2672        I != E; ++I, ++ArgNo) {
2673     QualType ParamType = I->type;
2674     const ABIArgInfo &AI = I->info;
2675     llvm::AttrBuilder Attrs(getLLVMContext());
2676 
2677     // Add attribute for padding argument, if necessary.
2678     if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2679       if (AI.getPaddingInReg()) {
2680         ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2681             llvm::AttributeSet::get(
2682                 getLLVMContext(),
2683                 llvm::AttrBuilder(getLLVMContext()).addAttribute(llvm::Attribute::InReg));
2684       }
2685     }
2686 
2687     // Decide whether the argument we're handling could be partially undef
2688     if (CodeGenOpts.EnableNoundefAttrs &&
2689         DetermineNoUndef(ParamType, getTypes(), DL, AI)) {
2690       Attrs.addAttribute(llvm::Attribute::NoUndef);
2691     }
2692 
2693     // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2694     // have the corresponding parameter variable.  It doesn't make
2695     // sense to do it here because parameters are so messed up.
2696     switch (AI.getKind()) {
2697     case ABIArgInfo::Extend:
2698       if (AI.isSignExt())
2699         Attrs.addAttribute(llvm::Attribute::SExt);
2700       else
2701         Attrs.addAttribute(llvm::Attribute::ZExt);
2702       [[fallthrough]];
2703     case ABIArgInfo::Direct:
2704       if (ArgNo == 0 && FI.isChainCall())
2705         Attrs.addAttribute(llvm::Attribute::Nest);
2706       else if (AI.getInReg())
2707         Attrs.addAttribute(llvm::Attribute::InReg);
2708       Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign()));
2709 
2710       if (canApplyNoFPClass(AI, ParamType, false))
2711         Attrs.addNoFPClassAttr(getNoFPClassTestMask(getLangOpts()));
2712       break;
2713     case ABIArgInfo::Indirect: {
2714       if (AI.getInReg())
2715         Attrs.addAttribute(llvm::Attribute::InReg);
2716 
2717       if (AI.getIndirectByVal())
2718         Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2719 
2720       auto *Decl = ParamType->getAsRecordDecl();
2721       if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2722           Decl->getArgPassingRestrictions() ==
2723               RecordArgPassingKind::CanPassInRegs)
2724         // When calling the function, the pointer passed in will be the only
2725         // reference to the underlying object. Mark it accordingly.
2726         Attrs.addAttribute(llvm::Attribute::NoAlias);
2727 
2728       // TODO: We could add the byref attribute if not byval, but it would
2729       // require updating many testcases.
2730 
2731       CharUnits Align = AI.getIndirectAlign();
2732 
2733       // In a byval argument, it is important that the required
2734       // alignment of the type is honored, as LLVM might be creating a
2735       // *new* stack object, and needs to know what alignment to give
2736       // it. (Sometimes it can deduce a sensible alignment on its own,
2737       // but not if clang decides it must emit a packed struct, or the
2738       // user specifies increased alignment requirements.)
2739       //
2740       // This is different from indirect *not* byval, where the object
2741       // exists already, and the align attribute is purely
2742       // informative.
2743       assert(!Align.isZero());
2744 
2745       // For now, only add this when we have a byval argument.
2746       // TODO: be less lazy about updating test cases.
2747       if (AI.getIndirectByVal())
2748         Attrs.addAlignmentAttr(Align.getQuantity());
2749 
2750       // byval disables readnone and readonly.
2751       AddPotentialArgAccess();
2752       break;
2753     }
2754     case ABIArgInfo::IndirectAliased: {
2755       CharUnits Align = AI.getIndirectAlign();
2756       Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2757       Attrs.addAlignmentAttr(Align.getQuantity());
2758       break;
2759     }
2760     case ABIArgInfo::Ignore:
2761     case ABIArgInfo::Expand:
2762     case ABIArgInfo::CoerceAndExpand:
2763       break;
2764 
2765     case ABIArgInfo::InAlloca:
2766       // inalloca disables readnone and readonly.
2767       AddPotentialArgAccess();
2768       continue;
2769     }
2770 
2771     if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2772       QualType PTy = RefTy->getPointeeType();
2773       if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2774         Attrs.addDereferenceableAttr(
2775             getMinimumObjectSize(PTy).getQuantity());
2776       if (getTypes().getTargetAddressSpace(PTy) == 0 &&
2777           !CodeGenOpts.NullPointerIsValid)
2778         Attrs.addAttribute(llvm::Attribute::NonNull);
2779       if (PTy->isObjectType()) {
2780         llvm::Align Alignment =
2781             getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2782         Attrs.addAlignmentAttr(Alignment);
2783       }
2784     }
2785 
2786     // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2787     // > For arguments to a __kernel function declared to be a pointer to a
2788     // > data type, the OpenCL compiler can assume that the pointee is always
2789     // > appropriately aligned as required by the data type.
2790     if (TargetDecl && TargetDecl->hasAttr<OpenCLKernelAttr>() &&
2791         ParamType->isPointerType()) {
2792       QualType PTy = ParamType->getPointeeType();
2793       if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2794         llvm::Align Alignment =
2795             getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2796         Attrs.addAlignmentAttr(Alignment);
2797       }
2798     }
2799 
2800     switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2801     case ParameterABI::Ordinary:
2802       break;
2803 
2804     case ParameterABI::SwiftIndirectResult: {
2805       // Add 'sret' if we haven't already used it for something, but
2806       // only if the result is void.
2807       if (!hasUsedSRet && RetTy->isVoidType()) {
2808         Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2809         hasUsedSRet = true;
2810       }
2811 
2812       // Add 'noalias' in either case.
2813       Attrs.addAttribute(llvm::Attribute::NoAlias);
2814 
2815       // Add 'dereferenceable' and 'alignment'.
2816       auto PTy = ParamType->getPointeeType();
2817       if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2818         auto info = getContext().getTypeInfoInChars(PTy);
2819         Attrs.addDereferenceableAttr(info.Width.getQuantity());
2820         Attrs.addAlignmentAttr(info.Align.getAsAlign());
2821       }
2822       break;
2823     }
2824 
2825     case ParameterABI::SwiftErrorResult:
2826       Attrs.addAttribute(llvm::Attribute::SwiftError);
2827       break;
2828 
2829     case ParameterABI::SwiftContext:
2830       Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2831       break;
2832 
2833     case ParameterABI::SwiftAsyncContext:
2834       Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2835       break;
2836     }
2837 
2838     if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2839       Attrs.addAttribute(llvm::Attribute::NoCapture);
2840 
2841     if (Attrs.hasAttributes()) {
2842       unsigned FirstIRArg, NumIRArgs;
2843       std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2844       for (unsigned i = 0; i < NumIRArgs; i++)
2845         ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
2846             getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), Attrs));
2847     }
2848   }
2849   assert(ArgNo == FI.arg_size());
2850 
2851   AttrList = llvm::AttributeList::get(
2852       getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2853       llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2854 }
2855 
2856 /// An argument came in as a promoted argument; demote it back to its
2857 /// declared type.
2858 static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2859                                          const VarDecl *var,
2860                                          llvm::Value *value) {
2861   llvm::Type *varType = CGF.ConvertType(var->getType());
2862 
2863   // This can happen with promotions that actually don't change the
2864   // underlying type, like the enum promotions.
2865   if (value->getType() == varType) return value;
2866 
2867   assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2868          && "unexpected promotion type");
2869 
2870   if (isa<llvm::IntegerType>(varType))
2871     return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2872 
2873   return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2874 }
2875 
2876 /// Returns the attribute (either parameter attribute, or function
2877 /// attribute), which declares argument ArgNo to be non-null.
2878 static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2879                                          QualType ArgType, unsigned ArgNo) {
2880   // FIXME: __attribute__((nonnull)) can also be applied to:
2881   //   - references to pointers, where the pointee is known to be
2882   //     nonnull (apparently a Clang extension)
2883   //   - transparent unions containing pointers
2884   // In the former case, LLVM IR cannot represent the constraint. In
2885   // the latter case, we have no guarantee that the transparent union
2886   // is in fact passed as a pointer.
2887   if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
2888     return nullptr;
2889   // First, check attribute on parameter itself.
2890   if (PVD) {
2891     if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2892       return ParmNNAttr;
2893   }
2894   // Check function attributes.
2895   if (!FD)
2896     return nullptr;
2897   for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2898     if (NNAttr->isNonNull(ArgNo))
2899       return NNAttr;
2900   }
2901   return nullptr;
2902 }
2903 
2904 namespace {
2905   struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2906     Address Temp;
2907     Address Arg;
2908     CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2909     void Emit(CodeGenFunction &CGF, Flags flags) override {
2910       llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2911       CGF.Builder.CreateStore(errorValue, Arg);
2912     }
2913   };
2914 }
2915 
2916 void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2917                                          llvm::Function *Fn,
2918                                          const FunctionArgList &Args) {
2919   if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
2920     // Naked functions don't have prologues.
2921     return;
2922 
2923   // If this is an implicit-return-zero function, go ahead and
2924   // initialize the return value.  TODO: it might be nice to have
2925   // a more general mechanism for this that didn't require synthesized
2926   // return statements.
2927   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2928     if (FD->hasImplicitReturnZero()) {
2929       QualType RetTy = FD->getReturnType().getUnqualifiedType();
2930       llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2931       llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2932       Builder.CreateStore(Zero, ReturnValue);
2933     }
2934   }
2935 
2936   // FIXME: We no longer need the types from FunctionArgList; lift up and
2937   // simplify.
2938 
2939   ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2940   assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2941 
2942   // If we're using inalloca, all the memory arguments are GEPs off of the last
2943   // parameter, which is a pointer to the complete memory area.
2944   Address ArgStruct = Address::invalid();
2945   if (IRFunctionArgs.hasInallocaArg())
2946     ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2947                         FI.getArgStruct(), FI.getArgStructAlignment());
2948 
2949   // Name the struct return parameter.
2950   if (IRFunctionArgs.hasSRetArg()) {
2951     auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2952     AI->setName("agg.result");
2953     AI->addAttr(llvm::Attribute::NoAlias);
2954   }
2955 
2956   // Track if we received the parameter as a pointer (indirect, byval, or
2957   // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
2958   // into a local alloca for us.
2959   SmallVector<ParamValue, 16> ArgVals;
2960   ArgVals.reserve(Args.size());
2961 
2962   // Create a pointer value for every parameter declaration.  This usually
2963   // entails copying one or more LLVM IR arguments into an alloca.  Don't push
2964   // any cleanups or do anything that might unwind.  We do that separately, so
2965   // we can push the cleanups in the correct order for the ABI.
2966   assert(FI.arg_size() == Args.size() &&
2967          "Mismatch between function signature & arguments.");
2968   unsigned ArgNo = 0;
2969   CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2970   for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2971        i != e; ++i, ++info_it, ++ArgNo) {
2972     const VarDecl *Arg = *i;
2973     const ABIArgInfo &ArgI = info_it->info;
2974 
2975     bool isPromoted =
2976       isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
2977     // We are converting from ABIArgInfo type to VarDecl type directly, unless
2978     // the parameter is promoted. In this case we convert to
2979     // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2980     QualType Ty = isPromoted ? info_it->type : Arg->getType();
2981     assert(hasScalarEvaluationKind(Ty) ==
2982            hasScalarEvaluationKind(Arg->getType()));
2983 
2984     unsigned FirstIRArg, NumIRArgs;
2985     std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2986 
2987     switch (ArgI.getKind()) {
2988     case ABIArgInfo::InAlloca: {
2989       assert(NumIRArgs == 0);
2990       auto FieldIndex = ArgI.getInAllocaFieldIndex();
2991       Address V =
2992           Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2993       if (ArgI.getInAllocaIndirect())
2994         V = Address(Builder.CreateLoad(V), ConvertTypeForMem(Ty),
2995                     getContext().getTypeAlignInChars(Ty));
2996       ArgVals.push_back(ParamValue::forIndirect(V));
2997       break;
2998     }
2999 
3000     case ABIArgInfo::Indirect:
3001     case ABIArgInfo::IndirectAliased: {
3002       assert(NumIRArgs == 1);
3003       Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty),
3004                                   ArgI.getIndirectAlign(), KnownNonNull);
3005 
3006       if (!hasScalarEvaluationKind(Ty)) {
3007         // Aggregates and complex variables are accessed by reference. All we
3008         // need to do is realign the value, if requested. Also, if the address
3009         // may be aliased, copy it to ensure that the parameter variable is
3010         // mutable and has a unique adress, as C requires.
3011         Address V = ParamAddr;
3012         if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
3013           Address AlignedTemp = CreateMemTemp(Ty, "coerce");
3014 
3015           // Copy from the incoming argument pointer to the temporary with the
3016           // appropriate alignment.
3017           //
3018           // FIXME: We should have a common utility for generating an aggregate
3019           // copy.
3020           CharUnits Size = getContext().getTypeSizeInChars(Ty);
3021           Builder.CreateMemCpy(
3022               AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
3023               ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
3024               llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
3025           V = AlignedTemp;
3026         }
3027         ArgVals.push_back(ParamValue::forIndirect(V));
3028       } else {
3029         // Load scalar value from indirect argument.
3030         llvm::Value *V =
3031             EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
3032 
3033         if (isPromoted)
3034           V = emitArgumentDemotion(*this, Arg, V);
3035         ArgVals.push_back(ParamValue::forDirect(V));
3036       }
3037       break;
3038     }
3039 
3040     case ABIArgInfo::Extend:
3041     case ABIArgInfo::Direct: {
3042       auto AI = Fn->getArg(FirstIRArg);
3043       llvm::Type *LTy = ConvertType(Arg->getType());
3044 
3045       // Prepare parameter attributes. So far, only attributes for pointer
3046       // parameters are prepared. See
3047       // http://llvm.org/docs/LangRef.html#paramattrs.
3048       if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
3049           ArgI.getCoerceToType()->isPointerTy()) {
3050         assert(NumIRArgs == 1);
3051 
3052         if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
3053           // Set `nonnull` attribute if any.
3054           if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
3055                              PVD->getFunctionScopeIndex()) &&
3056               !CGM.getCodeGenOpts().NullPointerIsValid)
3057             AI->addAttr(llvm::Attribute::NonNull);
3058 
3059           QualType OTy = PVD->getOriginalType();
3060           if (const auto *ArrTy =
3061               getContext().getAsConstantArrayType(OTy)) {
3062             // A C99 array parameter declaration with the static keyword also
3063             // indicates dereferenceability, and if the size is constant we can
3064             // use the dereferenceable attribute (which requires the size in
3065             // bytes).
3066             if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3067               QualType ETy = ArrTy->getElementType();
3068               llvm::Align Alignment =
3069                   CGM.getNaturalTypeAlignment(ETy).getAsAlign();
3070               AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
3071               uint64_t ArrSize = ArrTy->getSize().getZExtValue();
3072               if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
3073                   ArrSize) {
3074                 llvm::AttrBuilder Attrs(getLLVMContext());
3075                 Attrs.addDereferenceableAttr(
3076                     getContext().getTypeSizeInChars(ETy).getQuantity() *
3077                     ArrSize);
3078                 AI->addAttrs(Attrs);
3079               } else if (getContext().getTargetInfo().getNullPointerValue(
3080                              ETy.getAddressSpace()) == 0 &&
3081                          !CGM.getCodeGenOpts().NullPointerIsValid) {
3082                 AI->addAttr(llvm::Attribute::NonNull);
3083               }
3084             }
3085           } else if (const auto *ArrTy =
3086                      getContext().getAsVariableArrayType(OTy)) {
3087             // For C99 VLAs with the static keyword, we don't know the size so
3088             // we can't use the dereferenceable attribute, but in addrspace(0)
3089             // we know that it must be nonnull.
3090             if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {
3091               QualType ETy = ArrTy->getElementType();
3092               llvm::Align Alignment =
3093                   CGM.getNaturalTypeAlignment(ETy).getAsAlign();
3094               AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
3095               if (!getTypes().getTargetAddressSpace(ETy) &&
3096                   !CGM.getCodeGenOpts().NullPointerIsValid)
3097                 AI->addAttr(llvm::Attribute::NonNull);
3098             }
3099           }
3100 
3101           // Set `align` attribute if any.
3102           const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
3103           if (!AVAttr)
3104             if (const auto *TOTy = OTy->getAs<TypedefType>())
3105               AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
3106           if (AVAttr && !SanOpts.has(SanitizerKind::Alignment)) {
3107             // If alignment-assumption sanitizer is enabled, we do *not* add
3108             // alignment attribute here, but emit normal alignment assumption,
3109             // so the UBSAN check could function.
3110             llvm::ConstantInt *AlignmentCI =
3111                 cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
3112             uint64_t AlignmentInt =
3113                 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
3114             if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
3115               AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
3116               AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(
3117                   llvm::Align(AlignmentInt)));
3118             }
3119           }
3120         }
3121 
3122         // Set 'noalias' if an argument type has the `restrict` qualifier.
3123         if (Arg->getType().isRestrictQualified())
3124           AI->addAttr(llvm::Attribute::NoAlias);
3125       }
3126 
3127       // Prepare the argument value. If we have the trivial case, handle it
3128       // with no muss and fuss.
3129       if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
3130           ArgI.getCoerceToType() == ConvertType(Ty) &&
3131           ArgI.getDirectOffset() == 0) {
3132         assert(NumIRArgs == 1);
3133 
3134         // LLVM expects swifterror parameters to be used in very restricted
3135         // ways.  Copy the value into a less-restricted temporary.
3136         llvm::Value *V = AI;
3137         if (FI.getExtParameterInfo(ArgNo).getABI()
3138               == ParameterABI::SwiftErrorResult) {
3139           QualType pointeeTy = Ty->getPointeeType();
3140           assert(pointeeTy->isPointerType());
3141           Address temp =
3142             CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
3143           Address arg(V, ConvertTypeForMem(pointeeTy),
3144                       getContext().getTypeAlignInChars(pointeeTy));
3145           llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
3146           Builder.CreateStore(incomingErrorValue, temp);
3147           V = temp.getPointer();
3148 
3149           // Push a cleanup to copy the value back at the end of the function.
3150           // The convention does not guarantee that the value will be written
3151           // back if the function exits with an unwind exception.
3152           EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
3153         }
3154 
3155         // Ensure the argument is the correct type.
3156         if (V->getType() != ArgI.getCoerceToType())
3157           V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
3158 
3159         if (isPromoted)
3160           V = emitArgumentDemotion(*this, Arg, V);
3161 
3162         // Because of merging of function types from multiple decls it is
3163         // possible for the type of an argument to not match the corresponding
3164         // type in the function type. Since we are codegening the callee
3165         // in here, add a cast to the argument type.
3166         llvm::Type *LTy = ConvertType(Arg->getType());
3167         if (V->getType() != LTy)
3168           V = Builder.CreateBitCast(V, LTy);
3169 
3170         ArgVals.push_back(ParamValue::forDirect(V));
3171         break;
3172       }
3173 
3174       // VLST arguments are coerced to VLATs at the function boundary for
3175       // ABI consistency. If this is a VLST that was coerced to
3176       // a VLAT at the function boundary and the types match up, use
3177       // llvm.vector.extract to convert back to the original VLST.
3178       if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
3179         llvm::Value *Coerced = Fn->getArg(FirstIRArg);
3180         if (auto *VecTyFrom =
3181                 dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
3182           // If we are casting a scalable 16 x i1 predicate vector to a fixed i8
3183           // vector, bitcast the source and use a vector extract.
3184           auto PredType =
3185               llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
3186           if (VecTyFrom == PredType &&
3187               VecTyTo->getElementType() == Builder.getInt8Ty()) {
3188             VecTyFrom = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2);
3189             Coerced = Builder.CreateBitCast(Coerced, VecTyFrom);
3190           }
3191           if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
3192             llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
3193 
3194             assert(NumIRArgs == 1);
3195             Coerced->setName(Arg->getName() + ".coerce");
3196             ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector(
3197                 VecTyTo, Coerced, Zero, "cast.fixed")));
3198             break;
3199           }
3200         }
3201       }
3202 
3203       Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
3204                                      Arg->getName());
3205 
3206       // Pointer to store into.
3207       Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
3208 
3209       // Fast-isel and the optimizer generally like scalar values better than
3210       // FCAs, so we flatten them if this is safe to do for this argument.
3211       llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
3212       if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
3213           STy->getNumElements() > 1) {
3214         llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(STy);
3215         llvm::TypeSize PtrElementSize =
3216             CGM.getDataLayout().getTypeAllocSize(Ptr.getElementType());
3217         if (StructSize.isScalable()) {
3218           assert(STy->containsHomogeneousScalableVectorTypes() &&
3219                  "ABI only supports structure with homogeneous scalable vector "
3220                  "type");
3221           assert(StructSize == PtrElementSize &&
3222                  "Only allow non-fractional movement of structure with"
3223                  "homogeneous scalable vector type");
3224           assert(STy->getNumElements() == NumIRArgs);
3225 
3226           llvm::Value *LoadedStructValue = llvm::PoisonValue::get(STy);
3227           for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3228             auto *AI = Fn->getArg(FirstIRArg + i);
3229             AI->setName(Arg->getName() + ".coerce" + Twine(i));
3230             LoadedStructValue =
3231                 Builder.CreateInsertValue(LoadedStructValue, AI, i);
3232           }
3233 
3234           Builder.CreateStore(LoadedStructValue, Ptr);
3235         } else {
3236           uint64_t SrcSize = StructSize.getFixedValue();
3237           uint64_t DstSize = PtrElementSize.getFixedValue();
3238 
3239           Address AddrToStoreInto = Address::invalid();
3240           if (SrcSize <= DstSize) {
3241             AddrToStoreInto = Ptr.withElementType(STy);
3242           } else {
3243             AddrToStoreInto =
3244                 CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
3245           }
3246 
3247           assert(STy->getNumElements() == NumIRArgs);
3248           for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3249             auto AI = Fn->getArg(FirstIRArg + i);
3250             AI->setName(Arg->getName() + ".coerce" + Twine(i));
3251             Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
3252             Builder.CreateStore(AI, EltPtr);
3253           }
3254 
3255           if (SrcSize > DstSize) {
3256             Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
3257           }
3258         }
3259       } else {
3260         // Simple case, just do a coerced store of the argument into the alloca.
3261         assert(NumIRArgs == 1);
3262         auto AI = Fn->getArg(FirstIRArg);
3263         AI->setName(Arg->getName() + ".coerce");
3264         CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
3265       }
3266 
3267       // Match to what EmitParmDecl is expecting for this type.
3268       if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
3269         llvm::Value *V =
3270             EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
3271         if (isPromoted)
3272           V = emitArgumentDemotion(*this, Arg, V);
3273         ArgVals.push_back(ParamValue::forDirect(V));
3274       } else {
3275         ArgVals.push_back(ParamValue::forIndirect(Alloca));
3276       }
3277       break;
3278     }
3279 
3280     case ABIArgInfo::CoerceAndExpand: {
3281       // Reconstruct into a temporary.
3282       Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3283       ArgVals.push_back(ParamValue::forIndirect(alloca));
3284 
3285       auto coercionType = ArgI.getCoerceAndExpandType();
3286       alloca = alloca.withElementType(coercionType);
3287 
3288       unsigned argIndex = FirstIRArg;
3289       for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3290         llvm::Type *eltType = coercionType->getElementType(i);
3291         if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3292           continue;
3293 
3294         auto eltAddr = Builder.CreateStructGEP(alloca, i);
3295         auto elt = Fn->getArg(argIndex++);
3296         Builder.CreateStore(elt, eltAddr);
3297       }
3298       assert(argIndex == FirstIRArg + NumIRArgs);
3299       break;
3300     }
3301 
3302     case ABIArgInfo::Expand: {
3303       // If this structure was expanded into multiple arguments then
3304       // we need to create a temporary and reconstruct it from the
3305       // arguments.
3306       Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3307       LValue LV = MakeAddrLValue(Alloca, Ty);
3308       ArgVals.push_back(ParamValue::forIndirect(Alloca));
3309 
3310       auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3311       ExpandTypeFromArgs(Ty, LV, FnArgIter);
3312       assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3313       for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
3314         auto AI = Fn->getArg(FirstIRArg + i);
3315         AI->setName(Arg->getName() + "." + Twine(i));
3316       }
3317       break;
3318     }
3319 
3320     case ABIArgInfo::Ignore:
3321       assert(NumIRArgs == 0);
3322       // Initialize the local variable appropriately.
3323       if (!hasScalarEvaluationKind(Ty)) {
3324         ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
3325       } else {
3326         llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
3327         ArgVals.push_back(ParamValue::forDirect(U));
3328       }
3329       break;
3330     }
3331   }
3332 
3333   if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3334     for (int I = Args.size() - 1; I >= 0; --I)
3335       EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3336   } else {
3337     for (unsigned I = 0, E = Args.size(); I != E; ++I)
3338       EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3339   }
3340 }
3341 
3342 static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3343   while (insn->use_empty()) {
3344     llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
3345     if (!bitcast) return;
3346 
3347     // This is "safe" because we would have used a ConstantExpr otherwise.
3348     insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3349     bitcast->eraseFromParent();
3350   }
3351 }
3352 
3353 /// Try to emit a fused autorelease of a return result.
3354 static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3355                                                     llvm::Value *result) {
3356   // We must be immediately followed the cast.
3357   llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3358   if (BB->empty()) return nullptr;
3359   if (&BB->back() != result) return nullptr;
3360 
3361   llvm::Type *resultType = result->getType();
3362 
3363   // result is in a BasicBlock and is therefore an Instruction.
3364   llvm::Instruction *generator = cast<llvm::Instruction>(result);
3365 
3366   SmallVector<llvm::Instruction *, 4> InstsToKill;
3367 
3368   // Look for:
3369   //  %generator = bitcast %type1* %generator2 to %type2*
3370   while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3371     // We would have emitted this as a constant if the operand weren't
3372     // an Instruction.
3373     generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3374 
3375     // Require the generator to be immediately followed by the cast.
3376     if (generator->getNextNode() != bitcast)
3377       return nullptr;
3378 
3379     InstsToKill.push_back(bitcast);
3380   }
3381 
3382   // Look for:
3383   //   %generator = call i8* @objc_retain(i8* %originalResult)
3384   // or
3385   //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3386   llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3387   if (!call) return nullptr;
3388 
3389   bool doRetainAutorelease;
3390 
3391   if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3392     doRetainAutorelease = true;
3393   } else if (call->getCalledOperand() ==
3394              CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3395     doRetainAutorelease = false;
3396 
3397     // If we emitted an assembly marker for this call (and the
3398     // ARCEntrypoints field should have been set if so), go looking
3399     // for that call.  If we can't find it, we can't do this
3400     // optimization.  But it should always be the immediately previous
3401     // instruction, unless we needed bitcasts around the call.
3402     if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3403       llvm::Instruction *prev = call->getPrevNode();
3404       assert(prev);
3405       if (isa<llvm::BitCastInst>(prev)) {
3406         prev = prev->getPrevNode();
3407         assert(prev);
3408       }
3409       assert(isa<llvm::CallInst>(prev));
3410       assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3411              CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3412       InstsToKill.push_back(prev);
3413     }
3414   } else {
3415     return nullptr;
3416   }
3417 
3418   result = call->getArgOperand(0);
3419   InstsToKill.push_back(call);
3420 
3421   // Keep killing bitcasts, for sanity.  Note that we no longer care
3422   // about precise ordering as long as there's exactly one use.
3423   while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3424     if (!bitcast->hasOneUse()) break;
3425     InstsToKill.push_back(bitcast);
3426     result = bitcast->getOperand(0);
3427   }
3428 
3429   // Delete all the unnecessary instructions, from latest to earliest.
3430   for (auto *I : InstsToKill)
3431     I->eraseFromParent();
3432 
3433   // Do the fused retain/autorelease if we were asked to.
3434   if (doRetainAutorelease)
3435     result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
3436 
3437   // Cast back to the result type.
3438   return CGF.Builder.CreateBitCast(result, resultType);
3439 }
3440 
3441 /// If this is a +1 of the value of an immutable 'self', remove it.
3442 static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3443                                           llvm::Value *result) {
3444   // This is only applicable to a method with an immutable 'self'.
3445   const ObjCMethodDecl *method =
3446     dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
3447   if (!method) return nullptr;
3448   const VarDecl *self = method->getSelfDecl();
3449   if (!self->getType().isConstQualified()) return nullptr;
3450 
3451   // Look for a retain call. Note: stripPointerCasts looks through returned arg
3452   // functions, which would cause us to miss the retain.
3453   llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(result);
3454   if (!retainCall || retainCall->getCalledOperand() !=
3455                          CGF.CGM.getObjCEntrypoints().objc_retain)
3456     return nullptr;
3457 
3458   // Look for an ordinary load of 'self'.
3459   llvm::Value *retainedValue = retainCall->getArgOperand(0);
3460   llvm::LoadInst *load =
3461     dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3462   if (!load || load->isAtomic() || load->isVolatile() ||
3463       load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
3464     return nullptr;
3465 
3466   // Okay!  Burn it all down.  This relies for correctness on the
3467   // assumption that the retain is emitted as part of the return and
3468   // that thereafter everything is used "linearly".
3469   llvm::Type *resultType = result->getType();
3470   eraseUnusedBitCasts(cast<llvm::Instruction>(result));
3471   assert(retainCall->use_empty());
3472   retainCall->eraseFromParent();
3473   eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
3474 
3475   return CGF.Builder.CreateBitCast(load, resultType);
3476 }
3477 
3478 /// Emit an ARC autorelease of the result of a function.
3479 ///
3480 /// \return the value to actually return from the function
3481 static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3482                                             llvm::Value *result) {
3483   // If we're returning 'self', kill the initial retain.  This is a
3484   // heuristic attempt to "encourage correctness" in the really unfortunate
3485   // case where we have a return of self during a dealloc and we desperately
3486   // need to avoid the possible autorelease.
3487   if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3488     return self;
3489 
3490   // At -O0, try to emit a fused retain/autorelease.
3491   if (CGF.shouldUseFusedARCCalls())
3492     if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3493       return fused;
3494 
3495   return CGF.EmitARCAutoreleaseReturnValue(result);
3496 }
3497 
3498 /// Heuristically search for a dominating store to the return-value slot.
3499 static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3500   // Check if a User is a store which pointerOperand is the ReturnValue.
3501   // We are looking for stores to the ReturnValue, not for stores of the
3502   // ReturnValue to some other location.
3503   auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
3504     auto *SI = dyn_cast<llvm::StoreInst>(U);
3505     if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() ||
3506         SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
3507       return nullptr;
3508     // These aren't actually possible for non-coerced returns, and we
3509     // only care about non-coerced returns on this code path.
3510     // All memory instructions inside __try block are volatile.
3511     assert(!SI->isAtomic() &&
3512            (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));
3513     return SI;
3514   };
3515   // If there are multiple uses of the return-value slot, just check
3516   // for something immediately preceding the IP.  Sometimes this can
3517   // happen with how we generate implicit-returns; it can also happen
3518   // with noreturn cleanups.
3519   if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
3520     llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3521     if (IP->empty()) return nullptr;
3522 
3523     // Look at directly preceding instruction, skipping bitcasts and lifetime
3524     // markers.
3525     for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend())) {
3526       if (isa<llvm::BitCastInst>(&I))
3527         continue;
3528       if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I))
3529         if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3530           continue;
3531 
3532       return GetStoreIfValid(&I);
3533     }
3534     return nullptr;
3535   }
3536 
3537   llvm::StoreInst *store =
3538       GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
3539   if (!store) return nullptr;
3540 
3541   // Now do a first-and-dirty dominance check: just walk up the
3542   // single-predecessors chain from the current insertion point.
3543   llvm::BasicBlock *StoreBB = store->getParent();
3544   llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3545   llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;
3546   while (IP != StoreBB) {
3547     if (!SeenBBs.insert(IP).second || !(IP = IP->getSinglePredecessor()))
3548       return nullptr;
3549   }
3550 
3551   // Okay, the store's basic block dominates the insertion point; we
3552   // can do our thing.
3553   return store;
3554 }
3555 
3556 // Helper functions for EmitCMSEClearRecord
3557 
3558 // Set the bits corresponding to a field having width `BitWidth` and located at
3559 // offset `BitOffset` (from the least significant bit) within a storage unit of
3560 // `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3561 // Use little-endian layout, i.e.`Bits[0]` is the LSB.
3562 static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3563                         int BitWidth, int CharWidth) {
3564   assert(CharWidth <= 64);
3565   assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3566 
3567   int Pos = 0;
3568   if (BitOffset >= CharWidth) {
3569     Pos += BitOffset / CharWidth;
3570     BitOffset = BitOffset % CharWidth;
3571   }
3572 
3573   const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3574   if (BitOffset + BitWidth >= CharWidth) {
3575     Bits[Pos++] |= (Used << BitOffset) & Used;
3576     BitWidth -= CharWidth - BitOffset;
3577     BitOffset = 0;
3578   }
3579 
3580   while (BitWidth >= CharWidth) {
3581     Bits[Pos++] = Used;
3582     BitWidth -= CharWidth;
3583   }
3584 
3585   if (BitWidth > 0)
3586     Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3587 }
3588 
3589 // Set the bits corresponding to a field having width `BitWidth` and located at
3590 // offset `BitOffset` (from the least significant bit) within a storage unit of
3591 // `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3592 // `Bits` corresponds to one target byte. Use target endian layout.
3593 static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3594                         int StorageSize, int BitOffset, int BitWidth,
3595                         int CharWidth, bool BigEndian) {
3596 
3597   SmallVector<uint64_t, 8> TmpBits(StorageSize);
3598   setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3599 
3600   if (BigEndian)
3601     std::reverse(TmpBits.begin(), TmpBits.end());
3602 
3603   for (uint64_t V : TmpBits)
3604     Bits[StorageOffset++] |= V;
3605 }
3606 
3607 static void setUsedBits(CodeGenModule &, QualType, int,
3608                         SmallVectorImpl<uint64_t> &);
3609 
3610 // Set the bits in `Bits`, which correspond to the value representations of
3611 // the actual members of the record type `RTy`. Note that this function does
3612 // not handle base classes, virtual tables, etc, since they cannot happen in
3613 // CMSE function arguments or return. The bit mask corresponds to the target
3614 // memory layout, i.e. it's endian dependent.
3615 static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3616                         SmallVectorImpl<uint64_t> &Bits) {
3617   ASTContext &Context = CGM.getContext();
3618   int CharWidth = Context.getCharWidth();
3619   const RecordDecl *RD = RTy->getDecl()->getDefinition();
3620   const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3621   const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3622 
3623   int Idx = 0;
3624   for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3625     const FieldDecl *F = *I;
3626 
3627     if (F->isUnnamedBitfield() || F->isZeroLengthBitField(Context) ||
3628         F->getType()->isIncompleteArrayType())
3629       continue;
3630 
3631     if (F->isBitField()) {
3632       const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3633       setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
3634                   BFI.StorageSize / CharWidth, BFI.Offset,
3635                   BFI.Size, CharWidth,
3636                   CGM.getDataLayout().isBigEndian());
3637       continue;
3638     }
3639 
3640     setUsedBits(CGM, F->getType(),
3641                 Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3642   }
3643 }
3644 
3645 // Set the bits in `Bits`, which correspond to the value representations of
3646 // the elements of an array type `ATy`.
3647 static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3648                         int Offset, SmallVectorImpl<uint64_t> &Bits) {
3649   const ASTContext &Context = CGM.getContext();
3650 
3651   QualType ETy = Context.getBaseElementType(ATy);
3652   int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3653   SmallVector<uint64_t, 4> TmpBits(Size);
3654   setUsedBits(CGM, ETy, 0, TmpBits);
3655 
3656   for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; ++I) {
3657     auto Src = TmpBits.begin();
3658     auto Dst = Bits.begin() + Offset + I * Size;
3659     for (int J = 0; J < Size; ++J)
3660       *Dst++ |= *Src++;
3661   }
3662 }
3663 
3664 // Set the bits in `Bits`, which correspond to the value representations of
3665 // the type `QTy`.
3666 static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3667                         SmallVectorImpl<uint64_t> &Bits) {
3668   if (const auto *RTy = QTy->getAs<RecordType>())
3669     return setUsedBits(CGM, RTy, Offset, Bits);
3670 
3671   ASTContext &Context = CGM.getContext();
3672   if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3673     return setUsedBits(CGM, ATy, Offset, Bits);
3674 
3675   int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3676   if (Size <= 0)
3677     return;
3678 
3679   std::fill_n(Bits.begin() + Offset, Size,
3680               (uint64_t(1) << Context.getCharWidth()) - 1);
3681 }
3682 
3683 static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3684                                    int Pos, int Size, int CharWidth,
3685                                    bool BigEndian) {
3686   assert(Size > 0);
3687   uint64_t Mask = 0;
3688   if (BigEndian) {
3689     for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3690          ++P)
3691       Mask = (Mask << CharWidth) | *P;
3692   } else {
3693     auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3694     do
3695       Mask = (Mask << CharWidth) | *--P;
3696     while (P != End);
3697   }
3698   return Mask;
3699 }
3700 
3701 // Emit code to clear the bits in a record, which aren't a part of any user
3702 // declared member, when the record is a function return.
3703 llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3704                                                   llvm::IntegerType *ITy,
3705                                                   QualType QTy) {
3706   assert(Src->getType() == ITy);
3707   assert(ITy->getScalarSizeInBits() <= 64);
3708 
3709   const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3710   int Size = DataLayout.getTypeStoreSize(ITy);
3711   SmallVector<uint64_t, 4> Bits(Size);
3712   setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3713 
3714   int CharWidth = CGM.getContext().getCharWidth();
3715   uint64_t Mask =
3716       buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3717 
3718   return Builder.CreateAnd(Src, Mask, "cmse.clear");
3719 }
3720 
3721 // Emit code to clear the bits in a record, which aren't a part of any user
3722 // declared member, when the record is a function argument.
3723 llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3724                                                   llvm::ArrayType *ATy,
3725                                                   QualType QTy) {
3726   const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3727   int Size = DataLayout.getTypeStoreSize(ATy);
3728   SmallVector<uint64_t, 16> Bits(Size);
3729   setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3730 
3731   // Clear each element of the LLVM array.
3732   int CharWidth = CGM.getContext().getCharWidth();
3733   int CharsPerElt =
3734       ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3735   int MaskIndex = 0;
3736   llvm::Value *R = llvm::PoisonValue::get(ATy);
3737   for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3738     uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3739                                        DataLayout.isBigEndian());
3740     MaskIndex += CharsPerElt;
3741     llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3742     llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3743     R = Builder.CreateInsertValue(R, T1, I);
3744   }
3745 
3746   return R;
3747 }
3748 
3749 void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
3750                                          bool EmitRetDbgLoc,
3751                                          SourceLocation EndLoc) {
3752   if (FI.isNoReturn()) {
3753     // Noreturn functions don't return.
3754     EmitUnreachable(EndLoc);
3755     return;
3756   }
3757 
3758   if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
3759     // Naked functions don't have epilogues.
3760     Builder.CreateUnreachable();
3761     return;
3762   }
3763 
3764   // Functions with no result always return void.
3765   if (!ReturnValue.isValid()) {
3766     Builder.CreateRetVoid();
3767     return;
3768   }
3769 
3770   llvm::DebugLoc RetDbgLoc;
3771   llvm::Value *RV = nullptr;
3772   QualType RetTy = FI.getReturnType();
3773   const ABIArgInfo &RetAI = FI.getReturnInfo();
3774 
3775   switch (RetAI.getKind()) {
3776   case ABIArgInfo::InAlloca:
3777     // Aggregates get evaluated directly into the destination.  Sometimes we
3778     // need to return the sret value in a register, though.
3779     assert(hasAggregateEvaluationKind(RetTy));
3780     if (RetAI.getInAllocaSRet()) {
3781       llvm::Function::arg_iterator EI = CurFn->arg_end();
3782       --EI;
3783       llvm::Value *ArgStruct = &*EI;
3784       llvm::Value *SRet = Builder.CreateStructGEP(
3785           FI.getArgStruct(), ArgStruct, RetAI.getInAllocaFieldIndex());
3786       llvm::Type *Ty =
3787           cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
3788       RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
3789     }
3790     break;
3791 
3792   case ABIArgInfo::Indirect: {
3793     auto AI = CurFn->arg_begin();
3794     if (RetAI.isSRetAfterThis())
3795       ++AI;
3796     switch (getEvaluationKind(RetTy)) {
3797     case TEK_Complex: {
3798       ComplexPairTy RT =
3799         EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
3800       EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
3801                          /*isInit*/ true);
3802       break;
3803     }
3804     case TEK_Aggregate:
3805       // Do nothing; aggregates get evaluated directly into the destination.
3806       break;
3807     case TEK_Scalar: {
3808       LValueBaseInfo BaseInfo;
3809       TBAAAccessInfo TBAAInfo;
3810       CharUnits Alignment =
3811           CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo);
3812       Address ArgAddr(&*AI, ConvertType(RetTy), Alignment);
3813       LValue ArgVal =
3814           LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo);
3815       EmitStoreOfScalar(
3816           Builder.CreateLoad(ReturnValue), ArgVal, /*isInit*/ true);
3817       break;
3818     }
3819     }
3820     break;
3821   }
3822 
3823   case ABIArgInfo::Extend:
3824   case ABIArgInfo::Direct:
3825     if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
3826         RetAI.getDirectOffset() == 0) {
3827       // The internal return value temp always will have pointer-to-return-type
3828       // type, just do a load.
3829 
3830       // If there is a dominating store to ReturnValue, we can elide
3831       // the load, zap the store, and usually zap the alloca.
3832       if (llvm::StoreInst *SI =
3833               findDominatingStoreToReturnValue(*this)) {
3834         // Reuse the debug location from the store unless there is
3835         // cleanup code to be emitted between the store and return
3836         // instruction.
3837         if (EmitRetDbgLoc && !AutoreleaseResult)
3838           RetDbgLoc = SI->getDebugLoc();
3839         // Get the stored value and nuke the now-dead store.
3840         RV = SI->getValueOperand();
3841         SI->eraseFromParent();
3842 
3843       // Otherwise, we have to do a simple load.
3844       } else {
3845         RV = Builder.CreateLoad(ReturnValue);
3846       }
3847     } else {
3848       // If the value is offset in memory, apply the offset now.
3849       Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
3850 
3851       RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
3852     }
3853 
3854     // In ARC, end functions that return a retainable type with a call
3855     // to objc_autoreleaseReturnValue.
3856     if (AutoreleaseResult) {
3857 #ifndef NDEBUG
3858       // Type::isObjCRetainabletype has to be called on a QualType that hasn't
3859       // been stripped of the typedefs, so we cannot use RetTy here. Get the
3860       // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3861       // CurCodeDecl or BlockInfo.
3862       QualType RT;
3863 
3864       if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
3865         RT = FD->getReturnType();
3866       else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
3867         RT = MD->getReturnType();
3868       else if (isa<BlockDecl>(CurCodeDecl))
3869         RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
3870       else
3871         llvm_unreachable("Unexpected function/method type");
3872 
3873       assert(getLangOpts().ObjCAutoRefCount &&
3874              !FI.isReturnsRetained() &&
3875              RT->isObjCRetainableType());
3876 #endif
3877       RV = emitAutoreleaseOfResult(*this, RV);
3878     }
3879 
3880     break;
3881 
3882   case ABIArgInfo::Ignore:
3883     break;
3884 
3885   case ABIArgInfo::CoerceAndExpand: {
3886     auto coercionType = RetAI.getCoerceAndExpandType();
3887 
3888     // Load all of the coerced elements out into results.
3889     llvm::SmallVector<llvm::Value*, 4> results;
3890     Address addr = ReturnValue.withElementType(coercionType);
3891     for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3892       auto coercedEltType = coercionType->getElementType(i);
3893       if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
3894         continue;
3895 
3896       auto eltAddr = Builder.CreateStructGEP(addr, i);
3897       auto elt = Builder.CreateLoad(eltAddr);
3898       results.push_back(elt);
3899     }
3900 
3901     // If we have one result, it's the single direct result type.
3902     if (results.size() == 1) {
3903       RV = results[0];
3904 
3905     // Otherwise, we need to make a first-class aggregate.
3906     } else {
3907       // Construct a return type that lacks padding elements.
3908       llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3909 
3910       RV = llvm::PoisonValue::get(returnType);
3911       for (unsigned i = 0, e = results.size(); i != e; ++i) {
3912         RV = Builder.CreateInsertValue(RV, results[i], i);
3913       }
3914     }
3915     break;
3916   }
3917   case ABIArgInfo::Expand:
3918   case ABIArgInfo::IndirectAliased:
3919     llvm_unreachable("Invalid ABI kind for return argument");
3920   }
3921 
3922   llvm::Instruction *Ret;
3923   if (RV) {
3924     if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
3925       // For certain return types, clear padding bits, as they may reveal
3926       // sensitive information.
3927       // Small struct/union types are passed as integers.
3928       auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3929       if (ITy != nullptr && isa<RecordType>(RetTy.getCanonicalType()))
3930         RV = EmitCMSEClearRecord(RV, ITy, RetTy);
3931     }
3932     EmitReturnValueCheck(RV);
3933     Ret = Builder.CreateRet(RV);
3934   } else {
3935     Ret = Builder.CreateRetVoid();
3936   }
3937 
3938   if (RetDbgLoc)
3939     Ret->setDebugLoc(std::move(RetDbgLoc));
3940 }
3941 
3942 void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3943   // A current decl may not be available when emitting vtable thunks.
3944   if (!CurCodeDecl)
3945     return;
3946 
3947   // If the return block isn't reachable, neither is this check, so don't emit
3948   // it.
3949   if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
3950     return;
3951 
3952   ReturnsNonNullAttr *RetNNAttr = nullptr;
3953   if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3954     RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3955 
3956   if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
3957     return;
3958 
3959   // Prefer the returns_nonnull attribute if it's present.
3960   SourceLocation AttrLoc;
3961   SanitizerMask CheckKind;
3962   SanitizerHandler Handler;
3963   if (RetNNAttr) {
3964     assert(!requiresReturnValueNullabilityCheck() &&
3965            "Cannot check nullability and the nonnull attribute");
3966     AttrLoc = RetNNAttr->getLocation();
3967     CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3968     Handler = SanitizerHandler::NonnullReturn;
3969   } else {
3970     if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3971       if (auto *TSI = DD->getTypeSourceInfo())
3972         if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3973           AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3974     CheckKind = SanitizerKind::NullabilityReturn;
3975     Handler = SanitizerHandler::NullabilityReturn;
3976   }
3977 
3978   SanitizerScope SanScope(this);
3979 
3980   // Make sure the "return" source location is valid. If we're checking a
3981   // nullability annotation, make sure the preconditions for the check are met.
3982   llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3983   llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3984   llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3985   llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3986   if (requiresReturnValueNullabilityCheck())
3987     CanNullCheck =
3988         Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3989   Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3990   EmitBlock(Check);
3991 
3992   // Now do the null check.
3993   llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3994   llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3995   llvm::Value *DynamicData[] = {SLocPtr};
3996   EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3997 
3998   EmitBlock(NoCheck);
3999 
4000 #ifndef NDEBUG
4001   // The return location should not be used after the check has been emitted.
4002   ReturnLocation = Address::invalid();
4003 #endif
4004 }
4005 
4006 static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
4007   const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
4008   return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
4009 }
4010 
4011 static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
4012                                           QualType Ty) {
4013   // FIXME: Generate IR in one pass, rather than going back and fixing up these
4014   // placeholders.
4015   llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
4016   llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(CGF.getLLVMContext());
4017   llvm::Value *Placeholder = llvm::PoisonValue::get(IRPtrTy);
4018 
4019   // FIXME: When we generate this IR in one pass, we shouldn't need
4020   // this win32-specific alignment hack.
4021   CharUnits Align = CharUnits::fromQuantity(4);
4022   Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
4023 
4024   return AggValueSlot::forAddr(Address(Placeholder, IRTy, Align),
4025                                Ty.getQualifiers(),
4026                                AggValueSlot::IsNotDestructed,
4027                                AggValueSlot::DoesNotNeedGCBarriers,
4028                                AggValueSlot::IsNotAliased,
4029                                AggValueSlot::DoesNotOverlap);
4030 }
4031 
4032 void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
4033                                           const VarDecl *param,
4034                                           SourceLocation loc) {
4035   // StartFunction converted the ABI-lowered parameter(s) into a
4036   // local alloca.  We need to turn that into an r-value suitable
4037   // for EmitCall.
4038   Address local = GetAddrOfLocalVar(param);
4039 
4040   QualType type = param->getType();
4041 
4042   // GetAddrOfLocalVar returns a pointer-to-pointer for references,
4043   // but the argument needs to be the original pointer.
4044   if (type->isReferenceType()) {
4045     args.add(RValue::get(Builder.CreateLoad(local)), type);
4046 
4047   // In ARC, move out of consumed arguments so that the release cleanup
4048   // entered by StartFunction doesn't cause an over-release.  This isn't
4049   // optimal -O0 code generation, but it should get cleaned up when
4050   // optimization is enabled.  This also assumes that delegate calls are
4051   // performed exactly once for a set of arguments, but that should be safe.
4052   } else if (getLangOpts().ObjCAutoRefCount &&
4053              param->hasAttr<NSConsumedAttr>() &&
4054              type->isObjCRetainableType()) {
4055     llvm::Value *ptr = Builder.CreateLoad(local);
4056     auto null =
4057       llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
4058     Builder.CreateStore(null, local);
4059     args.add(RValue::get(ptr), type);
4060 
4061   // For the most part, we just need to load the alloca, except that
4062   // aggregate r-values are actually pointers to temporaries.
4063   } else {
4064     args.add(convertTempToRValue(local, type, loc), type);
4065   }
4066 
4067   // Deactivate the cleanup for the callee-destructed param that was pushed.
4068   if (type->isRecordType() && !CurFuncIsThunk &&
4069       type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
4070       param->needsDestruction(getContext())) {
4071     EHScopeStack::stable_iterator cleanup =
4072         CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
4073     assert(cleanup.isValid() &&
4074            "cleanup for callee-destructed param not recorded");
4075     // This unreachable is a temporary marker which will be removed later.
4076     llvm::Instruction *isActive = Builder.CreateUnreachable();
4077     args.addArgCleanupDeactivation(cleanup, isActive);
4078   }
4079 }
4080 
4081 static bool isProvablyNull(llvm::Value *addr) {
4082   return isa<llvm::ConstantPointerNull>(addr);
4083 }
4084 
4085 /// Emit the actual writing-back of a writeback.
4086 static void emitWriteback(CodeGenFunction &CGF,
4087                           const CallArgList::Writeback &writeback) {
4088   const LValue &srcLV = writeback.Source;
4089   Address srcAddr = srcLV.getAddress(CGF);
4090   assert(!isProvablyNull(srcAddr.getPointer()) &&
4091          "shouldn't have writeback for provably null argument");
4092 
4093   llvm::BasicBlock *contBB = nullptr;
4094 
4095   // If the argument wasn't provably non-null, we need to null check
4096   // before doing the store.
4097   bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
4098                                               CGF.CGM.getDataLayout());
4099   if (!provablyNonNull) {
4100     llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
4101     contBB = CGF.createBasicBlock("icr.done");
4102 
4103     llvm::Value *isNull =
4104       CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
4105     CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
4106     CGF.EmitBlock(writebackBB);
4107   }
4108 
4109   // Load the value to writeback.
4110   llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
4111 
4112   // Cast it back, in case we're writing an id to a Foo* or something.
4113   value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
4114                                     "icr.writeback-cast");
4115 
4116   // Perform the writeback.
4117 
4118   // If we have a "to use" value, it's something we need to emit a use
4119   // of.  This has to be carefully threaded in: if it's done after the
4120   // release it's potentially undefined behavior (and the optimizer
4121   // will ignore it), and if it happens before the retain then the
4122   // optimizer could move the release there.
4123   if (writeback.ToUse) {
4124     assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
4125 
4126     // Retain the new value.  No need to block-copy here:  the block's
4127     // being passed up the stack.
4128     value = CGF.EmitARCRetainNonBlock(value);
4129 
4130     // Emit the intrinsic use here.
4131     CGF.EmitARCIntrinsicUse(writeback.ToUse);
4132 
4133     // Load the old value (primitively).
4134     llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
4135 
4136     // Put the new value in place (primitively).
4137     CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
4138 
4139     // Release the old value.
4140     CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
4141 
4142   // Otherwise, we can just do a normal lvalue store.
4143   } else {
4144     CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
4145   }
4146 
4147   // Jump to the continuation block.
4148   if (!provablyNonNull)
4149     CGF.EmitBlock(contBB);
4150 }
4151 
4152 static void emitWritebacks(CodeGenFunction &CGF,
4153                            const CallArgList &args) {
4154   for (const auto &I : args.writebacks())
4155     emitWriteback(CGF, I);
4156 }
4157 
4158 static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
4159                                             const CallArgList &CallArgs) {
4160   ArrayRef<CallArgList::CallArgCleanup> Cleanups =
4161     CallArgs.getCleanupsToDeactivate();
4162   // Iterate in reverse to increase the likelihood of popping the cleanup.
4163   for (const auto &I : llvm::reverse(Cleanups)) {
4164     CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
4165     I.IsActiveIP->eraseFromParent();
4166   }
4167 }
4168 
4169 static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
4170   if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
4171     if (uop->getOpcode() == UO_AddrOf)
4172       return uop->getSubExpr();
4173   return nullptr;
4174 }
4175 
4176 /// Emit an argument that's being passed call-by-writeback.  That is,
4177 /// we are passing the address of an __autoreleased temporary; it
4178 /// might be copy-initialized with the current value of the given
4179 /// address, but it will definitely be copied out of after the call.
4180 static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
4181                              const ObjCIndirectCopyRestoreExpr *CRE) {
4182   LValue srcLV;
4183 
4184   // Make an optimistic effort to emit the address as an l-value.
4185   // This can fail if the argument expression is more complicated.
4186   if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
4187     srcLV = CGF.EmitLValue(lvExpr);
4188 
4189   // Otherwise, just emit it as a scalar.
4190   } else {
4191     Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
4192 
4193     QualType srcAddrType =
4194       CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
4195     srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
4196   }
4197   Address srcAddr = srcLV.getAddress(CGF);
4198 
4199   // The dest and src types don't necessarily match in LLVM terms
4200   // because of the crazy ObjC compatibility rules.
4201 
4202   llvm::PointerType *destType =
4203       cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
4204   llvm::Type *destElemType =
4205       CGF.ConvertTypeForMem(CRE->getType()->getPointeeType());
4206 
4207   // If the address is a constant null, just pass the appropriate null.
4208   if (isProvablyNull(srcAddr.getPointer())) {
4209     args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
4210              CRE->getType());
4211     return;
4212   }
4213 
4214   // Create the temporary.
4215   Address temp =
4216       CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");
4217   // Loading an l-value can introduce a cleanup if the l-value is __weak,
4218   // and that cleanup will be conditional if we can't prove that the l-value
4219   // isn't null, so we need to register a dominating point so that the cleanups
4220   // system will make valid IR.
4221   CodeGenFunction::ConditionalEvaluation condEval(CGF);
4222 
4223   // Zero-initialize it if we're not doing a copy-initialization.
4224   bool shouldCopy = CRE->shouldCopy();
4225   if (!shouldCopy) {
4226     llvm::Value *null =
4227         llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));
4228     CGF.Builder.CreateStore(null, temp);
4229   }
4230 
4231   llvm::BasicBlock *contBB = nullptr;
4232   llvm::BasicBlock *originBB = nullptr;
4233 
4234   // If the address is *not* known to be non-null, we need to switch.
4235   llvm::Value *finalArgument;
4236 
4237   bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
4238                                               CGF.CGM.getDataLayout());
4239   if (provablyNonNull) {
4240     finalArgument = temp.getPointer();
4241   } else {
4242     llvm::Value *isNull =
4243       CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
4244 
4245     finalArgument = CGF.Builder.CreateSelect(isNull,
4246                                    llvm::ConstantPointerNull::get(destType),
4247                                              temp.getPointer(), "icr.argument");
4248 
4249     // If we need to copy, then the load has to be conditional, which
4250     // means we need control flow.
4251     if (shouldCopy) {
4252       originBB = CGF.Builder.GetInsertBlock();
4253       contBB = CGF.createBasicBlock("icr.cont");
4254       llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
4255       CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
4256       CGF.EmitBlock(copyBB);
4257       condEval.begin(CGF);
4258     }
4259   }
4260 
4261   llvm::Value *valueToUse = nullptr;
4262 
4263   // Perform a copy if necessary.
4264   if (shouldCopy) {
4265     RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
4266     assert(srcRV.isScalar());
4267 
4268     llvm::Value *src = srcRV.getScalarVal();
4269     src = CGF.Builder.CreateBitCast(src, destElemType, "icr.cast");
4270 
4271     // Use an ordinary store, not a store-to-lvalue.
4272     CGF.Builder.CreateStore(src, temp);
4273 
4274     // If optimization is enabled, and the value was held in a
4275     // __strong variable, we need to tell the optimizer that this
4276     // value has to stay alive until we're doing the store back.
4277     // This is because the temporary is effectively unretained,
4278     // and so otherwise we can violate the high-level semantics.
4279     if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4280         srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
4281       valueToUse = src;
4282     }
4283   }
4284 
4285   // Finish the control flow if we needed it.
4286   if (shouldCopy && !provablyNonNull) {
4287     llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4288     CGF.EmitBlock(contBB);
4289 
4290     // Make a phi for the value to intrinsically use.
4291     if (valueToUse) {
4292       llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
4293                                                       "icr.to-use");
4294       phiToUse->addIncoming(valueToUse, copyBB);
4295       phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
4296                             originBB);
4297       valueToUse = phiToUse;
4298     }
4299 
4300     condEval.end(CGF);
4301   }
4302 
4303   args.addWriteback(srcLV, temp, valueToUse);
4304   args.add(RValue::get(finalArgument), CRE->getType());
4305 }
4306 
4307 void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4308   assert(!StackBase);
4309 
4310   // Save the stack.
4311   StackBase = CGF.Builder.CreateStackSave("inalloca.save");
4312 }
4313 
4314 void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4315   if (StackBase) {
4316     // Restore the stack after the call.
4317     CGF.Builder.CreateStackRestore(StackBase);
4318   }
4319 }
4320 
4321 void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4322                                           SourceLocation ArgLoc,
4323                                           AbstractCallee AC,
4324                                           unsigned ParmNum) {
4325   if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||
4326                          SanOpts.has(SanitizerKind::NullabilityArg)))
4327     return;
4328 
4329   // The param decl may be missing in a variadic function.
4330   auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum) : nullptr;
4331   unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;
4332 
4333   // Prefer the nonnull attribute if it's present.
4334   const NonNullAttr *NNAttr = nullptr;
4335   if (SanOpts.has(SanitizerKind::NonnullAttribute))
4336     NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
4337 
4338   bool CanCheckNullability = false;
4339   if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD) {
4340     auto Nullability = PVD->getType()->getNullability();
4341     CanCheckNullability = Nullability &&
4342                           *Nullability == NullabilityKind::NonNull &&
4343                           PVD->getTypeSourceInfo();
4344   }
4345 
4346   if (!NNAttr && !CanCheckNullability)
4347     return;
4348 
4349   SourceLocation AttrLoc;
4350   SanitizerMask CheckKind;
4351   SanitizerHandler Handler;
4352   if (NNAttr) {
4353     AttrLoc = NNAttr->getLocation();
4354     CheckKind = SanitizerKind::NonnullAttribute;
4355     Handler = SanitizerHandler::NonnullArg;
4356   } else {
4357     AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4358     CheckKind = SanitizerKind::NullabilityArg;
4359     Handler = SanitizerHandler::NullabilityArg;
4360   }
4361 
4362   SanitizerScope SanScope(this);
4363   llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
4364   llvm::Constant *StaticData[] = {
4365       EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
4366       llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
4367   };
4368   EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt);
4369 }
4370 
4371 // Check if the call is going to use the inalloca convention. This needs to
4372 // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4373 // later, so we can't check it directly.
4374 static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4375                             ArrayRef<QualType> ArgTypes) {
4376   // The Swift calling conventions don't go through the target-specific
4377   // argument classification, they never use inalloca.
4378   // TODO: Consider limiting inalloca use to only calling conventions supported
4379   // by MSVC.
4380   if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)
4381     return false;
4382   if (!CGM.getTarget().getCXXABI().isMicrosoft())
4383     return false;
4384   return llvm::any_of(ArgTypes, [&](QualType Ty) {
4385     return isInAllocaArgument(CGM.getCXXABI(), Ty);
4386   });
4387 }
4388 
4389 #ifndef NDEBUG
4390 // Determine whether the given argument is an Objective-C method
4391 // that may have type parameters in its signature.
4392 static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4393   const DeclContext *dc = method->getDeclContext();
4394   if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4395     return classDecl->getTypeParamListAsWritten();
4396   }
4397 
4398   if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4399     return catDecl->getTypeParamList();
4400   }
4401 
4402   return false;
4403 }
4404 #endif
4405 
4406 /// EmitCallArgs - Emit call arguments for a function.
4407 void CodeGenFunction::EmitCallArgs(
4408     CallArgList &Args, PrototypeWrapper Prototype,
4409     llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4410     AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4411   SmallVector<QualType, 16> ArgTypes;
4412 
4413   assert((ParamsToSkip == 0 || Prototype.P) &&
4414          "Can't skip parameters if type info is not provided");
4415 
4416   // This variable only captures *explicitly* written conventions, not those
4417   // applied by default via command line flags or target defaults, such as
4418   // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4419   // require knowing if this is a C++ instance method or being able to see
4420   // unprototyped FunctionTypes.
4421   CallingConv ExplicitCC = CC_C;
4422 
4423   // First, if a prototype was provided, use those argument types.
4424   bool IsVariadic = false;
4425   if (Prototype.P) {
4426     const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>();
4427     if (MD) {
4428       IsVariadic = MD->isVariadic();
4429       ExplicitCC = getCallingConventionForDecl(
4430           MD, CGM.getTarget().getTriple().isOSWindows());
4431       ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4432                       MD->param_type_end());
4433     } else {
4434       const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
4435       IsVariadic = FPT->isVariadic();
4436       ExplicitCC = FPT->getExtInfo().getCC();
4437       ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4438                       FPT->param_type_end());
4439     }
4440 
4441 #ifndef NDEBUG
4442     // Check that the prototyped types match the argument expression types.
4443     bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
4444     CallExpr::const_arg_iterator Arg = ArgRange.begin();
4445     for (QualType Ty : ArgTypes) {
4446       assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4447       assert(
4448           (isGenericMethod || Ty->isVariablyModifiedType() ||
4449            Ty.getNonReferenceType()->isObjCRetainableType() ||
4450            getContext()
4451                    .getCanonicalType(Ty.getNonReferenceType())
4452                    .getTypePtr() ==
4453                getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4454           "type mismatch in call argument!");
4455       ++Arg;
4456     }
4457 
4458     // Either we've emitted all the call args, or we have a call to variadic
4459     // function.
4460     assert((Arg == ArgRange.end() || IsVariadic) &&
4461            "Extra arguments in non-variadic function!");
4462 #endif
4463   }
4464 
4465   // If we still have any arguments, emit them using the type of the argument.
4466   for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
4467     ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());
4468   assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4469 
4470   // We must evaluate arguments from right to left in the MS C++ ABI,
4471   // because arguments are destroyed left to right in the callee. As a special
4472   // case, there are certain language constructs that require left-to-right
4473   // evaluation, and in those cases we consider the evaluation order requirement
4474   // to trump the "destruction order is reverse construction order" guarantee.
4475   bool LeftToRight =
4476       CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4477           ? Order == EvaluationOrder::ForceLeftToRight
4478           : Order != EvaluationOrder::ForceRightToLeft;
4479 
4480   auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4481                                          RValue EmittedArg) {
4482     if (!AC.hasFunctionDecl() || I >= AC.getNumParams())
4483       return;
4484     auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4485     if (PS == nullptr)
4486       return;
4487 
4488     const auto &Context = getContext();
4489     auto SizeTy = Context.getSizeType();
4490     auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
4491     assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4492     llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
4493                                                      EmittedArg.getScalarVal(),
4494                                                      PS->isDynamic());
4495     Args.add(RValue::get(V), SizeTy);
4496     // If we're emitting args in reverse, be sure to do so with
4497     // pass_object_size, as well.
4498     if (!LeftToRight)
4499       std::swap(Args.back(), *(&Args.back() - 1));
4500   };
4501 
4502   // Insert a stack save if we're going to need any inalloca args.
4503   if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4504     assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4505            "inalloca only supported on x86");
4506     Args.allocateArgumentMemory(*this);
4507   }
4508 
4509   // Evaluate each argument in the appropriate order.
4510   size_t CallArgsStart = Args.size();
4511   for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {
4512     unsigned Idx = LeftToRight ? I : E - I - 1;
4513     CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4514     unsigned InitialArgSize = Args.size();
4515     // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4516     // the argument and parameter match or the objc method is parameterized.
4517     assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4518             getContext().hasSameUnqualifiedType((*Arg)->getType(),
4519                                                 ArgTypes[Idx]) ||
4520             (isa<ObjCMethodDecl>(AC.getDecl()) &&
4521              isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4522            "Argument and parameter types don't match");
4523     EmitCallArg(Args, *Arg, ArgTypes[Idx]);
4524     // In particular, we depend on it being the last arg in Args, and the
4525     // objectsize bits depend on there only being one arg if !LeftToRight.
4526     assert(InitialArgSize + 1 == Args.size() &&
4527            "The code below depends on only adding one arg per EmitCallArg");
4528     (void)InitialArgSize;
4529     // Since pointer argument are never emitted as LValue, it is safe to emit
4530     // non-null argument check for r-value only.
4531     if (!Args.back().hasLValue()) {
4532       RValue RVArg = Args.back().getKnownRValue();
4533       EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
4534                           ParamsToSkip + Idx);
4535       // @llvm.objectsize should never have side-effects and shouldn't need
4536       // destruction/cleanups, so we can safely "emit" it after its arg,
4537       // regardless of right-to-leftness
4538       MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4539     }
4540   }
4541 
4542   if (!LeftToRight) {
4543     // Un-reverse the arguments we just evaluated so they match up with the LLVM
4544     // IR function.
4545     std::reverse(Args.begin() + CallArgsStart, Args.end());
4546   }
4547 }
4548 
4549 namespace {
4550 
4551 struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4552   DestroyUnpassedArg(Address Addr, QualType Ty)
4553       : Addr(Addr), Ty(Ty) {}
4554 
4555   Address Addr;
4556   QualType Ty;
4557 
4558   void Emit(CodeGenFunction &CGF, Flags flags) override {
4559     QualType::DestructionKind DtorKind = Ty.isDestructedType();
4560     if (DtorKind == QualType::DK_cxx_destructor) {
4561       const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4562       assert(!Dtor->isTrivial());
4563       CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
4564                                 /*Delegating=*/false, Addr, Ty);
4565     } else {
4566       CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
4567     }
4568   }
4569 };
4570 
4571 struct DisableDebugLocationUpdates {
4572   CodeGenFunction &CGF;
4573   bool disabledDebugInfo;
4574   DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4575     if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo()))
4576       CGF.disableDebugInfo();
4577   }
4578   ~DisableDebugLocationUpdates() {
4579     if (disabledDebugInfo)
4580       CGF.enableDebugInfo();
4581   }
4582 };
4583 
4584 } // end anonymous namespace
4585 
4586 RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4587   if (!HasLV)
4588     return RV;
4589   LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
4590   CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
4591                         LV.isVolatile());
4592   IsUsed = true;
4593   return RValue::getAggregate(Copy.getAddress(CGF));
4594 }
4595 
4596 void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4597   LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4598   if (!HasLV && RV.isScalar())
4599     CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
4600   else if (!HasLV && RV.isComplex())
4601     CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
4602   else {
4603     auto Addr = HasLV ? LV.getAddress(CGF) : RV.getAggregateAddress();
4604     LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4605     // We assume that call args are never copied into subobjects.
4606     CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
4607                           HasLV ? LV.isVolatileQualified()
4608                                 : RV.isVolatileQualified());
4609   }
4610   IsUsed = true;
4611 }
4612 
4613 void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4614                                   QualType type) {
4615   DisableDebugLocationUpdates Dis(*this, E);
4616   if (const ObjCIndirectCopyRestoreExpr *CRE
4617         = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4618     assert(getLangOpts().ObjCAutoRefCount);
4619     return emitWritebackArg(*this, args, CRE);
4620   }
4621 
4622   assert(type->isReferenceType() == E->isGLValue() &&
4623          "reference binding to unmaterialized r-value!");
4624 
4625   if (E->isGLValue()) {
4626     assert(E->getObjectKind() == OK_Ordinary);
4627     return args.add(EmitReferenceBindingToExpr(E), type);
4628   }
4629 
4630   bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
4631 
4632   // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4633   // However, we still have to push an EH-only cleanup in case we unwind before
4634   // we make it to the call.
4635   if (type->isRecordType() &&
4636       type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
4637     // If we're using inalloca, use the argument memory.  Otherwise, use a
4638     // temporary.
4639     AggValueSlot Slot = args.isUsingInAlloca()
4640         ? createPlaceholderSlot(*this, type) : CreateAggTemp(type, "agg.tmp");
4641 
4642     bool DestroyedInCallee = true, NeedsEHCleanup = true;
4643     if (const auto *RD = type->getAsCXXRecordDecl())
4644       DestroyedInCallee = RD->hasNonTrivialDestructor();
4645     else
4646       NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
4647 
4648     if (DestroyedInCallee)
4649       Slot.setExternallyDestructed();
4650 
4651     EmitAggExpr(E, Slot);
4652     RValue RV = Slot.asRValue();
4653     args.add(RV, type);
4654 
4655     if (DestroyedInCallee && NeedsEHCleanup) {
4656       // Create a no-op GEP between the placeholder and the cleanup so we can
4657       // RAUW it successfully.  It also serves as a marker of the first
4658       // instruction where the cleanup is active.
4659       pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
4660                                               type);
4661       // This unreachable is a temporary marker which will be removed later.
4662       llvm::Instruction *IsActive = Builder.CreateUnreachable();
4663       args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive);
4664     }
4665     return;
4666   }
4667 
4668   if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&
4669       cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
4670     LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
4671     assert(L.isSimple());
4672     args.addUncopiedAggregate(L, type);
4673     return;
4674   }
4675 
4676   args.add(EmitAnyExprToTemp(E), type);
4677 }
4678 
4679 QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4680   // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4681   // implicitly widens null pointer constants that are arguments to varargs
4682   // functions to pointer-sized ints.
4683   if (!getTarget().getTriple().isOSWindows())
4684     return Arg->getType();
4685 
4686   if (Arg->getType()->isIntegerType() &&
4687       getContext().getTypeSize(Arg->getType()) <
4688           getContext().getTargetInfo().getPointerWidth(LangAS::Default) &&
4689       Arg->isNullPointerConstant(getContext(),
4690                                  Expr::NPC_ValueDependentIsNotNull)) {
4691     return getContext().getIntPtrType();
4692   }
4693 
4694   return Arg->getType();
4695 }
4696 
4697 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4698 // optimizer it can aggressively ignore unwind edges.
4699 void
4700 CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4701   if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4702       !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
4703     Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
4704                       CGM.getNoObjCARCExceptionsMetadata());
4705 }
4706 
4707 /// Emits a call to the given no-arguments nounwind runtime function.
4708 llvm::CallInst *
4709 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4710                                          const llvm::Twine &name) {
4711   return EmitNounwindRuntimeCall(callee, std::nullopt, name);
4712 }
4713 
4714 /// Emits a call to the given nounwind runtime function.
4715 llvm::CallInst *
4716 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4717                                          ArrayRef<llvm::Value *> args,
4718                                          const llvm::Twine &name) {
4719   llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4720   call->setDoesNotThrow();
4721   return call;
4722 }
4723 
4724 /// Emits a simple call (never an invoke) to the given no-arguments
4725 /// runtime function.
4726 llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4727                                                  const llvm::Twine &name) {
4728   return EmitRuntimeCall(callee, std::nullopt, name);
4729 }
4730 
4731 // Calls which may throw must have operand bundles indicating which funclet
4732 // they are nested within.
4733 SmallVector<llvm::OperandBundleDef, 1>
4734 CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4735   // There is no need for a funclet operand bundle if we aren't inside a
4736   // funclet.
4737   if (!CurrentFuncletPad)
4738     return (SmallVector<llvm::OperandBundleDef, 1>());
4739 
4740   // Skip intrinsics which cannot throw (as long as they don't lower into
4741   // regular function calls in the course of IR transformations).
4742   if (auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts())) {
4743     if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {
4744       auto IID = CalleeFn->getIntrinsicID();
4745       if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))
4746         return (SmallVector<llvm::OperandBundleDef, 1>());
4747     }
4748   }
4749 
4750   SmallVector<llvm::OperandBundleDef, 1> BundleList;
4751   BundleList.emplace_back("funclet", CurrentFuncletPad);
4752   return BundleList;
4753 }
4754 
4755 /// Emits a simple call (never an invoke) to the given runtime function.
4756 llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4757                                                  ArrayRef<llvm::Value *> args,
4758                                                  const llvm::Twine &name) {
4759   llvm::CallInst *call = Builder.CreateCall(
4760       callee, args, getBundlesForFunclet(callee.getCallee()), name);
4761   call->setCallingConv(getRuntimeCC());
4762   return call;
4763 }
4764 
4765 /// Emits a call or invoke to the given noreturn runtime function.
4766 void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
4767     llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
4768   SmallVector<llvm::OperandBundleDef, 1> BundleList =
4769       getBundlesForFunclet(callee.getCallee());
4770 
4771   if (getInvokeDest()) {
4772     llvm::InvokeInst *invoke =
4773       Builder.CreateInvoke(callee,
4774                            getUnreachableBlock(),
4775                            getInvokeDest(),
4776                            args,
4777                            BundleList);
4778     invoke->setDoesNotReturn();
4779     invoke->setCallingConv(getRuntimeCC());
4780   } else {
4781     llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
4782     call->setDoesNotReturn();
4783     call->setCallingConv(getRuntimeCC());
4784     Builder.CreateUnreachable();
4785   }
4786 }
4787 
4788 /// Emits a call or invoke instruction to the given nullary runtime function.
4789 llvm::CallBase *
4790 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4791                                          const Twine &name) {
4792   return EmitRuntimeCallOrInvoke(callee, std::nullopt, name);
4793 }
4794 
4795 /// Emits a call or invoke instruction to the given runtime function.
4796 llvm::CallBase *
4797 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4798                                          ArrayRef<llvm::Value *> args,
4799                                          const Twine &name) {
4800   llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
4801   call->setCallingConv(getRuntimeCC());
4802   return call;
4803 }
4804 
4805 /// Emits a call or invoke instruction to the given function, depending
4806 /// on the current state of the EH stack.
4807 llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
4808                                                   ArrayRef<llvm::Value *> Args,
4809                                                   const Twine &Name) {
4810   llvm::BasicBlock *InvokeDest = getInvokeDest();
4811   SmallVector<llvm::OperandBundleDef, 1> BundleList =
4812       getBundlesForFunclet(Callee.getCallee());
4813 
4814   llvm::CallBase *Inst;
4815   if (!InvokeDest)
4816     Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
4817   else {
4818     llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
4819     Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4820                                 Name);
4821     EmitBlock(ContBB);
4822   }
4823 
4824   // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4825   // optimizer it can aggressively ignore unwind edges.
4826   if (CGM.getLangOpts().ObjCAutoRefCount)
4827     AddObjCARCExceptionMetadata(Inst);
4828 
4829   return Inst;
4830 }
4831 
4832 void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4833                                                   llvm::Value *New) {
4834   DeferredReplacements.push_back(
4835       std::make_pair(llvm::WeakTrackingVH(Old), New));
4836 }
4837 
4838 namespace {
4839 
4840 /// Specify given \p NewAlign as the alignment of return value attribute. If
4841 /// such attribute already exists, re-set it to the maximal one of two options.
4842 [[nodiscard]] llvm::AttributeList
4843 maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4844                                 const llvm::AttributeList &Attrs,
4845                                 llvm::Align NewAlign) {
4846   llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4847   if (CurAlign >= NewAlign)
4848     return Attrs;
4849   llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4850   return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
4851       .addRetAttribute(Ctx, AlignAttr);
4852 }
4853 
4854 template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
4855 protected:
4856   CodeGenFunction &CGF;
4857 
4858   /// We do nothing if this is, or becomes, nullptr.
4859   const AlignedAttrTy *AA = nullptr;
4860 
4861   llvm::Value *Alignment = nullptr;      // May or may not be a constant.
4862   llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
4863 
4864   AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4865       : CGF(CGF_) {
4866     if (!FuncDecl)
4867       return;
4868     AA = FuncDecl->getAttr<AlignedAttrTy>();
4869   }
4870 
4871 public:
4872   /// If we can, materialize the alignment as an attribute on return value.
4873   [[nodiscard]] llvm::AttributeList
4874   TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4875     if (!AA || OffsetCI || CGF.SanOpts.has(SanitizerKind::Alignment))
4876       return Attrs;
4877     const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4878     if (!AlignmentCI)
4879       return Attrs;
4880     // We may legitimately have non-power-of-2 alignment here.
4881     // If so, this is UB land, emit it via `@llvm.assume` instead.
4882     if (!AlignmentCI->getValue().isPowerOf2())
4883       return Attrs;
4884     llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4885         CGF.getLLVMContext(), Attrs,
4886         llvm::Align(
4887             AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4888     AA = nullptr; // We're done. Disallow doing anything else.
4889     return NewAttrs;
4890   }
4891 
4892   /// Emit alignment assumption.
4893   /// This is a general fallback that we take if either there is an offset,
4894   /// or the alignment is variable or we are sanitizing for alignment.
4895   void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4896     if (!AA)
4897       return;
4898     CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4899                                 AA->getLocation(), Alignment, OffsetCI);
4900     AA = nullptr; // We're done. Disallow doing anything else.
4901   }
4902 };
4903 
4904 /// Helper data structure to emit `AssumeAlignedAttr`.
4905 class AssumeAlignedAttrEmitter final
4906     : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4907 public:
4908   AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4909       : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4910     if (!AA)
4911       return;
4912     // It is guaranteed that the alignment/offset are constants.
4913     Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
4914     if (Expr *Offset = AA->getOffset()) {
4915       OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
4916       if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4917         OffsetCI = nullptr;
4918     }
4919   }
4920 };
4921 
4922 /// Helper data structure to emit `AllocAlignAttr`.
4923 class AllocAlignAttrEmitter final
4924     : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4925 public:
4926   AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
4927                         const CallArgList &CallArgs)
4928       : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4929     if (!AA)
4930       return;
4931     // Alignment may or may not be a constant, and that is okay.
4932     Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4933                     .getRValue(CGF)
4934                     .getScalarVal();
4935   }
4936 };
4937 
4938 } // namespace
4939 
4940 static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
4941   if (auto *VT = dyn_cast<llvm::VectorType>(Ty))
4942     return VT->getPrimitiveSizeInBits().getKnownMinValue();
4943   if (auto *AT = dyn_cast<llvm::ArrayType>(Ty))
4944     return getMaxVectorWidth(AT->getElementType());
4945 
4946   unsigned MaxVectorWidth = 0;
4947   if (auto *ST = dyn_cast<llvm::StructType>(Ty))
4948     for (auto *I : ST->elements())
4949       MaxVectorWidth = std::max(MaxVectorWidth, getMaxVectorWidth(I));
4950   return MaxVectorWidth;
4951 }
4952 
4953 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
4954                                  const CGCallee &Callee,
4955                                  ReturnValueSlot ReturnValue,
4956                                  const CallArgList &CallArgs,
4957                                  llvm::CallBase **callOrInvoke, bool IsMustTail,
4958                                  SourceLocation Loc) {
4959   // FIXME: We no longer need the types from CallArgs; lift up and simplify.
4960 
4961   assert(Callee.isOrdinary() || Callee.isVirtual());
4962 
4963   // Handle struct-return functions by passing a pointer to the
4964   // location that we would like to return into.
4965   QualType RetTy = CallInfo.getReturnType();
4966   const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
4967 
4968   llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
4969 
4970   const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
4971   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4972     // We can only guarantee that a function is called from the correct
4973     // context/function based on the appropriate target attributes,
4974     // so only check in the case where we have both always_inline and target
4975     // since otherwise we could be making a conditional call after a check for
4976     // the proper cpu features (and it won't cause code generation issues due to
4977     // function based code generation).
4978     if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
4979         (TargetDecl->hasAttr<TargetAttr>() ||
4980          (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>())))
4981       checkTargetFeatures(Loc, FD);
4982 
4983     // Some architectures (such as x86-64) have the ABI changed based on
4984     // attribute-target/features. Give them a chance to diagnose.
4985     CGM.getTargetCodeGenInfo().checkFunctionCallABI(
4986         CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
4987   }
4988 
4989   // 1. Set up the arguments.
4990 
4991   // If we're using inalloca, insert the allocation after the stack save.
4992   // FIXME: Do this earlier rather than hacking it in here!
4993   Address ArgMemory = Address::invalid();
4994   if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
4995     const llvm::DataLayout &DL = CGM.getDataLayout();
4996     llvm::Instruction *IP = CallArgs.getStackBase();
4997     llvm::AllocaInst *AI;
4998     if (IP) {
4999       IP = IP->getNextNode();
5000       AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
5001                                 "argmem", IP);
5002     } else {
5003       AI = CreateTempAlloca(ArgStruct, "argmem");
5004     }
5005     auto Align = CallInfo.getArgStructAlignment();
5006     AI->setAlignment(Align.getAsAlign());
5007     AI->setUsedWithInAlloca(true);
5008     assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
5009     ArgMemory = Address(AI, ArgStruct, Align);
5010   }
5011 
5012   ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
5013   SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
5014 
5015   // If the call returns a temporary with struct return, create a temporary
5016   // alloca to hold the result, unless one is given to us.
5017   Address SRetPtr = Address::invalid();
5018   Address SRetAlloca = Address::invalid();
5019   llvm::Value *UnusedReturnSizePtr = nullptr;
5020   if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
5021     if (!ReturnValue.isNull()) {
5022       SRetPtr = ReturnValue.getValue();
5023     } else {
5024       SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
5025       if (HaveInsertPoint() && ReturnValue.isUnused()) {
5026         llvm::TypeSize size =
5027             CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
5028         UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
5029       }
5030     }
5031     if (IRFunctionArgs.hasSRetArg()) {
5032       IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
5033     } else if (RetAI.isInAlloca()) {
5034       Address Addr =
5035           Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
5036       Builder.CreateStore(SRetPtr.getPointer(), Addr);
5037     }
5038   }
5039 
5040   Address swiftErrorTemp = Address::invalid();
5041   Address swiftErrorArg = Address::invalid();
5042 
5043   // When passing arguments using temporary allocas, we need to add the
5044   // appropriate lifetime markers. This vector keeps track of all the lifetime
5045   // markers that need to be ended right after the call.
5046   SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
5047 
5048   // Translate all of the arguments as necessary to match the IR lowering.
5049   assert(CallInfo.arg_size() == CallArgs.size() &&
5050          "Mismatch between function signature & arguments.");
5051   unsigned ArgNo = 0;
5052   CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
5053   for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
5054        I != E; ++I, ++info_it, ++ArgNo) {
5055     const ABIArgInfo &ArgInfo = info_it->info;
5056 
5057     // Insert a padding argument to ensure proper alignment.
5058     if (IRFunctionArgs.hasPaddingArg(ArgNo))
5059       IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
5060           llvm::UndefValue::get(ArgInfo.getPaddingType());
5061 
5062     unsigned FirstIRArg, NumIRArgs;
5063     std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
5064 
5065     bool ArgHasMaybeUndefAttr =
5066         IsArgumentMaybeUndef(TargetDecl, CallInfo.getNumRequiredArgs(), ArgNo);
5067 
5068     switch (ArgInfo.getKind()) {
5069     case ABIArgInfo::InAlloca: {
5070       assert(NumIRArgs == 0);
5071       assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
5072       if (I->isAggregate()) {
5073         Address Addr = I->hasLValue()
5074                            ? I->getKnownLValue().getAddress(*this)
5075                            : I->getKnownRValue().getAggregateAddress();
5076         llvm::Instruction *Placeholder =
5077             cast<llvm::Instruction>(Addr.getPointer());
5078 
5079         if (!ArgInfo.getInAllocaIndirect()) {
5080           // Replace the placeholder with the appropriate argument slot GEP.
5081           CGBuilderTy::InsertPoint IP = Builder.saveIP();
5082           Builder.SetInsertPoint(Placeholder);
5083           Addr = Builder.CreateStructGEP(ArgMemory,
5084                                          ArgInfo.getInAllocaFieldIndex());
5085           Builder.restoreIP(IP);
5086         } else {
5087           // For indirect things such as overaligned structs, replace the
5088           // placeholder with a regular aggregate temporary alloca. Store the
5089           // address of this alloca into the struct.
5090           Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
5091           Address ArgSlot = Builder.CreateStructGEP(
5092               ArgMemory, ArgInfo.getInAllocaFieldIndex());
5093           Builder.CreateStore(Addr.getPointer(), ArgSlot);
5094         }
5095         deferPlaceholderReplacement(Placeholder, Addr.getPointer());
5096       } else if (ArgInfo.getInAllocaIndirect()) {
5097         // Make a temporary alloca and store the address of it into the argument
5098         // struct.
5099         Address Addr = CreateMemTempWithoutCast(
5100             I->Ty, getContext().getTypeAlignInChars(I->Ty),
5101             "indirect-arg-temp");
5102         I->copyInto(*this, Addr);
5103         Address ArgSlot =
5104             Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
5105         Builder.CreateStore(Addr.getPointer(), ArgSlot);
5106       } else {
5107         // Store the RValue into the argument struct.
5108         Address Addr =
5109             Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
5110         Addr = Addr.withElementType(ConvertTypeForMem(I->Ty));
5111         I->copyInto(*this, Addr);
5112       }
5113       break;
5114     }
5115 
5116     case ABIArgInfo::Indirect:
5117     case ABIArgInfo::IndirectAliased: {
5118       assert(NumIRArgs == 1);
5119       if (!I->isAggregate()) {
5120         // Make a temporary alloca to pass the argument.
5121         Address Addr = CreateMemTempWithoutCast(
5122             I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
5123 
5124         llvm::Value *Val = Addr.getPointer();
5125         if (ArgHasMaybeUndefAttr)
5126           Val = Builder.CreateFreeze(Addr.getPointer());
5127         IRCallArgs[FirstIRArg] = Val;
5128 
5129         I->copyInto(*this, Addr);
5130       } else {
5131         // We want to avoid creating an unnecessary temporary+copy here;
5132         // however, we need one in three cases:
5133         // 1. If the argument is not byval, and we are required to copy the
5134         //    source.  (This case doesn't occur on any common architecture.)
5135         // 2. If the argument is byval, RV is not sufficiently aligned, and
5136         //    we cannot force it to be sufficiently aligned.
5137         // 3. If the argument is byval, but RV is not located in default
5138         //    or alloca address space.
5139         Address Addr = I->hasLValue()
5140                            ? I->getKnownLValue().getAddress(*this)
5141                            : I->getKnownRValue().getAggregateAddress();
5142         llvm::Value *V = Addr.getPointer();
5143         CharUnits Align = ArgInfo.getIndirectAlign();
5144         const llvm::DataLayout *TD = &CGM.getDataLayout();
5145 
5146         assert((FirstIRArg >= IRFuncTy->getNumParams() ||
5147                 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
5148                     TD->getAllocaAddrSpace()) &&
5149                "indirect argument must be in alloca address space");
5150 
5151         bool NeedCopy = false;
5152         if (Addr.getAlignment() < Align &&
5153             llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
5154                 Align.getAsAlign()) {
5155           NeedCopy = true;
5156         } else if (I->hasLValue()) {
5157           auto LV = I->getKnownLValue();
5158           auto AS = LV.getAddressSpace();
5159 
5160           bool isByValOrRef =
5161               ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();
5162 
5163           if (!isByValOrRef ||
5164               (LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) {
5165             NeedCopy = true;
5166           }
5167           if (!getLangOpts().OpenCL) {
5168             if ((isByValOrRef &&
5169                 (AS != LangAS::Default &&
5170                  AS != CGM.getASTAllocaAddressSpace()))) {
5171               NeedCopy = true;
5172             }
5173           }
5174           // For OpenCL even if RV is located in default or alloca address space
5175           // we don't want to perform address space cast for it.
5176           else if ((isByValOrRef &&
5177                     Addr.getType()->getAddressSpace() != IRFuncTy->
5178                       getParamType(FirstIRArg)->getPointerAddressSpace())) {
5179             NeedCopy = true;
5180           }
5181         }
5182 
5183         if (NeedCopy) {
5184           // Create an aligned temporary, and copy to it.
5185           Address AI = CreateMemTempWithoutCast(
5186               I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
5187           llvm::Value *Val = AI.getPointer();
5188           if (ArgHasMaybeUndefAttr)
5189             Val = Builder.CreateFreeze(AI.getPointer());
5190           IRCallArgs[FirstIRArg] = Val;
5191 
5192           // Emit lifetime markers for the temporary alloca.
5193           llvm::TypeSize ByvalTempElementSize =
5194               CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
5195           llvm::Value *LifetimeSize =
5196               EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
5197 
5198           // Add cleanup code to emit the end lifetime marker after the call.
5199           if (LifetimeSize) // In case we disabled lifetime markers.
5200             CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
5201 
5202           // Generate the copy.
5203           I->copyInto(*this, AI);
5204         } else {
5205           // Skip the extra memcpy call.
5206           auto *T = llvm::PointerType::get(
5207               CGM.getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace());
5208 
5209           llvm::Value *Val = getTargetHooks().performAddrSpaceCast(
5210               *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
5211               true);
5212           if (ArgHasMaybeUndefAttr)
5213             Val = Builder.CreateFreeze(Val);
5214           IRCallArgs[FirstIRArg] = Val;
5215         }
5216       }
5217       break;
5218     }
5219 
5220     case ABIArgInfo::Ignore:
5221       assert(NumIRArgs == 0);
5222       break;
5223 
5224     case ABIArgInfo::Extend:
5225     case ABIArgInfo::Direct: {
5226       if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
5227           ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
5228           ArgInfo.getDirectOffset() == 0) {
5229         assert(NumIRArgs == 1);
5230         llvm::Value *V;
5231         if (!I->isAggregate())
5232           V = I->getKnownRValue().getScalarVal();
5233         else
5234           V = Builder.CreateLoad(
5235               I->hasLValue() ? I->getKnownLValue().getAddress(*this)
5236                              : I->getKnownRValue().getAggregateAddress());
5237 
5238         // Implement swifterror by copying into a new swifterror argument.
5239         // We'll write back in the normal path out of the call.
5240         if (CallInfo.getExtParameterInfo(ArgNo).getABI()
5241               == ParameterABI::SwiftErrorResult) {
5242           assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
5243 
5244           QualType pointeeTy = I->Ty->getPointeeType();
5245           swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy),
5246                                   getContext().getTypeAlignInChars(pointeeTy));
5247 
5248           swiftErrorTemp =
5249             CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
5250           V = swiftErrorTemp.getPointer();
5251           cast<llvm::AllocaInst>(V)->setSwiftError(true);
5252 
5253           llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
5254           Builder.CreateStore(errorValue, swiftErrorTemp);
5255         }
5256 
5257         // We might have to widen integers, but we should never truncate.
5258         if (ArgInfo.getCoerceToType() != V->getType() &&
5259             V->getType()->isIntegerTy())
5260           V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
5261 
5262         // If the argument doesn't match, perform a bitcast to coerce it.  This
5263         // can happen due to trivial type mismatches.
5264         if (FirstIRArg < IRFuncTy->getNumParams() &&
5265             V->getType() != IRFuncTy->getParamType(FirstIRArg))
5266           V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
5267 
5268         if (ArgHasMaybeUndefAttr)
5269           V = Builder.CreateFreeze(V);
5270         IRCallArgs[FirstIRArg] = V;
5271         break;
5272       }
5273 
5274       // FIXME: Avoid the conversion through memory if possible.
5275       Address Src = Address::invalid();
5276       if (!I->isAggregate()) {
5277         Src = CreateMemTemp(I->Ty, "coerce");
5278         I->copyInto(*this, Src);
5279       } else {
5280         Src = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
5281                              : I->getKnownRValue().getAggregateAddress();
5282       }
5283 
5284       // If the value is offset in memory, apply the offset now.
5285       Src = emitAddressAtOffset(*this, Src, ArgInfo);
5286 
5287       // Fast-isel and the optimizer generally like scalar values better than
5288       // FCAs, so we flatten them if this is safe to do for this argument.
5289       llvm::StructType *STy =
5290             dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
5291       if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
5292         llvm::Type *SrcTy = Src.getElementType();
5293         llvm::TypeSize SrcTypeSize =
5294             CGM.getDataLayout().getTypeAllocSize(SrcTy);
5295         llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(STy);
5296         if (SrcTypeSize.isScalable()) {
5297           assert(STy->containsHomogeneousScalableVectorTypes() &&
5298                  "ABI only supports structure with homogeneous scalable vector "
5299                  "type");
5300           assert(SrcTypeSize == DstTypeSize &&
5301                  "Only allow non-fractional movement of structure with "
5302                  "homogeneous scalable vector type");
5303           assert(NumIRArgs == STy->getNumElements());
5304 
5305           llvm::Value *StoredStructValue =
5306               Builder.CreateLoad(Src, Src.getName() + ".tuple");
5307           for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5308             llvm::Value *Extract = Builder.CreateExtractValue(
5309                 StoredStructValue, i, Src.getName() + ".extract" + Twine(i));
5310             IRCallArgs[FirstIRArg + i] = Extract;
5311           }
5312         } else {
5313           uint64_t SrcSize = SrcTypeSize.getFixedValue();
5314           uint64_t DstSize = DstTypeSize.getFixedValue();
5315 
5316           // If the source type is smaller than the destination type of the
5317           // coerce-to logic, copy the source value into a temp alloca the size
5318           // of the destination type to allow loading all of it. The bits past
5319           // the source value are left undef.
5320           if (SrcSize < DstSize) {
5321             Address TempAlloca = CreateTempAlloca(STy, Src.getAlignment(),
5322                                                   Src.getName() + ".coerce");
5323             Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
5324             Src = TempAlloca;
5325           } else {
5326             Src = Src.withElementType(STy);
5327           }
5328 
5329           assert(NumIRArgs == STy->getNumElements());
5330           for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
5331             Address EltPtr = Builder.CreateStructGEP(Src, i);
5332             llvm::Value *LI = Builder.CreateLoad(EltPtr);
5333             if (ArgHasMaybeUndefAttr)
5334               LI = Builder.CreateFreeze(LI);
5335             IRCallArgs[FirstIRArg + i] = LI;
5336           }
5337         }
5338       } else {
5339         // In the simple case, just pass the coerced loaded value.
5340         assert(NumIRArgs == 1);
5341         llvm::Value *Load =
5342             CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
5343 
5344         if (CallInfo.isCmseNSCall()) {
5345           // For certain parameter types, clear padding bits, as they may reveal
5346           // sensitive information.
5347           // Small struct/union types are passed as integer arrays.
5348           auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
5349           if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType()))
5350             Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
5351         }
5352 
5353         if (ArgHasMaybeUndefAttr)
5354           Load = Builder.CreateFreeze(Load);
5355         IRCallArgs[FirstIRArg] = Load;
5356       }
5357 
5358       break;
5359     }
5360 
5361     case ABIArgInfo::CoerceAndExpand: {
5362       auto coercionType = ArgInfo.getCoerceAndExpandType();
5363       auto layout = CGM.getDataLayout().getStructLayout(coercionType);
5364 
5365       llvm::Value *tempSize = nullptr;
5366       Address addr = Address::invalid();
5367       Address AllocaAddr = Address::invalid();
5368       if (I->isAggregate()) {
5369         addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
5370                               : I->getKnownRValue().getAggregateAddress();
5371 
5372       } else {
5373         RValue RV = I->getKnownRValue();
5374         assert(RV.isScalar()); // complex should always just be direct
5375 
5376         llvm::Type *scalarType = RV.getScalarVal()->getType();
5377         auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
5378         auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(scalarType);
5379 
5380         // Materialize to a temporary.
5381         addr = CreateTempAlloca(
5382             RV.getScalarVal()->getType(),
5383             CharUnits::fromQuantity(std::max(layout->getAlignment(), scalarAlign)),
5384             "tmp",
5385             /*ArraySize=*/nullptr, &AllocaAddr);
5386         tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
5387 
5388         Builder.CreateStore(RV.getScalarVal(), addr);
5389       }
5390 
5391       addr = addr.withElementType(coercionType);
5392 
5393       unsigned IRArgPos = FirstIRArg;
5394       for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5395         llvm::Type *eltType = coercionType->getElementType(i);
5396         if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue;
5397         Address eltAddr = Builder.CreateStructGEP(addr, i);
5398         llvm::Value *elt = Builder.CreateLoad(eltAddr);
5399         if (ArgHasMaybeUndefAttr)
5400           elt = Builder.CreateFreeze(elt);
5401         IRCallArgs[IRArgPos++] = elt;
5402       }
5403       assert(IRArgPos == FirstIRArg + NumIRArgs);
5404 
5405       if (tempSize) {
5406         EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
5407       }
5408 
5409       break;
5410     }
5411 
5412     case ABIArgInfo::Expand: {
5413       unsigned IRArgPos = FirstIRArg;
5414       ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
5415       assert(IRArgPos == FirstIRArg + NumIRArgs);
5416       break;
5417     }
5418     }
5419   }
5420 
5421   const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
5422   llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5423 
5424   // If we're using inalloca, set up that argument.
5425   if (ArgMemory.isValid()) {
5426     llvm::Value *Arg = ArgMemory.getPointer();
5427     assert(IRFunctionArgs.hasInallocaArg());
5428     IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
5429   }
5430 
5431   // 2. Prepare the function pointer.
5432 
5433   // If the callee is a bitcast of a non-variadic function to have a
5434   // variadic function pointer type, check to see if we can remove the
5435   // bitcast.  This comes up with unprototyped functions.
5436   //
5437   // This makes the IR nicer, but more importantly it ensures that we
5438   // can inline the function at -O0 if it is marked always_inline.
5439   auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
5440                                    llvm::Value *Ptr) -> llvm::Function * {
5441     if (!CalleeFT->isVarArg())
5442       return nullptr;
5443 
5444     // Get underlying value if it's a bitcast
5445     if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) {
5446       if (CE->getOpcode() == llvm::Instruction::BitCast)
5447         Ptr = CE->getOperand(0);
5448     }
5449 
5450     llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr);
5451     if (!OrigFn)
5452       return nullptr;
5453 
5454     llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
5455 
5456     // If the original type is variadic, or if any of the component types
5457     // disagree, we cannot remove the cast.
5458     if (OrigFT->isVarArg() ||
5459         OrigFT->getNumParams() != CalleeFT->getNumParams() ||
5460         OrigFT->getReturnType() != CalleeFT->getReturnType())
5461       return nullptr;
5462 
5463     for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)
5464       if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
5465         return nullptr;
5466 
5467     return OrigFn;
5468   };
5469 
5470   if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
5471     CalleePtr = OrigFn;
5472     IRFuncTy = OrigFn->getFunctionType();
5473   }
5474 
5475   // 3. Perform the actual call.
5476 
5477   // Deactivate any cleanups that we're supposed to do immediately before
5478   // the call.
5479   if (!CallArgs.getCleanupsToDeactivate().empty())
5480     deactivateArgCleanupsBeforeCall(*this, CallArgs);
5481 
5482   // Assert that the arguments we computed match up.  The IR verifier
5483   // will catch this, but this is a common enough source of problems
5484   // during IRGen changes that it's way better for debugging to catch
5485   // it ourselves here.
5486 #ifndef NDEBUG
5487   assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
5488   for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
5489     // Inalloca argument can have different type.
5490     if (IRFunctionArgs.hasInallocaArg() &&
5491         i == IRFunctionArgs.getInallocaArgNo())
5492       continue;
5493     if (i < IRFuncTy->getNumParams())
5494       assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
5495   }
5496 #endif
5497 
5498   // Update the largest vector width if any arguments have vector types.
5499   for (unsigned i = 0; i < IRCallArgs.size(); ++i)
5500     LargestVectorWidth = std::max(LargestVectorWidth,
5501                                   getMaxVectorWidth(IRCallArgs[i]->getType()));
5502 
5503   // Compute the calling convention and attributes.
5504   unsigned CallingConv;
5505   llvm::AttributeList Attrs;
5506   CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,
5507                              Callee.getAbstractInfo(), Attrs, CallingConv,
5508                              /*AttrOnCallSite=*/true,
5509                              /*IsThunk=*/false);
5510 
5511   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
5512     if (FD->hasAttr<StrictFPAttr>())
5513       // All calls within a strictfp function are marked strictfp
5514       Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5515 
5516     // If -ffast-math is enabled and the function is guarded by an
5517     // '__attribute__((optnone)) adjust the memory attribute so the BE emits the
5518     // library call instead of the intrinsic.
5519     if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)
5520       CGM.AdjustMemoryAttribute(CalleePtr->getName(), Callee.getAbstractInfo(),
5521                                 Attrs);
5522   }
5523   // Add call-site nomerge attribute if exists.
5524   if (InNoMergeAttributedStmt)
5525     Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoMerge);
5526 
5527   // Add call-site noinline attribute if exists.
5528   if (InNoInlineAttributedStmt)
5529     Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5530 
5531   // Add call-site always_inline attribute if exists.
5532   if (InAlwaysInlineAttributedStmt)
5533     Attrs =
5534         Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5535 
5536   // Apply some call-site-specific attributes.
5537   // TODO: work this into building the attribute set.
5538 
5539   // Apply always_inline to all calls within flatten functions.
5540   // FIXME: should this really take priority over __try, below?
5541   if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
5542       !InNoInlineAttributedStmt &&
5543       !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>())) {
5544     Attrs =
5545         Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);
5546   }
5547 
5548   // Disable inlining inside SEH __try blocks.
5549   if (isSEHTryScope()) {
5550     Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);
5551   }
5552 
5553   // Decide whether to use a call or an invoke.
5554   bool CannotThrow;
5555   if (currentFunctionUsesSEHTry()) {
5556     // SEH cares about asynchronous exceptions, so everything can "throw."
5557     CannotThrow = false;
5558   } else if (isCleanupPadScope() &&
5559              EHPersonality::get(*this).isMSVCXXPersonality()) {
5560     // The MSVC++ personality will implicitly terminate the program if an
5561     // exception is thrown during a cleanup outside of a try/catch.
5562     // We don't need to model anything in IR to get this behavior.
5563     CannotThrow = true;
5564   } else {
5565     // Otherwise, nounwind call sites will never throw.
5566     CannotThrow = Attrs.hasFnAttr(llvm::Attribute::NoUnwind);
5567 
5568     if (auto *FPtr = dyn_cast<llvm::Function>(CalleePtr))
5569       if (FPtr->hasFnAttribute(llvm::Attribute::NoUnwind))
5570         CannotThrow = true;
5571   }
5572 
5573   // If we made a temporary, be sure to clean up after ourselves. Note that we
5574   // can't depend on being inside of an ExprWithCleanups, so we need to manually
5575   // pop this cleanup later on. Being eager about this is OK, since this
5576   // temporary is 'invisible' outside of the callee.
5577   if (UnusedReturnSizePtr)
5578     pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetAlloca,
5579                                          UnusedReturnSizePtr);
5580 
5581   llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
5582 
5583   SmallVector<llvm::OperandBundleDef, 1> BundleList =
5584       getBundlesForFunclet(CalleePtr);
5585 
5586   if (SanOpts.has(SanitizerKind::KCFI) &&
5587       !isa_and_nonnull<FunctionDecl>(TargetDecl))
5588     EmitKCFIOperandBundle(ConcreteCallee, BundleList);
5589 
5590   if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
5591     if (FD->hasAttr<StrictFPAttr>())
5592       // All calls within a strictfp function are marked strictfp
5593       Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);
5594 
5595   AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
5596   Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5597 
5598   AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
5599   Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
5600 
5601   // Emit the actual call/invoke instruction.
5602   llvm::CallBase *CI;
5603   if (!InvokeDest) {
5604     CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList);
5605   } else {
5606     llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
5607     CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs,
5608                               BundleList);
5609     EmitBlock(Cont);
5610   }
5611   if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
5612       CI->getCalledFunction()->getName().starts_with("_Z4sqrt")) {
5613     SetSqrtFPAccuracy(CI);
5614   }
5615   if (callOrInvoke)
5616     *callOrInvoke = CI;
5617 
5618   // If this is within a function that has the guard(nocf) attribute and is an
5619   // indirect call, add the "guard_nocf" attribute to this call to indicate that
5620   // Control Flow Guard checks should not be added, even if the call is inlined.
5621   if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
5622     if (const auto *A = FD->getAttr<CFGuardAttr>()) {
5623       if (A->getGuard() == CFGuardAttr::GuardArg::nocf && !CI->getCalledFunction())
5624         Attrs = Attrs.addFnAttribute(getLLVMContext(), "guard_nocf");
5625     }
5626   }
5627 
5628   // Apply the attributes and calling convention.
5629   CI->setAttributes(Attrs);
5630   CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
5631 
5632   // Apply various metadata.
5633 
5634   if (!CI->getType()->isVoidTy())
5635     CI->setName("call");
5636 
5637   // Update largest vector width from the return type.
5638   LargestVectorWidth =
5639       std::max(LargestVectorWidth, getMaxVectorWidth(CI->getType()));
5640 
5641   // Insert instrumentation or attach profile metadata at indirect call sites.
5642   // For more details, see the comment before the definition of
5643   // IPVK_IndirectCallTarget in InstrProfData.inc.
5644   if (!CI->getCalledFunction())
5645     PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget,
5646                      CI, CalleePtr);
5647 
5648   // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
5649   // optimizer it can aggressively ignore unwind edges.
5650   if (CGM.getLangOpts().ObjCAutoRefCount)
5651     AddObjCARCExceptionMetadata(CI);
5652 
5653   // Set tail call kind if necessary.
5654   if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
5655     if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())
5656       Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
5657     else if (IsMustTail)
5658       Call->setTailCallKind(llvm::CallInst::TCK_MustTail);
5659   }
5660 
5661   // Add metadata for calls to MSAllocator functions
5662   if (getDebugInfo() && TargetDecl &&
5663       TargetDecl->hasAttr<MSAllocatorAttr>())
5664     getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy->getPointeeType(), Loc);
5665 
5666   // Add metadata if calling an __attribute__((error(""))) or warning fn.
5667   if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {
5668     llvm::ConstantInt *Line =
5669         llvm::ConstantInt::get(Int32Ty, Loc.getRawEncoding());
5670     llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(Line);
5671     llvm::MDTuple *MDT = llvm::MDNode::get(getLLVMContext(), {MD});
5672     CI->setMetadata("srcloc", MDT);
5673   }
5674 
5675   // 4. Finish the call.
5676 
5677   // If the call doesn't return, finish the basic block and clear the
5678   // insertion point; this allows the rest of IRGen to discard
5679   // unreachable code.
5680   if (CI->doesNotReturn()) {
5681     if (UnusedReturnSizePtr)
5682       PopCleanupBlock();
5683 
5684     // Strip away the noreturn attribute to better diagnose unreachable UB.
5685     if (SanOpts.has(SanitizerKind::Unreachable)) {
5686       // Also remove from function since CallBase::hasFnAttr additionally checks
5687       // attributes of the called function.
5688       if (auto *F = CI->getCalledFunction())
5689         F->removeFnAttr(llvm::Attribute::NoReturn);
5690       CI->removeFnAttr(llvm::Attribute::NoReturn);
5691 
5692       // Avoid incompatibility with ASan which relies on the `noreturn`
5693       // attribute to insert handler calls.
5694       if (SanOpts.hasOneOf(SanitizerKind::Address |
5695                            SanitizerKind::KernelAddress)) {
5696         SanitizerScope SanScope(this);
5697         llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
5698         Builder.SetInsertPoint(CI);
5699         auto *FnType = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
5700         llvm::FunctionCallee Fn =
5701             CGM.CreateRuntimeFunction(FnType, "__asan_handle_no_return");
5702         EmitNounwindRuntimeCall(Fn);
5703       }
5704     }
5705 
5706     EmitUnreachable(Loc);
5707     Builder.ClearInsertionPoint();
5708 
5709     // FIXME: For now, emit a dummy basic block because expr emitters in
5710     // generally are not ready to handle emitting expressions at unreachable
5711     // points.
5712     EnsureInsertPoint();
5713 
5714     // Return a reasonable RValue.
5715     return GetUndefRValue(RetTy);
5716   }
5717 
5718   // If this is a musttail call, return immediately. We do not branch to the
5719   // epilogue in this case.
5720   if (IsMustTail) {
5721     for (auto it = EHStack.find(CurrentCleanupScopeDepth); it != EHStack.end();
5722          ++it) {
5723       EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(&*it);
5724       if (!(Cleanup && Cleanup->getCleanup()->isRedundantBeforeReturn()))
5725         CGM.ErrorUnsupported(MustTailCall, "tail call skipping over cleanups");
5726     }
5727     if (CI->getType()->isVoidTy())
5728       Builder.CreateRetVoid();
5729     else
5730       Builder.CreateRet(CI);
5731     Builder.ClearInsertionPoint();
5732     EnsureInsertPoint();
5733     return GetUndefRValue(RetTy);
5734   }
5735 
5736   // Perform the swifterror writeback.
5737   if (swiftErrorTemp.isValid()) {
5738     llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp);
5739     Builder.CreateStore(errorResult, swiftErrorArg);
5740   }
5741 
5742   // Emit any call-associated writebacks immediately.  Arguably this
5743   // should happen after any return-value munging.
5744   if (CallArgs.hasWritebacks())
5745     emitWritebacks(*this, CallArgs);
5746 
5747   // The stack cleanup for inalloca arguments has to run out of the normal
5748   // lexical order, so deactivate it and run it manually here.
5749   CallArgs.freeArgumentMemory(*this);
5750 
5751   // Extract the return value.
5752   RValue Ret = [&] {
5753     switch (RetAI.getKind()) {
5754     case ABIArgInfo::CoerceAndExpand: {
5755       auto coercionType = RetAI.getCoerceAndExpandType();
5756 
5757       Address addr = SRetPtr.withElementType(coercionType);
5758 
5759       assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
5760       bool requiresExtract = isa<llvm::StructType>(CI->getType());
5761 
5762       unsigned unpaddedIndex = 0;
5763       for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
5764         llvm::Type *eltType = coercionType->getElementType(i);
5765         if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue;
5766         Address eltAddr = Builder.CreateStructGEP(addr, i);
5767         llvm::Value *elt = CI;
5768         if (requiresExtract)
5769           elt = Builder.CreateExtractValue(elt, unpaddedIndex++);
5770         else
5771           assert(unpaddedIndex == 0);
5772         Builder.CreateStore(elt, eltAddr);
5773       }
5774       [[fallthrough]];
5775     }
5776 
5777     case ABIArgInfo::InAlloca:
5778     case ABIArgInfo::Indirect: {
5779       RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());
5780       if (UnusedReturnSizePtr)
5781         PopCleanupBlock();
5782       return ret;
5783     }
5784 
5785     case ABIArgInfo::Ignore:
5786       // If we are ignoring an argument that had a result, make sure to
5787       // construct the appropriate return value for our caller.
5788       return GetUndefRValue(RetTy);
5789 
5790     case ABIArgInfo::Extend:
5791     case ABIArgInfo::Direct: {
5792       llvm::Type *RetIRTy = ConvertType(RetTy);
5793       if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) {
5794         switch (getEvaluationKind(RetTy)) {
5795         case TEK_Complex: {
5796           llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
5797           llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
5798           return RValue::getComplex(std::make_pair(Real, Imag));
5799         }
5800         case TEK_Aggregate: {
5801           Address DestPtr = ReturnValue.getValue();
5802           bool DestIsVolatile = ReturnValue.isVolatile();
5803 
5804           if (!DestPtr.isValid()) {
5805             DestPtr = CreateMemTemp(RetTy, "agg.tmp");
5806             DestIsVolatile = false;
5807           }
5808           EmitAggregateStore(CI, DestPtr, DestIsVolatile);
5809           return RValue::getAggregate(DestPtr);
5810         }
5811         case TEK_Scalar: {
5812           // If the argument doesn't match, perform a bitcast to coerce it.  This
5813           // can happen due to trivial type mismatches.
5814           llvm::Value *V = CI;
5815           if (V->getType() != RetIRTy)
5816             V = Builder.CreateBitCast(V, RetIRTy);
5817           return RValue::get(V);
5818         }
5819         }
5820         llvm_unreachable("bad evaluation kind");
5821       }
5822 
5823       // If coercing a fixed vector from a scalable vector for ABI
5824       // compatibility, and the types match, use the llvm.vector.extract
5825       // intrinsic to perform the conversion.
5826       if (auto *FixedDst = dyn_cast<llvm::FixedVectorType>(RetIRTy)) {
5827         llvm::Value *V = CI;
5828         if (auto *ScalableSrc = dyn_cast<llvm::ScalableVectorType>(V->getType())) {
5829           if (FixedDst->getElementType() == ScalableSrc->getElementType()) {
5830             llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
5831             V = Builder.CreateExtractVector(FixedDst, V, Zero, "cast.fixed");
5832             return RValue::get(V);
5833           }
5834         }
5835       }
5836 
5837       Address DestPtr = ReturnValue.getValue();
5838       bool DestIsVolatile = ReturnValue.isVolatile();
5839 
5840       if (!DestPtr.isValid()) {
5841         DestPtr = CreateMemTemp(RetTy, "coerce");
5842         DestIsVolatile = false;
5843       }
5844 
5845       // An empty record can overlap other data (if declared with
5846       // no_unique_address); omit the store for such types - as there is no
5847       // actual data to store.
5848       if (!isEmptyRecord(getContext(), RetTy, true)) {
5849         // If the value is offset in memory, apply the offset now.
5850         Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);
5851         CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
5852       }
5853 
5854       return convertTempToRValue(DestPtr, RetTy, SourceLocation());
5855     }
5856 
5857     case ABIArgInfo::Expand:
5858     case ABIArgInfo::IndirectAliased:
5859       llvm_unreachable("Invalid ABI kind for return argument");
5860     }
5861 
5862     llvm_unreachable("Unhandled ABIArgInfo::Kind");
5863   } ();
5864 
5865   // Emit the assume_aligned check on the return value.
5866   if (Ret.isScalar() && TargetDecl) {
5867     AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
5868     AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
5869   }
5870 
5871   // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
5872   // we can't use the full cleanup mechanism.
5873   for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
5874     LifetimeEnd.Emit(*this, /*Flags=*/{});
5875 
5876   if (!ReturnValue.isExternallyDestructed() &&
5877       RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)
5878     pushDestroy(QualType::DK_nontrivial_c_struct, Ret.getAggregateAddress(),
5879                 RetTy);
5880 
5881   return Ret;
5882 }
5883 
5884 CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
5885   if (isVirtual()) {
5886     const CallExpr *CE = getVirtualCallExpr();
5887     return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
5888         CGF, getVirtualMethodDecl(), getThisAddress(), getVirtualFunctionType(),
5889         CE ? CE->getBeginLoc() : SourceLocation());
5890   }
5891 
5892   return *this;
5893 }
5894 
5895 /* VarArg handling */
5896 
5897 Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) {
5898   VAListAddr = VE->isMicrosoftABI()
5899                  ? EmitMSVAListRef(VE->getSubExpr())
5900                  : EmitVAListRef(VE->getSubExpr());
5901   QualType Ty = VE->getType();
5902   if (VE->isMicrosoftABI())
5903     return CGM.getTypes().getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty);
5904   return CGM.getTypes().getABIInfo().EmitVAArg(*this, VAListAddr, Ty);
5905 }
5906