1 //===--- CGCall.cpp - Encapsulate calling convention details --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // These classes wrap the information about a call or function 10 // definition used to handle ABI compliancy. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CGCall.h" 15 #include "ABIInfo.h" 16 #include "CGBlocks.h" 17 #include "CGCXXABI.h" 18 #include "CGCleanup.h" 19 #include "CGRecordLayout.h" 20 #include "CodeGenFunction.h" 21 #include "CodeGenModule.h" 22 #include "TargetInfo.h" 23 #include "clang/AST/Attr.h" 24 #include "clang/AST/Decl.h" 25 #include "clang/AST/DeclCXX.h" 26 #include "clang/AST/DeclObjC.h" 27 #include "clang/Basic/CodeGenOptions.h" 28 #include "clang/Basic/TargetBuiltins.h" 29 #include "clang/Basic/TargetInfo.h" 30 #include "clang/CodeGen/CGFunctionInfo.h" 31 #include "clang/CodeGen/SwiftCallingConv.h" 32 #include "llvm/ADT/StringExtras.h" 33 #include "llvm/Analysis/ValueTracking.h" 34 #include "llvm/IR/Assumptions.h" 35 #include "llvm/IR/Attributes.h" 36 #include "llvm/IR/CallingConv.h" 37 #include "llvm/IR/DataLayout.h" 38 #include "llvm/IR/InlineAsm.h" 39 #include "llvm/IR/IntrinsicInst.h" 40 #include "llvm/IR/Intrinsics.h" 41 #include "llvm/Transforms/Utils/Local.h" 42 using namespace clang; 43 using namespace CodeGen; 44 45 /***/ 46 47 unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) { 48 switch (CC) { 49 default: return llvm::CallingConv::C; 50 case CC_X86StdCall: return llvm::CallingConv::X86_StdCall; 51 case CC_X86FastCall: return llvm::CallingConv::X86_FastCall; 52 case CC_X86RegCall: return llvm::CallingConv::X86_RegCall; 53 case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall; 54 case CC_Win64: return llvm::CallingConv::Win64; 55 case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV; 56 case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS; 57 case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP; 58 case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI; 59 // TODO: Add support for __pascal to LLVM. 60 case CC_X86Pascal: return llvm::CallingConv::C; 61 // TODO: Add support for __vectorcall to LLVM. 62 case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall; 63 case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall; 64 case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC; 65 case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv(); 66 case CC_PreserveMost: return llvm::CallingConv::PreserveMost; 67 case CC_PreserveAll: return llvm::CallingConv::PreserveAll; 68 case CC_Swift: return llvm::CallingConv::Swift; 69 } 70 } 71 72 /// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR 73 /// qualification. Either or both of RD and MD may be null. A null RD indicates 74 /// that there is no meaningful 'this' type, and a null MD can occur when 75 /// calling a method pointer. 76 CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD, 77 const CXXMethodDecl *MD) { 78 QualType RecTy; 79 if (RD) 80 RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal(); 81 else 82 RecTy = Context.VoidTy; 83 84 if (MD) 85 RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace()); 86 return Context.getPointerType(CanQualType::CreateUnsafe(RecTy)); 87 } 88 89 /// Returns the canonical formal type of the given C++ method. 90 static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) { 91 return MD->getType()->getCanonicalTypeUnqualified() 92 .getAs<FunctionProtoType>(); 93 } 94 95 /// Returns the "extra-canonicalized" return type, which discards 96 /// qualifiers on the return type. Codegen doesn't care about them, 97 /// and it makes ABI code a little easier to be able to assume that 98 /// all parameter and return types are top-level unqualified. 99 static CanQualType GetReturnType(QualType RetTy) { 100 return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType(); 101 } 102 103 /// Arrange the argument and result information for a value of the given 104 /// unprototyped freestanding function type. 105 const CGFunctionInfo & 106 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) { 107 // When translating an unprototyped function type, always use a 108 // variadic type. 109 return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(), 110 /*instanceMethod=*/false, 111 /*chainCall=*/false, None, 112 FTNP->getExtInfo(), {}, RequiredArgs(0)); 113 } 114 115 static void addExtParameterInfosForCall( 116 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> ¶mInfos, 117 const FunctionProtoType *proto, 118 unsigned prefixArgs, 119 unsigned totalArgs) { 120 assert(proto->hasExtParameterInfos()); 121 assert(paramInfos.size() <= prefixArgs); 122 assert(proto->getNumParams() + prefixArgs <= totalArgs); 123 124 paramInfos.reserve(totalArgs); 125 126 // Add default infos for any prefix args that don't already have infos. 127 paramInfos.resize(prefixArgs); 128 129 // Add infos for the prototype. 130 for (const auto &ParamInfo : proto->getExtParameterInfos()) { 131 paramInfos.push_back(ParamInfo); 132 // pass_object_size params have no parameter info. 133 if (ParamInfo.hasPassObjectSize()) 134 paramInfos.emplace_back(); 135 } 136 137 assert(paramInfos.size() <= totalArgs && 138 "Did we forget to insert pass_object_size args?"); 139 // Add default infos for the variadic and/or suffix arguments. 140 paramInfos.resize(totalArgs); 141 } 142 143 /// Adds the formal parameters in FPT to the given prefix. If any parameter in 144 /// FPT has pass_object_size attrs, then we'll add parameters for those, too. 145 static void appendParameterTypes(const CodeGenTypes &CGT, 146 SmallVectorImpl<CanQualType> &prefix, 147 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> ¶mInfos, 148 CanQual<FunctionProtoType> FPT) { 149 // Fast path: don't touch param info if we don't need to. 150 if (!FPT->hasExtParameterInfos()) { 151 assert(paramInfos.empty() && 152 "We have paramInfos, but the prototype doesn't?"); 153 prefix.append(FPT->param_type_begin(), FPT->param_type_end()); 154 return; 155 } 156 157 unsigned PrefixSize = prefix.size(); 158 // In the vast majority of cases, we'll have precisely FPT->getNumParams() 159 // parameters; the only thing that can change this is the presence of 160 // pass_object_size. So, we preallocate for the common case. 161 prefix.reserve(prefix.size() + FPT->getNumParams()); 162 163 auto ExtInfos = FPT->getExtParameterInfos(); 164 assert(ExtInfos.size() == FPT->getNumParams()); 165 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) { 166 prefix.push_back(FPT->getParamType(I)); 167 if (ExtInfos[I].hasPassObjectSize()) 168 prefix.push_back(CGT.getContext().getSizeType()); 169 } 170 171 addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize, 172 prefix.size()); 173 } 174 175 /// Arrange the LLVM function layout for a value of the given function 176 /// type, on top of any implicit parameters already stored. 177 static const CGFunctionInfo & 178 arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod, 179 SmallVectorImpl<CanQualType> &prefix, 180 CanQual<FunctionProtoType> FTP) { 181 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; 182 RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size()); 183 // FIXME: Kill copy. 184 appendParameterTypes(CGT, prefix, paramInfos, FTP); 185 CanQualType resultType = FTP->getReturnType().getUnqualifiedType(); 186 187 return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod, 188 /*chainCall=*/false, prefix, 189 FTP->getExtInfo(), paramInfos, 190 Required); 191 } 192 193 /// Arrange the argument and result information for a value of the 194 /// given freestanding function type. 195 const CGFunctionInfo & 196 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) { 197 SmallVector<CanQualType, 16> argTypes; 198 return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes, 199 FTP); 200 } 201 202 static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D, 203 bool IsWindows) { 204 // Set the appropriate calling convention for the Function. 205 if (D->hasAttr<StdCallAttr>()) 206 return CC_X86StdCall; 207 208 if (D->hasAttr<FastCallAttr>()) 209 return CC_X86FastCall; 210 211 if (D->hasAttr<RegCallAttr>()) 212 return CC_X86RegCall; 213 214 if (D->hasAttr<ThisCallAttr>()) 215 return CC_X86ThisCall; 216 217 if (D->hasAttr<VectorCallAttr>()) 218 return CC_X86VectorCall; 219 220 if (D->hasAttr<PascalAttr>()) 221 return CC_X86Pascal; 222 223 if (PcsAttr *PCS = D->getAttr<PcsAttr>()) 224 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP); 225 226 if (D->hasAttr<AArch64VectorPcsAttr>()) 227 return CC_AArch64VectorCall; 228 229 if (D->hasAttr<IntelOclBiccAttr>()) 230 return CC_IntelOclBicc; 231 232 if (D->hasAttr<MSABIAttr>()) 233 return IsWindows ? CC_C : CC_Win64; 234 235 if (D->hasAttr<SysVABIAttr>()) 236 return IsWindows ? CC_X86_64SysV : CC_C; 237 238 if (D->hasAttr<PreserveMostAttr>()) 239 return CC_PreserveMost; 240 241 if (D->hasAttr<PreserveAllAttr>()) 242 return CC_PreserveAll; 243 244 return CC_C; 245 } 246 247 /// Arrange the argument and result information for a call to an 248 /// unknown C++ non-static member function of the given abstract type. 249 /// (A null RD means we don't have any meaningful "this" argument type, 250 /// so fall back to a generic pointer type). 251 /// The member function must be an ordinary function, i.e. not a 252 /// constructor or destructor. 253 const CGFunctionInfo & 254 CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD, 255 const FunctionProtoType *FTP, 256 const CXXMethodDecl *MD) { 257 SmallVector<CanQualType, 16> argTypes; 258 259 // Add the 'this' pointer. 260 argTypes.push_back(DeriveThisType(RD, MD)); 261 262 return ::arrangeLLVMFunctionInfo( 263 *this, true, argTypes, 264 FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>()); 265 } 266 267 /// Set calling convention for CUDA/HIP kernel. 268 static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM, 269 const FunctionDecl *FD) { 270 if (FD->hasAttr<CUDAGlobalAttr>()) { 271 const FunctionType *FT = FTy->getAs<FunctionType>(); 272 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT); 273 FTy = FT->getCanonicalTypeUnqualified(); 274 } 275 } 276 277 /// Arrange the argument and result information for a declaration or 278 /// definition of the given C++ non-static member function. The 279 /// member function must be an ordinary function, i.e. not a 280 /// constructor or destructor. 281 const CGFunctionInfo & 282 CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) { 283 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!"); 284 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!"); 285 286 CanQualType FT = GetFormalType(MD).getAs<Type>(); 287 setCUDAKernelCallingConvention(FT, CGM, MD); 288 auto prototype = FT.getAs<FunctionProtoType>(); 289 290 if (MD->isInstance()) { 291 // The abstract case is perfectly fine. 292 const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD); 293 return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD); 294 } 295 296 return arrangeFreeFunctionType(prototype); 297 } 298 299 bool CodeGenTypes::inheritingCtorHasParams( 300 const InheritedConstructor &Inherited, CXXCtorType Type) { 301 // Parameters are unnecessary if we're constructing a base class subobject 302 // and the inherited constructor lives in a virtual base. 303 return Type == Ctor_Complete || 304 !Inherited.getShadowDecl()->constructsVirtualBase() || 305 !Target.getCXXABI().hasConstructorVariants(); 306 } 307 308 const CGFunctionInfo & 309 CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) { 310 auto *MD = cast<CXXMethodDecl>(GD.getDecl()); 311 312 SmallVector<CanQualType, 16> argTypes; 313 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; 314 argTypes.push_back(DeriveThisType(MD->getParent(), MD)); 315 316 bool PassParams = true; 317 318 if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) { 319 // A base class inheriting constructor doesn't get forwarded arguments 320 // needed to construct a virtual base (or base class thereof). 321 if (auto Inherited = CD->getInheritedConstructor()) 322 PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType()); 323 } 324 325 CanQual<FunctionProtoType> FTP = GetFormalType(MD); 326 327 // Add the formal parameters. 328 if (PassParams) 329 appendParameterTypes(*this, argTypes, paramInfos, FTP); 330 331 CGCXXABI::AddedStructorArgCounts AddedArgs = 332 TheCXXABI.buildStructorSignature(GD, argTypes); 333 if (!paramInfos.empty()) { 334 // Note: prefix implies after the first param. 335 if (AddedArgs.Prefix) 336 paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix, 337 FunctionProtoType::ExtParameterInfo{}); 338 if (AddedArgs.Suffix) 339 paramInfos.append(AddedArgs.Suffix, 340 FunctionProtoType::ExtParameterInfo{}); 341 } 342 343 RequiredArgs required = 344 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size()) 345 : RequiredArgs::All); 346 347 FunctionType::ExtInfo extInfo = FTP->getExtInfo(); 348 CanQualType resultType = TheCXXABI.HasThisReturn(GD) 349 ? argTypes.front() 350 : TheCXXABI.hasMostDerivedReturn(GD) 351 ? CGM.getContext().VoidPtrTy 352 : Context.VoidTy; 353 return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true, 354 /*chainCall=*/false, argTypes, extInfo, 355 paramInfos, required); 356 } 357 358 static SmallVector<CanQualType, 16> 359 getArgTypesForCall(ASTContext &ctx, const CallArgList &args) { 360 SmallVector<CanQualType, 16> argTypes; 361 for (auto &arg : args) 362 argTypes.push_back(ctx.getCanonicalParamType(arg.Ty)); 363 return argTypes; 364 } 365 366 static SmallVector<CanQualType, 16> 367 getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) { 368 SmallVector<CanQualType, 16> argTypes; 369 for (auto &arg : args) 370 argTypes.push_back(ctx.getCanonicalParamType(arg->getType())); 371 return argTypes; 372 } 373 374 static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> 375 getExtParameterInfosForCall(const FunctionProtoType *proto, 376 unsigned prefixArgs, unsigned totalArgs) { 377 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result; 378 if (proto->hasExtParameterInfos()) { 379 addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs); 380 } 381 return result; 382 } 383 384 /// Arrange a call to a C++ method, passing the given arguments. 385 /// 386 /// ExtraPrefixArgs is the number of ABI-specific args passed after the `this` 387 /// parameter. 388 /// ExtraSuffixArgs is the number of ABI-specific args passed at the end of 389 /// args. 390 /// PassProtoArgs indicates whether `args` has args for the parameters in the 391 /// given CXXConstructorDecl. 392 const CGFunctionInfo & 393 CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args, 394 const CXXConstructorDecl *D, 395 CXXCtorType CtorKind, 396 unsigned ExtraPrefixArgs, 397 unsigned ExtraSuffixArgs, 398 bool PassProtoArgs) { 399 // FIXME: Kill copy. 400 SmallVector<CanQualType, 16> ArgTypes; 401 for (const auto &Arg : args) 402 ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty)); 403 404 // +1 for implicit this, which should always be args[0]. 405 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs; 406 407 CanQual<FunctionProtoType> FPT = GetFormalType(D); 408 RequiredArgs Required = PassProtoArgs 409 ? RequiredArgs::forPrototypePlus( 410 FPT, TotalPrefixArgs + ExtraSuffixArgs) 411 : RequiredArgs::All; 412 413 GlobalDecl GD(D, CtorKind); 414 CanQualType ResultType = TheCXXABI.HasThisReturn(GD) 415 ? ArgTypes.front() 416 : TheCXXABI.hasMostDerivedReturn(GD) 417 ? CGM.getContext().VoidPtrTy 418 : Context.VoidTy; 419 420 FunctionType::ExtInfo Info = FPT->getExtInfo(); 421 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos; 422 // If the prototype args are elided, we should only have ABI-specific args, 423 // which never have param info. 424 if (PassProtoArgs && FPT->hasExtParameterInfos()) { 425 // ABI-specific suffix arguments are treated the same as variadic arguments. 426 addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs, 427 ArgTypes.size()); 428 } 429 return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true, 430 /*chainCall=*/false, ArgTypes, Info, 431 ParamInfos, Required); 432 } 433 434 /// Arrange the argument and result information for the declaration or 435 /// definition of the given function. 436 const CGFunctionInfo & 437 CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) { 438 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD)) 439 if (MD->isInstance()) 440 return arrangeCXXMethodDeclaration(MD); 441 442 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified(); 443 444 assert(isa<FunctionType>(FTy)); 445 setCUDAKernelCallingConvention(FTy, CGM, FD); 446 447 // When declaring a function without a prototype, always use a 448 // non-variadic type. 449 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) { 450 return arrangeLLVMFunctionInfo( 451 noProto->getReturnType(), /*instanceMethod=*/false, 452 /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All); 453 } 454 455 return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>()); 456 } 457 458 /// Arrange the argument and result information for the declaration or 459 /// definition of an Objective-C method. 460 const CGFunctionInfo & 461 CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) { 462 // It happens that this is the same as a call with no optional 463 // arguments, except also using the formal 'self' type. 464 return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType()); 465 } 466 467 /// Arrange the argument and result information for the function type 468 /// through which to perform a send to the given Objective-C method, 469 /// using the given receiver type. The receiver type is not always 470 /// the 'self' type of the method or even an Objective-C pointer type. 471 /// This is *not* the right method for actually performing such a 472 /// message send, due to the possibility of optional arguments. 473 const CGFunctionInfo & 474 CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD, 475 QualType receiverType) { 476 SmallVector<CanQualType, 16> argTys; 477 SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2); 478 argTys.push_back(Context.getCanonicalParamType(receiverType)); 479 argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType())); 480 // FIXME: Kill copy? 481 for (const auto *I : MD->parameters()) { 482 argTys.push_back(Context.getCanonicalParamType(I->getType())); 483 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape( 484 I->hasAttr<NoEscapeAttr>()); 485 extParamInfos.push_back(extParamInfo); 486 } 487 488 FunctionType::ExtInfo einfo; 489 bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows(); 490 einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows)); 491 492 if (getContext().getLangOpts().ObjCAutoRefCount && 493 MD->hasAttr<NSReturnsRetainedAttr>()) 494 einfo = einfo.withProducesResult(true); 495 496 RequiredArgs required = 497 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All); 498 499 return arrangeLLVMFunctionInfo( 500 GetReturnType(MD->getReturnType()), /*instanceMethod=*/false, 501 /*chainCall=*/false, argTys, einfo, extParamInfos, required); 502 } 503 504 const CGFunctionInfo & 505 CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType, 506 const CallArgList &args) { 507 auto argTypes = getArgTypesForCall(Context, args); 508 FunctionType::ExtInfo einfo; 509 510 return arrangeLLVMFunctionInfo( 511 GetReturnType(returnType), /*instanceMethod=*/false, 512 /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All); 513 } 514 515 const CGFunctionInfo & 516 CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) { 517 // FIXME: Do we need to handle ObjCMethodDecl? 518 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); 519 520 if (isa<CXXConstructorDecl>(GD.getDecl()) || 521 isa<CXXDestructorDecl>(GD.getDecl())) 522 return arrangeCXXStructorDeclaration(GD); 523 524 return arrangeFunctionDeclaration(FD); 525 } 526 527 /// Arrange a thunk that takes 'this' as the first parameter followed by 528 /// varargs. Return a void pointer, regardless of the actual return type. 529 /// The body of the thunk will end in a musttail call to a function of the 530 /// correct type, and the caller will bitcast the function to the correct 531 /// prototype. 532 const CGFunctionInfo & 533 CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) { 534 assert(MD->isVirtual() && "only methods have thunks"); 535 CanQual<FunctionProtoType> FTP = GetFormalType(MD); 536 CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)}; 537 return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false, 538 /*chainCall=*/false, ArgTys, 539 FTP->getExtInfo(), {}, RequiredArgs(1)); 540 } 541 542 const CGFunctionInfo & 543 CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD, 544 CXXCtorType CT) { 545 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure); 546 547 CanQual<FunctionProtoType> FTP = GetFormalType(CD); 548 SmallVector<CanQualType, 2> ArgTys; 549 const CXXRecordDecl *RD = CD->getParent(); 550 ArgTys.push_back(DeriveThisType(RD, CD)); 551 if (CT == Ctor_CopyingClosure) 552 ArgTys.push_back(*FTP->param_type_begin()); 553 if (RD->getNumVBases() > 0) 554 ArgTys.push_back(Context.IntTy); 555 CallingConv CC = Context.getDefaultCallingConvention( 556 /*IsVariadic=*/false, /*IsCXXMethod=*/true); 557 return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true, 558 /*chainCall=*/false, ArgTys, 559 FunctionType::ExtInfo(CC), {}, 560 RequiredArgs::All); 561 } 562 563 /// Arrange a call as unto a free function, except possibly with an 564 /// additional number of formal parameters considered required. 565 static const CGFunctionInfo & 566 arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, 567 CodeGenModule &CGM, 568 const CallArgList &args, 569 const FunctionType *fnType, 570 unsigned numExtraRequiredArgs, 571 bool chainCall) { 572 assert(args.size() >= numExtraRequiredArgs); 573 574 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; 575 576 // In most cases, there are no optional arguments. 577 RequiredArgs required = RequiredArgs::All; 578 579 // If we have a variadic prototype, the required arguments are the 580 // extra prefix plus the arguments in the prototype. 581 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) { 582 if (proto->isVariadic()) 583 required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs); 584 585 if (proto->hasExtParameterInfos()) 586 addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs, 587 args.size()); 588 589 // If we don't have a prototype at all, but we're supposed to 590 // explicitly use the variadic convention for unprototyped calls, 591 // treat all of the arguments as required but preserve the nominal 592 // possibility of variadics. 593 } else if (CGM.getTargetCodeGenInfo() 594 .isNoProtoCallVariadic(args, 595 cast<FunctionNoProtoType>(fnType))) { 596 required = RequiredArgs(args.size()); 597 } 598 599 // FIXME: Kill copy. 600 SmallVector<CanQualType, 16> argTypes; 601 for (const auto &arg : args) 602 argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty)); 603 return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()), 604 /*instanceMethod=*/false, chainCall, 605 argTypes, fnType->getExtInfo(), paramInfos, 606 required); 607 } 608 609 /// Figure out the rules for calling a function with the given formal 610 /// type using the given arguments. The arguments are necessary 611 /// because the function might be unprototyped, in which case it's 612 /// target-dependent in crazy ways. 613 const CGFunctionInfo & 614 CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args, 615 const FunctionType *fnType, 616 bool chainCall) { 617 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 618 chainCall ? 1 : 0, chainCall); 619 } 620 621 /// A block function is essentially a free function with an 622 /// extra implicit argument. 623 const CGFunctionInfo & 624 CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args, 625 const FunctionType *fnType) { 626 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1, 627 /*chainCall=*/false); 628 } 629 630 const CGFunctionInfo & 631 CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto, 632 const FunctionArgList ¶ms) { 633 auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size()); 634 auto argTypes = getArgTypesForDeclaration(Context, params); 635 636 return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()), 637 /*instanceMethod*/ false, /*chainCall*/ false, 638 argTypes, proto->getExtInfo(), paramInfos, 639 RequiredArgs::forPrototypePlus(proto, 1)); 640 } 641 642 const CGFunctionInfo & 643 CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType, 644 const CallArgList &args) { 645 // FIXME: Kill copy. 646 SmallVector<CanQualType, 16> argTypes; 647 for (const auto &Arg : args) 648 argTypes.push_back(Context.getCanonicalParamType(Arg.Ty)); 649 return arrangeLLVMFunctionInfo( 650 GetReturnType(resultType), /*instanceMethod=*/false, 651 /*chainCall=*/false, argTypes, FunctionType::ExtInfo(), 652 /*paramInfos=*/ {}, RequiredArgs::All); 653 } 654 655 const CGFunctionInfo & 656 CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType, 657 const FunctionArgList &args) { 658 auto argTypes = getArgTypesForDeclaration(Context, args); 659 660 return arrangeLLVMFunctionInfo( 661 GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false, 662 argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All); 663 } 664 665 const CGFunctionInfo & 666 CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType, 667 ArrayRef<CanQualType> argTypes) { 668 return arrangeLLVMFunctionInfo( 669 resultType, /*instanceMethod=*/false, /*chainCall=*/false, 670 argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All); 671 } 672 673 /// Arrange a call to a C++ method, passing the given arguments. 674 /// 675 /// numPrefixArgs is the number of ABI-specific prefix arguments we have. It 676 /// does not count `this`. 677 const CGFunctionInfo & 678 CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args, 679 const FunctionProtoType *proto, 680 RequiredArgs required, 681 unsigned numPrefixArgs) { 682 assert(numPrefixArgs + 1 <= args.size() && 683 "Emitting a call with less args than the required prefix?"); 684 // Add one to account for `this`. It's a bit awkward here, but we don't count 685 // `this` in similar places elsewhere. 686 auto paramInfos = 687 getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size()); 688 689 // FIXME: Kill copy. 690 auto argTypes = getArgTypesForCall(Context, args); 691 692 FunctionType::ExtInfo info = proto->getExtInfo(); 693 return arrangeLLVMFunctionInfo( 694 GetReturnType(proto->getReturnType()), /*instanceMethod=*/true, 695 /*chainCall=*/false, argTypes, info, paramInfos, required); 696 } 697 698 const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() { 699 return arrangeLLVMFunctionInfo( 700 getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false, 701 None, FunctionType::ExtInfo(), {}, RequiredArgs::All); 702 } 703 704 const CGFunctionInfo & 705 CodeGenTypes::arrangeCall(const CGFunctionInfo &signature, 706 const CallArgList &args) { 707 assert(signature.arg_size() <= args.size()); 708 if (signature.arg_size() == args.size()) 709 return signature; 710 711 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; 712 auto sigParamInfos = signature.getExtParameterInfos(); 713 if (!sigParamInfos.empty()) { 714 paramInfos.append(sigParamInfos.begin(), sigParamInfos.end()); 715 paramInfos.resize(args.size()); 716 } 717 718 auto argTypes = getArgTypesForCall(Context, args); 719 720 assert(signature.getRequiredArgs().allowsOptionalArgs()); 721 return arrangeLLVMFunctionInfo(signature.getReturnType(), 722 signature.isInstanceMethod(), 723 signature.isChainCall(), 724 argTypes, 725 signature.getExtInfo(), 726 paramInfos, 727 signature.getRequiredArgs()); 728 } 729 730 namespace clang { 731 namespace CodeGen { 732 void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI); 733 } 734 } 735 736 /// Arrange the argument and result information for an abstract value 737 /// of a given function type. This is the method which all of the 738 /// above functions ultimately defer to. 739 const CGFunctionInfo & 740 CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType, 741 bool instanceMethod, 742 bool chainCall, 743 ArrayRef<CanQualType> argTypes, 744 FunctionType::ExtInfo info, 745 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos, 746 RequiredArgs required) { 747 assert(llvm::all_of(argTypes, 748 [](CanQualType T) { return T.isCanonicalAsParam(); })); 749 750 // Lookup or create unique function info. 751 llvm::FoldingSetNodeID ID; 752 CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos, 753 required, resultType, argTypes); 754 755 void *insertPos = nullptr; 756 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos); 757 if (FI) 758 return *FI; 759 760 unsigned CC = ClangCallConvToLLVMCallConv(info.getCC()); 761 762 // Construct the function info. We co-allocate the ArgInfos. 763 FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info, 764 paramInfos, resultType, argTypes, required); 765 FunctionInfos.InsertNode(FI, insertPos); 766 767 bool inserted = FunctionsBeingProcessed.insert(FI).second; 768 (void)inserted; 769 assert(inserted && "Recursively being processed?"); 770 771 // Compute ABI information. 772 if (CC == llvm::CallingConv::SPIR_KERNEL) { 773 // Force target independent argument handling for the host visible 774 // kernel functions. 775 computeSPIRKernelABIInfo(CGM, *FI); 776 } else if (info.getCC() == CC_Swift) { 777 swiftcall::computeABIInfo(CGM, *FI); 778 } else { 779 getABIInfo().computeInfo(*FI); 780 } 781 782 // Loop over all of the computed argument and return value info. If any of 783 // them are direct or extend without a specified coerce type, specify the 784 // default now. 785 ABIArgInfo &retInfo = FI->getReturnInfo(); 786 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr) 787 retInfo.setCoerceToType(ConvertType(FI->getReturnType())); 788 789 for (auto &I : FI->arguments()) 790 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr) 791 I.info.setCoerceToType(ConvertType(I.type)); 792 793 bool erased = FunctionsBeingProcessed.erase(FI); (void)erased; 794 assert(erased && "Not in set?"); 795 796 return *FI; 797 } 798 799 CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, 800 bool instanceMethod, 801 bool chainCall, 802 const FunctionType::ExtInfo &info, 803 ArrayRef<ExtParameterInfo> paramInfos, 804 CanQualType resultType, 805 ArrayRef<CanQualType> argTypes, 806 RequiredArgs required) { 807 assert(paramInfos.empty() || paramInfos.size() == argTypes.size()); 808 assert(!required.allowsOptionalArgs() || 809 required.getNumRequiredArgs() <= argTypes.size()); 810 811 void *buffer = 812 operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>( 813 argTypes.size() + 1, paramInfos.size())); 814 815 CGFunctionInfo *FI = new(buffer) CGFunctionInfo(); 816 FI->CallingConvention = llvmCC; 817 FI->EffectiveCallingConvention = llvmCC; 818 FI->ASTCallingConvention = info.getCC(); 819 FI->InstanceMethod = instanceMethod; 820 FI->ChainCall = chainCall; 821 FI->CmseNSCall = info.getCmseNSCall(); 822 FI->NoReturn = info.getNoReturn(); 823 FI->ReturnsRetained = info.getProducesResult(); 824 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs(); 825 FI->NoCfCheck = info.getNoCfCheck(); 826 FI->Required = required; 827 FI->HasRegParm = info.getHasRegParm(); 828 FI->RegParm = info.getRegParm(); 829 FI->ArgStruct = nullptr; 830 FI->ArgStructAlign = 0; 831 FI->NumArgs = argTypes.size(); 832 FI->HasExtParameterInfos = !paramInfos.empty(); 833 FI->getArgsBuffer()[0].type = resultType; 834 for (unsigned i = 0, e = argTypes.size(); i != e; ++i) 835 FI->getArgsBuffer()[i + 1].type = argTypes[i]; 836 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i) 837 FI->getExtParameterInfosBuffer()[i] = paramInfos[i]; 838 return FI; 839 } 840 841 /***/ 842 843 namespace { 844 // ABIArgInfo::Expand implementation. 845 846 // Specifies the way QualType passed as ABIArgInfo::Expand is expanded. 847 struct TypeExpansion { 848 enum TypeExpansionKind { 849 // Elements of constant arrays are expanded recursively. 850 TEK_ConstantArray, 851 // Record fields are expanded recursively (but if record is a union, only 852 // the field with the largest size is expanded). 853 TEK_Record, 854 // For complex types, real and imaginary parts are expanded recursively. 855 TEK_Complex, 856 // All other types are not expandable. 857 TEK_None 858 }; 859 860 const TypeExpansionKind Kind; 861 862 TypeExpansion(TypeExpansionKind K) : Kind(K) {} 863 virtual ~TypeExpansion() {} 864 }; 865 866 struct ConstantArrayExpansion : TypeExpansion { 867 QualType EltTy; 868 uint64_t NumElts; 869 870 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts) 871 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {} 872 static bool classof(const TypeExpansion *TE) { 873 return TE->Kind == TEK_ConstantArray; 874 } 875 }; 876 877 struct RecordExpansion : TypeExpansion { 878 SmallVector<const CXXBaseSpecifier *, 1> Bases; 879 880 SmallVector<const FieldDecl *, 1> Fields; 881 882 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases, 883 SmallVector<const FieldDecl *, 1> &&Fields) 884 : TypeExpansion(TEK_Record), Bases(std::move(Bases)), 885 Fields(std::move(Fields)) {} 886 static bool classof(const TypeExpansion *TE) { 887 return TE->Kind == TEK_Record; 888 } 889 }; 890 891 struct ComplexExpansion : TypeExpansion { 892 QualType EltTy; 893 894 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {} 895 static bool classof(const TypeExpansion *TE) { 896 return TE->Kind == TEK_Complex; 897 } 898 }; 899 900 struct NoExpansion : TypeExpansion { 901 NoExpansion() : TypeExpansion(TEK_None) {} 902 static bool classof(const TypeExpansion *TE) { 903 return TE->Kind == TEK_None; 904 } 905 }; 906 } // namespace 907 908 static std::unique_ptr<TypeExpansion> 909 getTypeExpansion(QualType Ty, const ASTContext &Context) { 910 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) { 911 return std::make_unique<ConstantArrayExpansion>( 912 AT->getElementType(), AT->getSize().getZExtValue()); 913 } 914 if (const RecordType *RT = Ty->getAs<RecordType>()) { 915 SmallVector<const CXXBaseSpecifier *, 1> Bases; 916 SmallVector<const FieldDecl *, 1> Fields; 917 const RecordDecl *RD = RT->getDecl(); 918 assert(!RD->hasFlexibleArrayMember() && 919 "Cannot expand structure with flexible array."); 920 if (RD->isUnion()) { 921 // Unions can be here only in degenerative cases - all the fields are same 922 // after flattening. Thus we have to use the "largest" field. 923 const FieldDecl *LargestFD = nullptr; 924 CharUnits UnionSize = CharUnits::Zero(); 925 926 for (const auto *FD : RD->fields()) { 927 if (FD->isZeroLengthBitField(Context)) 928 continue; 929 assert(!FD->isBitField() && 930 "Cannot expand structure with bit-field members."); 931 CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType()); 932 if (UnionSize < FieldSize) { 933 UnionSize = FieldSize; 934 LargestFD = FD; 935 } 936 } 937 if (LargestFD) 938 Fields.push_back(LargestFD); 939 } else { 940 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) { 941 assert(!CXXRD->isDynamicClass() && 942 "cannot expand vtable pointers in dynamic classes"); 943 for (const CXXBaseSpecifier &BS : CXXRD->bases()) 944 Bases.push_back(&BS); 945 } 946 947 for (const auto *FD : RD->fields()) { 948 if (FD->isZeroLengthBitField(Context)) 949 continue; 950 assert(!FD->isBitField() && 951 "Cannot expand structure with bit-field members."); 952 Fields.push_back(FD); 953 } 954 } 955 return std::make_unique<RecordExpansion>(std::move(Bases), 956 std::move(Fields)); 957 } 958 if (const ComplexType *CT = Ty->getAs<ComplexType>()) { 959 return std::make_unique<ComplexExpansion>(CT->getElementType()); 960 } 961 return std::make_unique<NoExpansion>(); 962 } 963 964 static int getExpansionSize(QualType Ty, const ASTContext &Context) { 965 auto Exp = getTypeExpansion(Ty, Context); 966 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { 967 return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context); 968 } 969 if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { 970 int Res = 0; 971 for (auto BS : RExp->Bases) 972 Res += getExpansionSize(BS->getType(), Context); 973 for (auto FD : RExp->Fields) 974 Res += getExpansionSize(FD->getType(), Context); 975 return Res; 976 } 977 if (isa<ComplexExpansion>(Exp.get())) 978 return 2; 979 assert(isa<NoExpansion>(Exp.get())); 980 return 1; 981 } 982 983 void 984 CodeGenTypes::getExpandedTypes(QualType Ty, 985 SmallVectorImpl<llvm::Type *>::iterator &TI) { 986 auto Exp = getTypeExpansion(Ty, Context); 987 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { 988 for (int i = 0, n = CAExp->NumElts; i < n; i++) { 989 getExpandedTypes(CAExp->EltTy, TI); 990 } 991 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { 992 for (auto BS : RExp->Bases) 993 getExpandedTypes(BS->getType(), TI); 994 for (auto FD : RExp->Fields) 995 getExpandedTypes(FD->getType(), TI); 996 } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) { 997 llvm::Type *EltTy = ConvertType(CExp->EltTy); 998 *TI++ = EltTy; 999 *TI++ = EltTy; 1000 } else { 1001 assert(isa<NoExpansion>(Exp.get())); 1002 *TI++ = ConvertType(Ty); 1003 } 1004 } 1005 1006 static void forConstantArrayExpansion(CodeGenFunction &CGF, 1007 ConstantArrayExpansion *CAE, 1008 Address BaseAddr, 1009 llvm::function_ref<void(Address)> Fn) { 1010 CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy); 1011 CharUnits EltAlign = 1012 BaseAddr.getAlignment().alignmentOfArrayElement(EltSize); 1013 1014 for (int i = 0, n = CAE->NumElts; i < n; i++) { 1015 llvm::Value *EltAddr = 1016 CGF.Builder.CreateConstGEP2_32(nullptr, BaseAddr.getPointer(), 0, i); 1017 Fn(Address(EltAddr, EltAlign)); 1018 } 1019 } 1020 1021 void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV, 1022 llvm::Function::arg_iterator &AI) { 1023 assert(LV.isSimple() && 1024 "Unexpected non-simple lvalue during struct expansion."); 1025 1026 auto Exp = getTypeExpansion(Ty, getContext()); 1027 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { 1028 forConstantArrayExpansion( 1029 *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) { 1030 LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy); 1031 ExpandTypeFromArgs(CAExp->EltTy, LV, AI); 1032 }); 1033 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { 1034 Address This = LV.getAddress(*this); 1035 for (const CXXBaseSpecifier *BS : RExp->Bases) { 1036 // Perform a single step derived-to-base conversion. 1037 Address Base = 1038 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1, 1039 /*NullCheckValue=*/false, SourceLocation()); 1040 LValue SubLV = MakeAddrLValue(Base, BS->getType()); 1041 1042 // Recurse onto bases. 1043 ExpandTypeFromArgs(BS->getType(), SubLV, AI); 1044 } 1045 for (auto FD : RExp->Fields) { 1046 // FIXME: What are the right qualifiers here? 1047 LValue SubLV = EmitLValueForFieldInitialization(LV, FD); 1048 ExpandTypeFromArgs(FD->getType(), SubLV, AI); 1049 } 1050 } else if (isa<ComplexExpansion>(Exp.get())) { 1051 auto realValue = &*AI++; 1052 auto imagValue = &*AI++; 1053 EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true); 1054 } else { 1055 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a 1056 // primitive store. 1057 assert(isa<NoExpansion>(Exp.get())); 1058 if (LV.isBitField()) 1059 EmitStoreThroughLValue(RValue::get(&*AI++), LV); 1060 else 1061 EmitStoreOfScalar(&*AI++, LV); 1062 } 1063 } 1064 1065 void CodeGenFunction::ExpandTypeToArgs( 1066 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy, 1067 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) { 1068 auto Exp = getTypeExpansion(Ty, getContext()); 1069 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { 1070 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this) 1071 : Arg.getKnownRValue().getAggregateAddress(); 1072 forConstantArrayExpansion( 1073 *this, CAExp, Addr, [&](Address EltAddr) { 1074 CallArg EltArg = CallArg( 1075 convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()), 1076 CAExp->EltTy); 1077 ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs, 1078 IRCallArgPos); 1079 }); 1080 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { 1081 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this) 1082 : Arg.getKnownRValue().getAggregateAddress(); 1083 for (const CXXBaseSpecifier *BS : RExp->Bases) { 1084 // Perform a single step derived-to-base conversion. 1085 Address Base = 1086 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1, 1087 /*NullCheckValue=*/false, SourceLocation()); 1088 CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType()); 1089 1090 // Recurse onto bases. 1091 ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs, 1092 IRCallArgPos); 1093 } 1094 1095 LValue LV = MakeAddrLValue(This, Ty); 1096 for (auto FD : RExp->Fields) { 1097 CallArg FldArg = 1098 CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType()); 1099 ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs, 1100 IRCallArgPos); 1101 } 1102 } else if (isa<ComplexExpansion>(Exp.get())) { 1103 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal(); 1104 IRCallArgs[IRCallArgPos++] = CV.first; 1105 IRCallArgs[IRCallArgPos++] = CV.second; 1106 } else { 1107 assert(isa<NoExpansion>(Exp.get())); 1108 auto RV = Arg.getKnownRValue(); 1109 assert(RV.isScalar() && 1110 "Unexpected non-scalar rvalue during struct expansion."); 1111 1112 // Insert a bitcast as needed. 1113 llvm::Value *V = RV.getScalarVal(); 1114 if (IRCallArgPos < IRFuncTy->getNumParams() && 1115 V->getType() != IRFuncTy->getParamType(IRCallArgPos)) 1116 V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos)); 1117 1118 IRCallArgs[IRCallArgPos++] = V; 1119 } 1120 } 1121 1122 /// Create a temporary allocation for the purposes of coercion. 1123 static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty, 1124 CharUnits MinAlign, 1125 const Twine &Name = "tmp") { 1126 // Don't use an alignment that's worse than what LLVM would prefer. 1127 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty); 1128 CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign)); 1129 1130 return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce"); 1131 } 1132 1133 /// EnterStructPointerForCoercedAccess - Given a struct pointer that we are 1134 /// accessing some number of bytes out of it, try to gep into the struct to get 1135 /// at its inner goodness. Dive as deep as possible without entering an element 1136 /// with an in-memory size smaller than DstSize. 1137 static Address 1138 EnterStructPointerForCoercedAccess(Address SrcPtr, 1139 llvm::StructType *SrcSTy, 1140 uint64_t DstSize, CodeGenFunction &CGF) { 1141 // We can't dive into a zero-element struct. 1142 if (SrcSTy->getNumElements() == 0) return SrcPtr; 1143 1144 llvm::Type *FirstElt = SrcSTy->getElementType(0); 1145 1146 // If the first elt is at least as large as what we're looking for, or if the 1147 // first element is the same size as the whole struct, we can enter it. The 1148 // comparison must be made on the store size and not the alloca size. Using 1149 // the alloca size may overstate the size of the load. 1150 uint64_t FirstEltSize = 1151 CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt); 1152 if (FirstEltSize < DstSize && 1153 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy)) 1154 return SrcPtr; 1155 1156 // GEP into the first element. 1157 SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive"); 1158 1159 // If the first element is a struct, recurse. 1160 llvm::Type *SrcTy = SrcPtr.getElementType(); 1161 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) 1162 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF); 1163 1164 return SrcPtr; 1165 } 1166 1167 /// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both 1168 /// are either integers or pointers. This does a truncation of the value if it 1169 /// is too large or a zero extension if it is too small. 1170 /// 1171 /// This behaves as if the value were coerced through memory, so on big-endian 1172 /// targets the high bits are preserved in a truncation, while little-endian 1173 /// targets preserve the low bits. 1174 static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, 1175 llvm::Type *Ty, 1176 CodeGenFunction &CGF) { 1177 if (Val->getType() == Ty) 1178 return Val; 1179 1180 if (isa<llvm::PointerType>(Val->getType())) { 1181 // If this is Pointer->Pointer avoid conversion to and from int. 1182 if (isa<llvm::PointerType>(Ty)) 1183 return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val"); 1184 1185 // Convert the pointer to an integer so we can play with its width. 1186 Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi"); 1187 } 1188 1189 llvm::Type *DestIntTy = Ty; 1190 if (isa<llvm::PointerType>(DestIntTy)) 1191 DestIntTy = CGF.IntPtrTy; 1192 1193 if (Val->getType() != DestIntTy) { 1194 const llvm::DataLayout &DL = CGF.CGM.getDataLayout(); 1195 if (DL.isBigEndian()) { 1196 // Preserve the high bits on big-endian targets. 1197 // That is what memory coercion does. 1198 uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType()); 1199 uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy); 1200 1201 if (SrcSize > DstSize) { 1202 Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits"); 1203 Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii"); 1204 } else { 1205 Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii"); 1206 Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits"); 1207 } 1208 } else { 1209 // Little-endian targets preserve the low bits. No shifts required. 1210 Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii"); 1211 } 1212 } 1213 1214 if (isa<llvm::PointerType>(Ty)) 1215 Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip"); 1216 return Val; 1217 } 1218 1219 1220 1221 /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as 1222 /// a pointer to an object of type \arg Ty, known to be aligned to 1223 /// \arg SrcAlign bytes. 1224 /// 1225 /// This safely handles the case when the src type is smaller than the 1226 /// destination type; in this situation the values of bits which not 1227 /// present in the src are undefined. 1228 static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, 1229 CodeGenFunction &CGF) { 1230 llvm::Type *SrcTy = Src.getElementType(); 1231 1232 // If SrcTy and Ty are the same, just do a load. 1233 if (SrcTy == Ty) 1234 return CGF.Builder.CreateLoad(Src); 1235 1236 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty); 1237 1238 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) { 1239 Src = EnterStructPointerForCoercedAccess(Src, SrcSTy, 1240 DstSize.getFixedSize(), CGF); 1241 SrcTy = Src.getElementType(); 1242 } 1243 1244 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy); 1245 1246 // If the source and destination are integer or pointer types, just do an 1247 // extension or truncation to the desired type. 1248 if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) && 1249 (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) { 1250 llvm::Value *Load = CGF.Builder.CreateLoad(Src); 1251 return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF); 1252 } 1253 1254 // If load is legal, just bitcast the src pointer. 1255 if (!SrcSize.isScalable() && !DstSize.isScalable() && 1256 SrcSize.getFixedSize() >= DstSize.getFixedSize()) { 1257 // Generally SrcSize is never greater than DstSize, since this means we are 1258 // losing bits. However, this can happen in cases where the structure has 1259 // additional padding, for example due to a user specified alignment. 1260 // 1261 // FIXME: Assert that we aren't truncating non-padding bits when have access 1262 // to that information. 1263 Src = CGF.Builder.CreateBitCast(Src, 1264 Ty->getPointerTo(Src.getAddressSpace())); 1265 return CGF.Builder.CreateLoad(Src); 1266 } 1267 1268 // If coercing a fixed vector to a scalable vector for ABI compatibility, and 1269 // the types match, use the llvm.experimental.vector.insert intrinsic to 1270 // perform the conversion. 1271 if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) { 1272 if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) { 1273 if (ScalableDst->getElementType() == FixedSrc->getElementType()) { 1274 auto *Load = CGF.Builder.CreateLoad(Src); 1275 auto *UndefVec = llvm::UndefValue::get(ScalableDst); 1276 auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty); 1277 return CGF.Builder.CreateInsertVector(ScalableDst, UndefVec, Load, Zero, 1278 "castScalableSve"); 1279 } 1280 } 1281 } 1282 1283 // Otherwise do coercion through memory. This is stupid, but simple. 1284 Address Tmp = 1285 CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName()); 1286 CGF.Builder.CreateMemCpy( 1287 Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(), 1288 Src.getAlignment().getAsAlign(), 1289 llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize())); 1290 return CGF.Builder.CreateLoad(Tmp); 1291 } 1292 1293 // Function to store a first-class aggregate into memory. We prefer to 1294 // store the elements rather than the aggregate to be more friendly to 1295 // fast-isel. 1296 // FIXME: Do we need to recurse here? 1297 void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest, 1298 bool DestIsVolatile) { 1299 // Prefer scalar stores to first-class aggregate stores. 1300 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) { 1301 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 1302 Address EltPtr = Builder.CreateStructGEP(Dest, i); 1303 llvm::Value *Elt = Builder.CreateExtractValue(Val, i); 1304 Builder.CreateStore(Elt, EltPtr, DestIsVolatile); 1305 } 1306 } else { 1307 Builder.CreateStore(Val, Dest, DestIsVolatile); 1308 } 1309 } 1310 1311 /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src, 1312 /// where the source and destination may have different types. The 1313 /// destination is known to be aligned to \arg DstAlign bytes. 1314 /// 1315 /// This safely handles the case when the src type is larger than the 1316 /// destination type; the upper bits of the src will be lost. 1317 static void CreateCoercedStore(llvm::Value *Src, 1318 Address Dst, 1319 bool DstIsVolatile, 1320 CodeGenFunction &CGF) { 1321 llvm::Type *SrcTy = Src->getType(); 1322 llvm::Type *DstTy = Dst.getElementType(); 1323 if (SrcTy == DstTy) { 1324 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); 1325 return; 1326 } 1327 1328 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy); 1329 1330 if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) { 1331 Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, 1332 SrcSize.getFixedSize(), CGF); 1333 DstTy = Dst.getElementType(); 1334 } 1335 1336 llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy); 1337 llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy); 1338 if (SrcPtrTy && DstPtrTy && 1339 SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) { 1340 Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy); 1341 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); 1342 return; 1343 } 1344 1345 // If the source and destination are integer or pointer types, just do an 1346 // extension or truncation to the desired type. 1347 if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) && 1348 (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) { 1349 Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF); 1350 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); 1351 return; 1352 } 1353 1354 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy); 1355 1356 // If store is legal, just bitcast the src pointer. 1357 if (isa<llvm::ScalableVectorType>(SrcTy) || 1358 isa<llvm::ScalableVectorType>(DstTy) || 1359 SrcSize.getFixedSize() <= DstSize.getFixedSize()) { 1360 Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy); 1361 CGF.EmitAggregateStore(Src, Dst, DstIsVolatile); 1362 } else { 1363 // Otherwise do coercion through memory. This is stupid, but 1364 // simple. 1365 1366 // Generally SrcSize is never greater than DstSize, since this means we are 1367 // losing bits. However, this can happen in cases where the structure has 1368 // additional padding, for example due to a user specified alignment. 1369 // 1370 // FIXME: Assert that we aren't truncating non-padding bits when have access 1371 // to that information. 1372 Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); 1373 CGF.Builder.CreateStore(Src, Tmp); 1374 CGF.Builder.CreateMemCpy( 1375 Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(), 1376 Tmp.getAlignment().getAsAlign(), 1377 llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize())); 1378 } 1379 } 1380 1381 static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr, 1382 const ABIArgInfo &info) { 1383 if (unsigned offset = info.getDirectOffset()) { 1384 addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty); 1385 addr = CGF.Builder.CreateConstInBoundsByteGEP(addr, 1386 CharUnits::fromQuantity(offset)); 1387 addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType()); 1388 } 1389 return addr; 1390 } 1391 1392 namespace { 1393 1394 /// Encapsulates information about the way function arguments from 1395 /// CGFunctionInfo should be passed to actual LLVM IR function. 1396 class ClangToLLVMArgMapping { 1397 static const unsigned InvalidIndex = ~0U; 1398 unsigned InallocaArgNo; 1399 unsigned SRetArgNo; 1400 unsigned TotalIRArgs; 1401 1402 /// Arguments of LLVM IR function corresponding to single Clang argument. 1403 struct IRArgs { 1404 unsigned PaddingArgIndex; 1405 // Argument is expanded to IR arguments at positions 1406 // [FirstArgIndex, FirstArgIndex + NumberOfArgs). 1407 unsigned FirstArgIndex; 1408 unsigned NumberOfArgs; 1409 1410 IRArgs() 1411 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex), 1412 NumberOfArgs(0) {} 1413 }; 1414 1415 SmallVector<IRArgs, 8> ArgInfo; 1416 1417 public: 1418 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI, 1419 bool OnlyRequiredArgs = false) 1420 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0), 1421 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) { 1422 construct(Context, FI, OnlyRequiredArgs); 1423 } 1424 1425 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; } 1426 unsigned getInallocaArgNo() const { 1427 assert(hasInallocaArg()); 1428 return InallocaArgNo; 1429 } 1430 1431 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; } 1432 unsigned getSRetArgNo() const { 1433 assert(hasSRetArg()); 1434 return SRetArgNo; 1435 } 1436 1437 unsigned totalIRArgs() const { return TotalIRArgs; } 1438 1439 bool hasPaddingArg(unsigned ArgNo) const { 1440 assert(ArgNo < ArgInfo.size()); 1441 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex; 1442 } 1443 unsigned getPaddingArgNo(unsigned ArgNo) const { 1444 assert(hasPaddingArg(ArgNo)); 1445 return ArgInfo[ArgNo].PaddingArgIndex; 1446 } 1447 1448 /// Returns index of first IR argument corresponding to ArgNo, and their 1449 /// quantity. 1450 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const { 1451 assert(ArgNo < ArgInfo.size()); 1452 return std::make_pair(ArgInfo[ArgNo].FirstArgIndex, 1453 ArgInfo[ArgNo].NumberOfArgs); 1454 } 1455 1456 private: 1457 void construct(const ASTContext &Context, const CGFunctionInfo &FI, 1458 bool OnlyRequiredArgs); 1459 }; 1460 1461 void ClangToLLVMArgMapping::construct(const ASTContext &Context, 1462 const CGFunctionInfo &FI, 1463 bool OnlyRequiredArgs) { 1464 unsigned IRArgNo = 0; 1465 bool SwapThisWithSRet = false; 1466 const ABIArgInfo &RetAI = FI.getReturnInfo(); 1467 1468 if (RetAI.getKind() == ABIArgInfo::Indirect) { 1469 SwapThisWithSRet = RetAI.isSRetAfterThis(); 1470 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++; 1471 } 1472 1473 unsigned ArgNo = 0; 1474 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size(); 1475 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs; 1476 ++I, ++ArgNo) { 1477 assert(I != FI.arg_end()); 1478 QualType ArgType = I->type; 1479 const ABIArgInfo &AI = I->info; 1480 // Collect data about IR arguments corresponding to Clang argument ArgNo. 1481 auto &IRArgs = ArgInfo[ArgNo]; 1482 1483 if (AI.getPaddingType()) 1484 IRArgs.PaddingArgIndex = IRArgNo++; 1485 1486 switch (AI.getKind()) { 1487 case ABIArgInfo::Extend: 1488 case ABIArgInfo::Direct: { 1489 // FIXME: handle sseregparm someday... 1490 llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType()); 1491 if (AI.isDirect() && AI.getCanBeFlattened() && STy) { 1492 IRArgs.NumberOfArgs = STy->getNumElements(); 1493 } else { 1494 IRArgs.NumberOfArgs = 1; 1495 } 1496 break; 1497 } 1498 case ABIArgInfo::Indirect: 1499 case ABIArgInfo::IndirectAliased: 1500 IRArgs.NumberOfArgs = 1; 1501 break; 1502 case ABIArgInfo::Ignore: 1503 case ABIArgInfo::InAlloca: 1504 // ignore and inalloca doesn't have matching LLVM parameters. 1505 IRArgs.NumberOfArgs = 0; 1506 break; 1507 case ABIArgInfo::CoerceAndExpand: 1508 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size(); 1509 break; 1510 case ABIArgInfo::Expand: 1511 IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context); 1512 break; 1513 } 1514 1515 if (IRArgs.NumberOfArgs > 0) { 1516 IRArgs.FirstArgIndex = IRArgNo; 1517 IRArgNo += IRArgs.NumberOfArgs; 1518 } 1519 1520 // Skip over the sret parameter when it comes second. We already handled it 1521 // above. 1522 if (IRArgNo == 1 && SwapThisWithSRet) 1523 IRArgNo++; 1524 } 1525 assert(ArgNo == ArgInfo.size()); 1526 1527 if (FI.usesInAlloca()) 1528 InallocaArgNo = IRArgNo++; 1529 1530 TotalIRArgs = IRArgNo; 1531 } 1532 } // namespace 1533 1534 /***/ 1535 1536 bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) { 1537 const auto &RI = FI.getReturnInfo(); 1538 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet()); 1539 } 1540 1541 bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) { 1542 return ReturnTypeUsesSRet(FI) && 1543 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs(); 1544 } 1545 1546 bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) { 1547 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) { 1548 switch (BT->getKind()) { 1549 default: 1550 return false; 1551 case BuiltinType::Float: 1552 return getTarget().useObjCFPRetForRealType(TargetInfo::Float); 1553 case BuiltinType::Double: 1554 return getTarget().useObjCFPRetForRealType(TargetInfo::Double); 1555 case BuiltinType::LongDouble: 1556 return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble); 1557 } 1558 } 1559 1560 return false; 1561 } 1562 1563 bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) { 1564 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) { 1565 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) { 1566 if (BT->getKind() == BuiltinType::LongDouble) 1567 return getTarget().useObjCFP2RetForComplexLongDouble(); 1568 } 1569 } 1570 1571 return false; 1572 } 1573 1574 llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) { 1575 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD); 1576 return GetFunctionType(FI); 1577 } 1578 1579 llvm::FunctionType * 1580 CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) { 1581 1582 bool Inserted = FunctionsBeingProcessed.insert(&FI).second; 1583 (void)Inserted; 1584 assert(Inserted && "Recursively being processed?"); 1585 1586 llvm::Type *resultType = nullptr; 1587 const ABIArgInfo &retAI = FI.getReturnInfo(); 1588 switch (retAI.getKind()) { 1589 case ABIArgInfo::Expand: 1590 case ABIArgInfo::IndirectAliased: 1591 llvm_unreachable("Invalid ABI kind for return argument"); 1592 1593 case ABIArgInfo::Extend: 1594 case ABIArgInfo::Direct: 1595 resultType = retAI.getCoerceToType(); 1596 break; 1597 1598 case ABIArgInfo::InAlloca: 1599 if (retAI.getInAllocaSRet()) { 1600 // sret things on win32 aren't void, they return the sret pointer. 1601 QualType ret = FI.getReturnType(); 1602 llvm::Type *ty = ConvertType(ret); 1603 unsigned addressSpace = Context.getTargetAddressSpace(ret); 1604 resultType = llvm::PointerType::get(ty, addressSpace); 1605 } else { 1606 resultType = llvm::Type::getVoidTy(getLLVMContext()); 1607 } 1608 break; 1609 1610 case ABIArgInfo::Indirect: 1611 case ABIArgInfo::Ignore: 1612 resultType = llvm::Type::getVoidTy(getLLVMContext()); 1613 break; 1614 1615 case ABIArgInfo::CoerceAndExpand: 1616 resultType = retAI.getUnpaddedCoerceAndExpandType(); 1617 break; 1618 } 1619 1620 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true); 1621 SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs()); 1622 1623 // Add type for sret argument. 1624 if (IRFunctionArgs.hasSRetArg()) { 1625 QualType Ret = FI.getReturnType(); 1626 llvm::Type *Ty = ConvertType(Ret); 1627 unsigned AddressSpace = Context.getTargetAddressSpace(Ret); 1628 ArgTypes[IRFunctionArgs.getSRetArgNo()] = 1629 llvm::PointerType::get(Ty, AddressSpace); 1630 } 1631 1632 // Add type for inalloca argument. 1633 if (IRFunctionArgs.hasInallocaArg()) { 1634 auto ArgStruct = FI.getArgStruct(); 1635 assert(ArgStruct); 1636 ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo(); 1637 } 1638 1639 // Add in all of the required arguments. 1640 unsigned ArgNo = 0; 1641 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(), 1642 ie = it + FI.getNumRequiredArgs(); 1643 for (; it != ie; ++it, ++ArgNo) { 1644 const ABIArgInfo &ArgInfo = it->info; 1645 1646 // Insert a padding type to ensure proper alignment. 1647 if (IRFunctionArgs.hasPaddingArg(ArgNo)) 1648 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] = 1649 ArgInfo.getPaddingType(); 1650 1651 unsigned FirstIRArg, NumIRArgs; 1652 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo); 1653 1654 switch (ArgInfo.getKind()) { 1655 case ABIArgInfo::Ignore: 1656 case ABIArgInfo::InAlloca: 1657 assert(NumIRArgs == 0); 1658 break; 1659 1660 case ABIArgInfo::Indirect: { 1661 assert(NumIRArgs == 1); 1662 // indirect arguments are always on the stack, which is alloca addr space. 1663 llvm::Type *LTy = ConvertTypeForMem(it->type); 1664 ArgTypes[FirstIRArg] = LTy->getPointerTo( 1665 CGM.getDataLayout().getAllocaAddrSpace()); 1666 break; 1667 } 1668 case ABIArgInfo::IndirectAliased: { 1669 assert(NumIRArgs == 1); 1670 llvm::Type *LTy = ConvertTypeForMem(it->type); 1671 ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace()); 1672 break; 1673 } 1674 case ABIArgInfo::Extend: 1675 case ABIArgInfo::Direct: { 1676 // Fast-isel and the optimizer generally like scalar values better than 1677 // FCAs, so we flatten them if this is safe to do for this argument. 1678 llvm::Type *argType = ArgInfo.getCoerceToType(); 1679 llvm::StructType *st = dyn_cast<llvm::StructType>(argType); 1680 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) { 1681 assert(NumIRArgs == st->getNumElements()); 1682 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i) 1683 ArgTypes[FirstIRArg + i] = st->getElementType(i); 1684 } else { 1685 assert(NumIRArgs == 1); 1686 ArgTypes[FirstIRArg] = argType; 1687 } 1688 break; 1689 } 1690 1691 case ABIArgInfo::CoerceAndExpand: { 1692 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg; 1693 for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) { 1694 *ArgTypesIter++ = EltTy; 1695 } 1696 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs); 1697 break; 1698 } 1699 1700 case ABIArgInfo::Expand: 1701 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg; 1702 getExpandedTypes(it->type, ArgTypesIter); 1703 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs); 1704 break; 1705 } 1706 } 1707 1708 bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased; 1709 assert(Erased && "Not in set?"); 1710 1711 return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic()); 1712 } 1713 1714 llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) { 1715 const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); 1716 const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>(); 1717 1718 if (!isFuncTypeConvertible(FPT)) 1719 return llvm::StructType::get(getLLVMContext()); 1720 1721 return GetFunctionType(GD); 1722 } 1723 1724 static void AddAttributesFromFunctionProtoType(ASTContext &Ctx, 1725 llvm::AttrBuilder &FuncAttrs, 1726 const FunctionProtoType *FPT) { 1727 if (!FPT) 1728 return; 1729 1730 if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) && 1731 FPT->isNothrow()) 1732 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); 1733 } 1734 1735 void CodeGenModule::getDefaultFunctionAttributes(StringRef Name, 1736 bool HasOptnone, 1737 bool AttrOnCallSite, 1738 llvm::AttrBuilder &FuncAttrs) { 1739 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed. 1740 if (!HasOptnone) { 1741 if (CodeGenOpts.OptimizeSize) 1742 FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize); 1743 if (CodeGenOpts.OptimizeSize == 2) 1744 FuncAttrs.addAttribute(llvm::Attribute::MinSize); 1745 } 1746 1747 if (CodeGenOpts.DisableRedZone) 1748 FuncAttrs.addAttribute(llvm::Attribute::NoRedZone); 1749 if (CodeGenOpts.IndirectTlsSegRefs) 1750 FuncAttrs.addAttribute("indirect-tls-seg-refs"); 1751 if (CodeGenOpts.NoImplicitFloat) 1752 FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat); 1753 1754 if (AttrOnCallSite) { 1755 // Attributes that should go on the call site only. 1756 if (!CodeGenOpts.SimplifyLibCalls || 1757 CodeGenOpts.isNoBuiltinFunc(Name.data())) 1758 FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin); 1759 if (!CodeGenOpts.TrapFuncName.empty()) 1760 FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName); 1761 } else { 1762 StringRef FpKind; 1763 switch (CodeGenOpts.getFramePointer()) { 1764 case CodeGenOptions::FramePointerKind::None: 1765 FpKind = "none"; 1766 break; 1767 case CodeGenOptions::FramePointerKind::NonLeaf: 1768 FpKind = "non-leaf"; 1769 break; 1770 case CodeGenOptions::FramePointerKind::All: 1771 FpKind = "all"; 1772 break; 1773 } 1774 FuncAttrs.addAttribute("frame-pointer", FpKind); 1775 1776 FuncAttrs.addAttribute("less-precise-fpmad", 1777 llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD)); 1778 1779 if (CodeGenOpts.NullPointerIsValid) 1780 FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid); 1781 1782 if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE()) 1783 FuncAttrs.addAttribute("denormal-fp-math", 1784 CodeGenOpts.FPDenormalMode.str()); 1785 if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) { 1786 FuncAttrs.addAttribute( 1787 "denormal-fp-math-f32", 1788 CodeGenOpts.FP32DenormalMode.str()); 1789 } 1790 1791 FuncAttrs.addAttribute("no-trapping-math", 1792 llvm::toStringRef(LangOpts.getFPExceptionMode() == 1793 LangOptions::FPE_Ignore)); 1794 1795 // Strict (compliant) code is the default, so only add this attribute to 1796 // indicate that we are trying to workaround a problem case. 1797 if (!CodeGenOpts.StrictFloatCastOverflow) 1798 FuncAttrs.addAttribute("strict-float-cast-overflow", "false"); 1799 1800 // TODO: Are these all needed? 1801 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags. 1802 FuncAttrs.addAttribute("no-infs-fp-math", 1803 llvm::toStringRef(LangOpts.NoHonorInfs)); 1804 FuncAttrs.addAttribute("no-nans-fp-math", 1805 llvm::toStringRef(LangOpts.NoHonorNaNs)); 1806 FuncAttrs.addAttribute("unsafe-fp-math", 1807 llvm::toStringRef(LangOpts.UnsafeFPMath)); 1808 FuncAttrs.addAttribute("use-soft-float", 1809 llvm::toStringRef(CodeGenOpts.SoftFloat)); 1810 FuncAttrs.addAttribute("stack-protector-buffer-size", 1811 llvm::utostr(CodeGenOpts.SSPBufferSize)); 1812 FuncAttrs.addAttribute("no-signed-zeros-fp-math", 1813 llvm::toStringRef(LangOpts.NoSignedZero)); 1814 1815 // TODO: Reciprocal estimate codegen options should apply to instructions? 1816 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals; 1817 if (!Recips.empty()) 1818 FuncAttrs.addAttribute("reciprocal-estimates", 1819 llvm::join(Recips, ",")); 1820 1821 if (!CodeGenOpts.PreferVectorWidth.empty() && 1822 CodeGenOpts.PreferVectorWidth != "none") 1823 FuncAttrs.addAttribute("prefer-vector-width", 1824 CodeGenOpts.PreferVectorWidth); 1825 1826 if (CodeGenOpts.StackRealignment) 1827 FuncAttrs.addAttribute("stackrealign"); 1828 if (CodeGenOpts.Backchain) 1829 FuncAttrs.addAttribute("backchain"); 1830 if (CodeGenOpts.EnableSegmentedStacks) 1831 FuncAttrs.addAttribute("split-stack"); 1832 1833 if (CodeGenOpts.SpeculativeLoadHardening) 1834 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening); 1835 } 1836 1837 if (getLangOpts().assumeFunctionsAreConvergent()) { 1838 // Conservatively, mark all functions and calls in CUDA and OpenCL as 1839 // convergent (meaning, they may call an intrinsically convergent op, such 1840 // as __syncthreads() / barrier(), and so can't have certain optimizations 1841 // applied around them). LLVM will remove this attribute where it safely 1842 // can. 1843 FuncAttrs.addAttribute(llvm::Attribute::Convergent); 1844 } 1845 1846 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) { 1847 // Exceptions aren't supported in CUDA device code. 1848 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); 1849 } 1850 1851 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) { 1852 StringRef Var, Value; 1853 std::tie(Var, Value) = Attr.split('='); 1854 FuncAttrs.addAttribute(Var, Value); 1855 } 1856 } 1857 1858 void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) { 1859 llvm::AttrBuilder FuncAttrs; 1860 getDefaultFunctionAttributes(F.getName(), F.hasOptNone(), 1861 /* AttrOnCallSite = */ false, FuncAttrs); 1862 // TODO: call GetCPUAndFeaturesAttributes? 1863 F.addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs); 1864 } 1865 1866 void CodeGenModule::addDefaultFunctionDefinitionAttributes( 1867 llvm::AttrBuilder &attrs) { 1868 getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false, 1869 /*for call*/ false, attrs); 1870 GetCPUAndFeaturesAttributes(GlobalDecl(), attrs); 1871 } 1872 1873 static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs, 1874 const LangOptions &LangOpts, 1875 const NoBuiltinAttr *NBA = nullptr) { 1876 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) { 1877 SmallString<32> AttributeName; 1878 AttributeName += "no-builtin-"; 1879 AttributeName += BuiltinName; 1880 FuncAttrs.addAttribute(AttributeName); 1881 }; 1882 1883 // First, handle the language options passed through -fno-builtin. 1884 if (LangOpts.NoBuiltin) { 1885 // -fno-builtin disables them all. 1886 FuncAttrs.addAttribute("no-builtins"); 1887 return; 1888 } 1889 1890 // Then, add attributes for builtins specified through -fno-builtin-<name>. 1891 llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr); 1892 1893 // Now, let's check the __attribute__((no_builtin("...")) attribute added to 1894 // the source. 1895 if (!NBA) 1896 return; 1897 1898 // If there is a wildcard in the builtin names specified through the 1899 // attribute, disable them all. 1900 if (llvm::is_contained(NBA->builtinNames(), "*")) { 1901 FuncAttrs.addAttribute("no-builtins"); 1902 return; 1903 } 1904 1905 // And last, add the rest of the builtin names. 1906 llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr); 1907 } 1908 1909 /// Construct the IR attribute list of a function or call. 1910 /// 1911 /// When adding an attribute, please consider where it should be handled: 1912 /// 1913 /// - getDefaultFunctionAttributes is for attributes that are essentially 1914 /// part of the global target configuration (but perhaps can be 1915 /// overridden on a per-function basis). Adding attributes there 1916 /// will cause them to also be set in frontends that build on Clang's 1917 /// target-configuration logic, as well as for code defined in library 1918 /// modules such as CUDA's libdevice. 1919 /// 1920 /// - ConstructAttributeList builds on top of getDefaultFunctionAttributes 1921 /// and adds declaration-specific, convention-specific, and 1922 /// frontend-specific logic. The last is of particular importance: 1923 /// attributes that restrict how the frontend generates code must be 1924 /// added here rather than getDefaultFunctionAttributes. 1925 /// 1926 void CodeGenModule::ConstructAttributeList( 1927 StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo, 1928 llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) { 1929 llvm::AttrBuilder FuncAttrs; 1930 llvm::AttrBuilder RetAttrs; 1931 1932 // Collect function IR attributes from the CC lowering. 1933 // We'll collect the paramete and result attributes later. 1934 CallingConv = FI.getEffectiveCallingConvention(); 1935 if (FI.isNoReturn()) 1936 FuncAttrs.addAttribute(llvm::Attribute::NoReturn); 1937 if (FI.isCmseNSCall()) 1938 FuncAttrs.addAttribute("cmse_nonsecure_call"); 1939 1940 // Collect function IR attributes from the callee prototype if we have one. 1941 AddAttributesFromFunctionProtoType(getContext(), FuncAttrs, 1942 CalleeInfo.getCalleeFunctionProtoType()); 1943 1944 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl(); 1945 1946 bool HasOptnone = false; 1947 // The NoBuiltinAttr attached to the target FunctionDecl. 1948 const NoBuiltinAttr *NBA = nullptr; 1949 1950 // Collect function IR attributes based on declaration-specific 1951 // information. 1952 // FIXME: handle sseregparm someday... 1953 if (TargetDecl) { 1954 if (TargetDecl->hasAttr<ReturnsTwiceAttr>()) 1955 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice); 1956 if (TargetDecl->hasAttr<NoThrowAttr>()) 1957 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); 1958 if (TargetDecl->hasAttr<NoReturnAttr>()) 1959 FuncAttrs.addAttribute(llvm::Attribute::NoReturn); 1960 if (TargetDecl->hasAttr<ColdAttr>()) 1961 FuncAttrs.addAttribute(llvm::Attribute::Cold); 1962 if (TargetDecl->hasAttr<HotAttr>()) 1963 FuncAttrs.addAttribute(llvm::Attribute::Hot); 1964 if (TargetDecl->hasAttr<NoDuplicateAttr>()) 1965 FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate); 1966 if (TargetDecl->hasAttr<ConvergentAttr>()) 1967 FuncAttrs.addAttribute(llvm::Attribute::Convergent); 1968 1969 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) { 1970 AddAttributesFromFunctionProtoType( 1971 getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>()); 1972 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) { 1973 // A sane operator new returns a non-aliasing pointer. 1974 auto Kind = Fn->getDeclName().getCXXOverloadedOperator(); 1975 if (getCodeGenOpts().AssumeSaneOperatorNew && 1976 (Kind == OO_New || Kind == OO_Array_New)) 1977 RetAttrs.addAttribute(llvm::Attribute::NoAlias); 1978 } 1979 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn); 1980 const bool IsVirtualCall = MD && MD->isVirtual(); 1981 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a 1982 // virtual function. These attributes are not inherited by overloads. 1983 if (!(AttrOnCallSite && IsVirtualCall)) { 1984 if (Fn->isNoReturn()) 1985 FuncAttrs.addAttribute(llvm::Attribute::NoReturn); 1986 NBA = Fn->getAttr<NoBuiltinAttr>(); 1987 } 1988 // Only place nomerge attribute on call sites, never functions. This 1989 // allows it to work on indirect virtual function calls. 1990 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>()) 1991 FuncAttrs.addAttribute(llvm::Attribute::NoMerge); 1992 } 1993 1994 // 'const', 'pure' and 'noalias' attributed functions are also nounwind. 1995 if (TargetDecl->hasAttr<ConstAttr>()) { 1996 FuncAttrs.addAttribute(llvm::Attribute::ReadNone); 1997 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); 1998 // gcc specifies that 'const' functions have greater restrictions than 1999 // 'pure' functions, so they also cannot have infinite loops. 2000 FuncAttrs.addAttribute(llvm::Attribute::WillReturn); 2001 } else if (TargetDecl->hasAttr<PureAttr>()) { 2002 FuncAttrs.addAttribute(llvm::Attribute::ReadOnly); 2003 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); 2004 // gcc specifies that 'pure' functions cannot have infinite loops. 2005 FuncAttrs.addAttribute(llvm::Attribute::WillReturn); 2006 } else if (TargetDecl->hasAttr<NoAliasAttr>()) { 2007 FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly); 2008 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); 2009 } 2010 if (TargetDecl->hasAttr<RestrictAttr>()) 2011 RetAttrs.addAttribute(llvm::Attribute::NoAlias); 2012 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() && 2013 !CodeGenOpts.NullPointerIsValid) 2014 RetAttrs.addAttribute(llvm::Attribute::NonNull); 2015 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>()) 2016 FuncAttrs.addAttribute("no_caller_saved_registers"); 2017 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>()) 2018 FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck); 2019 if (TargetDecl->hasAttr<LeafAttr>()) 2020 FuncAttrs.addAttribute(llvm::Attribute::NoCallback); 2021 2022 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>(); 2023 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) { 2024 Optional<unsigned> NumElemsParam; 2025 if (AllocSize->getNumElemsParam().isValid()) 2026 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex(); 2027 FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(), 2028 NumElemsParam); 2029 } 2030 2031 if (TargetDecl->hasAttr<OpenCLKernelAttr>()) { 2032 if (getLangOpts().OpenCLVersion <= 120) { 2033 // OpenCL v1.2 Work groups are always uniform 2034 FuncAttrs.addAttribute("uniform-work-group-size", "true"); 2035 } else { 2036 // OpenCL v2.0 Work groups may be whether uniform or not. 2037 // '-cl-uniform-work-group-size' compile option gets a hint 2038 // to the compiler that the global work-size be a multiple of 2039 // the work-group size specified to clEnqueueNDRangeKernel 2040 // (i.e. work groups are uniform). 2041 FuncAttrs.addAttribute("uniform-work-group-size", 2042 llvm::toStringRef(CodeGenOpts.UniformWGSize)); 2043 } 2044 } 2045 2046 std::string AssumptionValueStr; 2047 for (AssumptionAttr *AssumptionA : 2048 TargetDecl->specific_attrs<AssumptionAttr>()) { 2049 std::string AS = AssumptionA->getAssumption().str(); 2050 if (!AS.empty() && !AssumptionValueStr.empty()) 2051 AssumptionValueStr += ","; 2052 AssumptionValueStr += AS; 2053 } 2054 2055 if (!AssumptionValueStr.empty()) 2056 FuncAttrs.addAttribute(llvm::AssumptionAttrKey, AssumptionValueStr); 2057 } 2058 2059 // Attach "no-builtins" attributes to: 2060 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>". 2061 // * definitions: "no-builtins" or "no-builtin-<name>" only. 2062 // The attributes can come from: 2063 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name> 2064 // * FunctionDecl attributes: __attribute__((no_builtin(...))) 2065 addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA); 2066 2067 // Collect function IR attributes based on global settiings. 2068 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs); 2069 2070 // Override some default IR attributes based on declaration-specific 2071 // information. 2072 if (TargetDecl) { 2073 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>()) 2074 FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening); 2075 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>()) 2076 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening); 2077 if (TargetDecl->hasAttr<NoSplitStackAttr>()) 2078 FuncAttrs.removeAttribute("split-stack"); 2079 2080 // Add NonLazyBind attribute to function declarations when -fno-plt 2081 // is used. 2082 // FIXME: what if we just haven't processed the function definition 2083 // yet, or if it's an external definition like C99 inline? 2084 if (CodeGenOpts.NoPLT) { 2085 if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) { 2086 if (!Fn->isDefined() && !AttrOnCallSite) { 2087 FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind); 2088 } 2089 } 2090 } 2091 } 2092 2093 // Collect non-call-site function IR attributes from declaration-specific 2094 // information. 2095 if (!AttrOnCallSite) { 2096 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>()) 2097 FuncAttrs.addAttribute("cmse_nonsecure_entry"); 2098 2099 // Whether tail calls are enabled. 2100 auto shouldDisableTailCalls = [&] { 2101 // Should this be honored in getDefaultFunctionAttributes? 2102 if (CodeGenOpts.DisableTailCalls) 2103 return true; 2104 2105 if (!TargetDecl) 2106 return false; 2107 2108 if (TargetDecl->hasAttr<DisableTailCallsAttr>() || 2109 TargetDecl->hasAttr<AnyX86InterruptAttr>()) 2110 return true; 2111 2112 if (CodeGenOpts.NoEscapingBlockTailCalls) { 2113 if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl)) 2114 if (!BD->doesNotEscape()) 2115 return true; 2116 } 2117 2118 return false; 2119 }; 2120 FuncAttrs.addAttribute("disable-tail-calls", 2121 llvm::toStringRef(shouldDisableTailCalls())); 2122 2123 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes 2124 // handles these separately to set them based on the global defaults. 2125 GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs); 2126 } 2127 2128 // Collect attributes from arguments and return values. 2129 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI); 2130 2131 QualType RetTy = FI.getReturnType(); 2132 const ABIArgInfo &RetAI = FI.getReturnInfo(); 2133 switch (RetAI.getKind()) { 2134 case ABIArgInfo::Extend: 2135 if (RetAI.isSignExt()) 2136 RetAttrs.addAttribute(llvm::Attribute::SExt); 2137 else 2138 RetAttrs.addAttribute(llvm::Attribute::ZExt); 2139 LLVM_FALLTHROUGH; 2140 case ABIArgInfo::Direct: 2141 if (RetAI.getInReg()) 2142 RetAttrs.addAttribute(llvm::Attribute::InReg); 2143 break; 2144 case ABIArgInfo::Ignore: 2145 break; 2146 2147 case ABIArgInfo::InAlloca: 2148 case ABIArgInfo::Indirect: { 2149 // inalloca and sret disable readnone and readonly 2150 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly) 2151 .removeAttribute(llvm::Attribute::ReadNone); 2152 break; 2153 } 2154 2155 case ABIArgInfo::CoerceAndExpand: 2156 break; 2157 2158 case ABIArgInfo::Expand: 2159 case ABIArgInfo::IndirectAliased: 2160 llvm_unreachable("Invalid ABI kind for return argument"); 2161 } 2162 2163 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) { 2164 QualType PTy = RefTy->getPointeeType(); 2165 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) 2166 RetAttrs.addDereferenceableAttr( 2167 getMinimumObjectSize(PTy).getQuantity()); 2168 if (getContext().getTargetAddressSpace(PTy) == 0 && 2169 !CodeGenOpts.NullPointerIsValid) 2170 RetAttrs.addAttribute(llvm::Attribute::NonNull); 2171 if (PTy->isObjectType()) { 2172 llvm::Align Alignment = 2173 getNaturalPointeeTypeAlignment(RetTy).getAsAlign(); 2174 RetAttrs.addAlignmentAttr(Alignment); 2175 } 2176 } 2177 2178 bool hasUsedSRet = false; 2179 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs()); 2180 2181 // Attach attributes to sret. 2182 if (IRFunctionArgs.hasSRetArg()) { 2183 llvm::AttrBuilder SRETAttrs; 2184 SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy)); 2185 hasUsedSRet = true; 2186 if (RetAI.getInReg()) 2187 SRETAttrs.addAttribute(llvm::Attribute::InReg); 2188 SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity()); 2189 ArgAttrs[IRFunctionArgs.getSRetArgNo()] = 2190 llvm::AttributeSet::get(getLLVMContext(), SRETAttrs); 2191 } 2192 2193 // Attach attributes to inalloca argument. 2194 if (IRFunctionArgs.hasInallocaArg()) { 2195 llvm::AttrBuilder Attrs; 2196 Attrs.addAttribute(llvm::Attribute::InAlloca); 2197 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] = 2198 llvm::AttributeSet::get(getLLVMContext(), Attrs); 2199 } 2200 2201 // Apply `nonnull` and `dereferencable(N)` to the `this` argument. 2202 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() && 2203 !FI.arg_begin()->type->isVoidPointerType()) { 2204 auto IRArgs = IRFunctionArgs.getIRArgs(0); 2205 2206 assert(IRArgs.second == 1 && "Expected only a single `this` pointer."); 2207 2208 llvm::AttrBuilder Attrs; 2209 2210 if (!CodeGenOpts.NullPointerIsValid && 2211 getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0) { 2212 Attrs.addAttribute(llvm::Attribute::NonNull); 2213 Attrs.addDereferenceableAttr( 2214 getMinimumObjectSize( 2215 FI.arg_begin()->type.castAs<PointerType>()->getPointeeType()) 2216 .getQuantity()); 2217 } else { 2218 // FIXME dereferenceable should be correct here, regardless of 2219 // NullPointerIsValid. However, dereferenceable currently does not always 2220 // respect NullPointerIsValid and may imply nonnull and break the program. 2221 // See https://reviews.llvm.org/D66618 for discussions. 2222 Attrs.addDereferenceableOrNullAttr( 2223 getMinimumObjectSize( 2224 FI.arg_begin()->type.castAs<PointerType>()->getPointeeType()) 2225 .getQuantity()); 2226 } 2227 2228 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs); 2229 } 2230 2231 unsigned ArgNo = 0; 2232 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), 2233 E = FI.arg_end(); 2234 I != E; ++I, ++ArgNo) { 2235 QualType ParamType = I->type; 2236 const ABIArgInfo &AI = I->info; 2237 llvm::AttrBuilder Attrs; 2238 2239 // Add attribute for padding argument, if necessary. 2240 if (IRFunctionArgs.hasPaddingArg(ArgNo)) { 2241 if (AI.getPaddingInReg()) { 2242 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] = 2243 llvm::AttributeSet::get( 2244 getLLVMContext(), 2245 llvm::AttrBuilder().addAttribute(llvm::Attribute::InReg)); 2246 } 2247 } 2248 2249 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we 2250 // have the corresponding parameter variable. It doesn't make 2251 // sense to do it here because parameters are so messed up. 2252 switch (AI.getKind()) { 2253 case ABIArgInfo::Extend: 2254 if (AI.isSignExt()) 2255 Attrs.addAttribute(llvm::Attribute::SExt); 2256 else 2257 Attrs.addAttribute(llvm::Attribute::ZExt); 2258 LLVM_FALLTHROUGH; 2259 case ABIArgInfo::Direct: 2260 if (ArgNo == 0 && FI.isChainCall()) 2261 Attrs.addAttribute(llvm::Attribute::Nest); 2262 else if (AI.getInReg()) 2263 Attrs.addAttribute(llvm::Attribute::InReg); 2264 break; 2265 2266 case ABIArgInfo::Indirect: { 2267 if (AI.getInReg()) 2268 Attrs.addAttribute(llvm::Attribute::InReg); 2269 2270 if (AI.getIndirectByVal()) 2271 Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType)); 2272 2273 auto *Decl = ParamType->getAsRecordDecl(); 2274 if (CodeGenOpts.PassByValueIsNoAlias && Decl && 2275 Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs) 2276 // When calling the function, the pointer passed in will be the only 2277 // reference to the underlying object. Mark it accordingly. 2278 Attrs.addAttribute(llvm::Attribute::NoAlias); 2279 2280 // TODO: We could add the byref attribute if not byval, but it would 2281 // require updating many testcases. 2282 2283 CharUnits Align = AI.getIndirectAlign(); 2284 2285 // In a byval argument, it is important that the required 2286 // alignment of the type is honored, as LLVM might be creating a 2287 // *new* stack object, and needs to know what alignment to give 2288 // it. (Sometimes it can deduce a sensible alignment on its own, 2289 // but not if clang decides it must emit a packed struct, or the 2290 // user specifies increased alignment requirements.) 2291 // 2292 // This is different from indirect *not* byval, where the object 2293 // exists already, and the align attribute is purely 2294 // informative. 2295 assert(!Align.isZero()); 2296 2297 // For now, only add this when we have a byval argument. 2298 // TODO: be less lazy about updating test cases. 2299 if (AI.getIndirectByVal()) 2300 Attrs.addAlignmentAttr(Align.getQuantity()); 2301 2302 // byval disables readnone and readonly. 2303 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly) 2304 .removeAttribute(llvm::Attribute::ReadNone); 2305 2306 break; 2307 } 2308 case ABIArgInfo::IndirectAliased: { 2309 CharUnits Align = AI.getIndirectAlign(); 2310 Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType)); 2311 Attrs.addAlignmentAttr(Align.getQuantity()); 2312 break; 2313 } 2314 case ABIArgInfo::Ignore: 2315 case ABIArgInfo::Expand: 2316 case ABIArgInfo::CoerceAndExpand: 2317 break; 2318 2319 case ABIArgInfo::InAlloca: 2320 // inalloca disables readnone and readonly. 2321 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly) 2322 .removeAttribute(llvm::Attribute::ReadNone); 2323 continue; 2324 } 2325 2326 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) { 2327 QualType PTy = RefTy->getPointeeType(); 2328 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) 2329 Attrs.addDereferenceableAttr( 2330 getMinimumObjectSize(PTy).getQuantity()); 2331 if (getContext().getTargetAddressSpace(PTy) == 0 && 2332 !CodeGenOpts.NullPointerIsValid) 2333 Attrs.addAttribute(llvm::Attribute::NonNull); 2334 if (PTy->isObjectType()) { 2335 llvm::Align Alignment = 2336 getNaturalPointeeTypeAlignment(ParamType).getAsAlign(); 2337 Attrs.addAlignmentAttr(Alignment); 2338 } 2339 } 2340 2341 switch (FI.getExtParameterInfo(ArgNo).getABI()) { 2342 case ParameterABI::Ordinary: 2343 break; 2344 2345 case ParameterABI::SwiftIndirectResult: { 2346 // Add 'sret' if we haven't already used it for something, but 2347 // only if the result is void. 2348 if (!hasUsedSRet && RetTy->isVoidType()) { 2349 Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType)); 2350 hasUsedSRet = true; 2351 } 2352 2353 // Add 'noalias' in either case. 2354 Attrs.addAttribute(llvm::Attribute::NoAlias); 2355 2356 // Add 'dereferenceable' and 'alignment'. 2357 auto PTy = ParamType->getPointeeType(); 2358 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) { 2359 auto info = getContext().getTypeInfoInChars(PTy); 2360 Attrs.addDereferenceableAttr(info.Width.getQuantity()); 2361 Attrs.addAlignmentAttr(info.Align.getAsAlign()); 2362 } 2363 break; 2364 } 2365 2366 case ParameterABI::SwiftErrorResult: 2367 Attrs.addAttribute(llvm::Attribute::SwiftError); 2368 break; 2369 2370 case ParameterABI::SwiftContext: 2371 Attrs.addAttribute(llvm::Attribute::SwiftSelf); 2372 break; 2373 } 2374 2375 if (FI.getExtParameterInfo(ArgNo).isNoEscape()) 2376 Attrs.addAttribute(llvm::Attribute::NoCapture); 2377 2378 if (Attrs.hasAttributes()) { 2379 unsigned FirstIRArg, NumIRArgs; 2380 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo); 2381 for (unsigned i = 0; i < NumIRArgs; i++) 2382 ArgAttrs[FirstIRArg + i] = 2383 llvm::AttributeSet::get(getLLVMContext(), Attrs); 2384 } 2385 } 2386 assert(ArgNo == FI.arg_size()); 2387 2388 AttrList = llvm::AttributeList::get( 2389 getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs), 2390 llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs); 2391 } 2392 2393 /// An argument came in as a promoted argument; demote it back to its 2394 /// declared type. 2395 static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF, 2396 const VarDecl *var, 2397 llvm::Value *value) { 2398 llvm::Type *varType = CGF.ConvertType(var->getType()); 2399 2400 // This can happen with promotions that actually don't change the 2401 // underlying type, like the enum promotions. 2402 if (value->getType() == varType) return value; 2403 2404 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) 2405 && "unexpected promotion type"); 2406 2407 if (isa<llvm::IntegerType>(varType)) 2408 return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote"); 2409 2410 return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote"); 2411 } 2412 2413 /// Returns the attribute (either parameter attribute, or function 2414 /// attribute), which declares argument ArgNo to be non-null. 2415 static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD, 2416 QualType ArgType, unsigned ArgNo) { 2417 // FIXME: __attribute__((nonnull)) can also be applied to: 2418 // - references to pointers, where the pointee is known to be 2419 // nonnull (apparently a Clang extension) 2420 // - transparent unions containing pointers 2421 // In the former case, LLVM IR cannot represent the constraint. In 2422 // the latter case, we have no guarantee that the transparent union 2423 // is in fact passed as a pointer. 2424 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType()) 2425 return nullptr; 2426 // First, check attribute on parameter itself. 2427 if (PVD) { 2428 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>()) 2429 return ParmNNAttr; 2430 } 2431 // Check function attributes. 2432 if (!FD) 2433 return nullptr; 2434 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) { 2435 if (NNAttr->isNonNull(ArgNo)) 2436 return NNAttr; 2437 } 2438 return nullptr; 2439 } 2440 2441 namespace { 2442 struct CopyBackSwiftError final : EHScopeStack::Cleanup { 2443 Address Temp; 2444 Address Arg; 2445 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {} 2446 void Emit(CodeGenFunction &CGF, Flags flags) override { 2447 llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp); 2448 CGF.Builder.CreateStore(errorValue, Arg); 2449 } 2450 }; 2451 } 2452 2453 void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, 2454 llvm::Function *Fn, 2455 const FunctionArgList &Args) { 2456 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) 2457 // Naked functions don't have prologues. 2458 return; 2459 2460 // If this is an implicit-return-zero function, go ahead and 2461 // initialize the return value. TODO: it might be nice to have 2462 // a more general mechanism for this that didn't require synthesized 2463 // return statements. 2464 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) { 2465 if (FD->hasImplicitReturnZero()) { 2466 QualType RetTy = FD->getReturnType().getUnqualifiedType(); 2467 llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy); 2468 llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy); 2469 Builder.CreateStore(Zero, ReturnValue); 2470 } 2471 } 2472 2473 // FIXME: We no longer need the types from FunctionArgList; lift up and 2474 // simplify. 2475 2476 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI); 2477 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs()); 2478 2479 // If we're using inalloca, all the memory arguments are GEPs off of the last 2480 // parameter, which is a pointer to the complete memory area. 2481 Address ArgStruct = Address::invalid(); 2482 if (IRFunctionArgs.hasInallocaArg()) { 2483 ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()), 2484 FI.getArgStructAlignment()); 2485 2486 assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo()); 2487 } 2488 2489 // Name the struct return parameter. 2490 if (IRFunctionArgs.hasSRetArg()) { 2491 auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo()); 2492 AI->setName("agg.result"); 2493 AI->addAttr(llvm::Attribute::NoAlias); 2494 } 2495 2496 // Track if we received the parameter as a pointer (indirect, byval, or 2497 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it 2498 // into a local alloca for us. 2499 SmallVector<ParamValue, 16> ArgVals; 2500 ArgVals.reserve(Args.size()); 2501 2502 // Create a pointer value for every parameter declaration. This usually 2503 // entails copying one or more LLVM IR arguments into an alloca. Don't push 2504 // any cleanups or do anything that might unwind. We do that separately, so 2505 // we can push the cleanups in the correct order for the ABI. 2506 assert(FI.arg_size() == Args.size() && 2507 "Mismatch between function signature & arguments."); 2508 unsigned ArgNo = 0; 2509 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin(); 2510 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); 2511 i != e; ++i, ++info_it, ++ArgNo) { 2512 const VarDecl *Arg = *i; 2513 const ABIArgInfo &ArgI = info_it->info; 2514 2515 bool isPromoted = 2516 isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted(); 2517 // We are converting from ABIArgInfo type to VarDecl type directly, unless 2518 // the parameter is promoted. In this case we convert to 2519 // CGFunctionInfo::ArgInfo type with subsequent argument demotion. 2520 QualType Ty = isPromoted ? info_it->type : Arg->getType(); 2521 assert(hasScalarEvaluationKind(Ty) == 2522 hasScalarEvaluationKind(Arg->getType())); 2523 2524 unsigned FirstIRArg, NumIRArgs; 2525 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo); 2526 2527 switch (ArgI.getKind()) { 2528 case ABIArgInfo::InAlloca: { 2529 assert(NumIRArgs == 0); 2530 auto FieldIndex = ArgI.getInAllocaFieldIndex(); 2531 Address V = 2532 Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName()); 2533 if (ArgI.getInAllocaIndirect()) 2534 V = Address(Builder.CreateLoad(V), 2535 getContext().getTypeAlignInChars(Ty)); 2536 ArgVals.push_back(ParamValue::forIndirect(V)); 2537 break; 2538 } 2539 2540 case ABIArgInfo::Indirect: 2541 case ABIArgInfo::IndirectAliased: { 2542 assert(NumIRArgs == 1); 2543 Address ParamAddr = 2544 Address(Fn->getArg(FirstIRArg), ArgI.getIndirectAlign()); 2545 2546 if (!hasScalarEvaluationKind(Ty)) { 2547 // Aggregates and complex variables are accessed by reference. All we 2548 // need to do is realign the value, if requested. Also, if the address 2549 // may be aliased, copy it to ensure that the parameter variable is 2550 // mutable and has a unique adress, as C requires. 2551 Address V = ParamAddr; 2552 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) { 2553 Address AlignedTemp = CreateMemTemp(Ty, "coerce"); 2554 2555 // Copy from the incoming argument pointer to the temporary with the 2556 // appropriate alignment. 2557 // 2558 // FIXME: We should have a common utility for generating an aggregate 2559 // copy. 2560 CharUnits Size = getContext().getTypeSizeInChars(Ty); 2561 Builder.CreateMemCpy( 2562 AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(), 2563 ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(), 2564 llvm::ConstantInt::get(IntPtrTy, Size.getQuantity())); 2565 V = AlignedTemp; 2566 } 2567 ArgVals.push_back(ParamValue::forIndirect(V)); 2568 } else { 2569 // Load scalar value from indirect argument. 2570 llvm::Value *V = 2571 EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc()); 2572 2573 if (isPromoted) 2574 V = emitArgumentDemotion(*this, Arg, V); 2575 ArgVals.push_back(ParamValue::forDirect(V)); 2576 } 2577 break; 2578 } 2579 2580 case ABIArgInfo::Extend: 2581 case ABIArgInfo::Direct: { 2582 auto AI = Fn->getArg(FirstIRArg); 2583 llvm::Type *LTy = ConvertType(Arg->getType()); 2584 2585 // Prepare parameter attributes. So far, only attributes for pointer 2586 // parameters are prepared. See 2587 // http://llvm.org/docs/LangRef.html#paramattrs. 2588 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() && 2589 ArgI.getCoerceToType()->isPointerTy()) { 2590 assert(NumIRArgs == 1); 2591 2592 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) { 2593 // Set `nonnull` attribute if any. 2594 if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(), 2595 PVD->getFunctionScopeIndex()) && 2596 !CGM.getCodeGenOpts().NullPointerIsValid) 2597 AI->addAttr(llvm::Attribute::NonNull); 2598 2599 QualType OTy = PVD->getOriginalType(); 2600 if (const auto *ArrTy = 2601 getContext().getAsConstantArrayType(OTy)) { 2602 // A C99 array parameter declaration with the static keyword also 2603 // indicates dereferenceability, and if the size is constant we can 2604 // use the dereferenceable attribute (which requires the size in 2605 // bytes). 2606 if (ArrTy->getSizeModifier() == ArrayType::Static) { 2607 QualType ETy = ArrTy->getElementType(); 2608 llvm::Align Alignment = 2609 CGM.getNaturalTypeAlignment(ETy).getAsAlign(); 2610 AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment)); 2611 uint64_t ArrSize = ArrTy->getSize().getZExtValue(); 2612 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() && 2613 ArrSize) { 2614 llvm::AttrBuilder Attrs; 2615 Attrs.addDereferenceableAttr( 2616 getContext().getTypeSizeInChars(ETy).getQuantity() * 2617 ArrSize); 2618 AI->addAttrs(Attrs); 2619 } else if (getContext().getTargetInfo().getNullPointerValue( 2620 ETy.getAddressSpace()) == 0 && 2621 !CGM.getCodeGenOpts().NullPointerIsValid) { 2622 AI->addAttr(llvm::Attribute::NonNull); 2623 } 2624 } 2625 } else if (const auto *ArrTy = 2626 getContext().getAsVariableArrayType(OTy)) { 2627 // For C99 VLAs with the static keyword, we don't know the size so 2628 // we can't use the dereferenceable attribute, but in addrspace(0) 2629 // we know that it must be nonnull. 2630 if (ArrTy->getSizeModifier() == VariableArrayType::Static) { 2631 QualType ETy = ArrTy->getElementType(); 2632 llvm::Align Alignment = 2633 CGM.getNaturalTypeAlignment(ETy).getAsAlign(); 2634 AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment)); 2635 if (!getContext().getTargetAddressSpace(ETy) && 2636 !CGM.getCodeGenOpts().NullPointerIsValid) 2637 AI->addAttr(llvm::Attribute::NonNull); 2638 } 2639 } 2640 2641 // Set `align` attribute if any. 2642 const auto *AVAttr = PVD->getAttr<AlignValueAttr>(); 2643 if (!AVAttr) 2644 if (const auto *TOTy = dyn_cast<TypedefType>(OTy)) 2645 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>(); 2646 if (AVAttr && !SanOpts.has(SanitizerKind::Alignment)) { 2647 // If alignment-assumption sanitizer is enabled, we do *not* add 2648 // alignment attribute here, but emit normal alignment assumption, 2649 // so the UBSAN check could function. 2650 llvm::ConstantInt *AlignmentCI = 2651 cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment())); 2652 unsigned AlignmentInt = 2653 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment); 2654 if (AI->getParamAlign().valueOrOne() < AlignmentInt) { 2655 AI->removeAttr(llvm::Attribute::AttrKind::Alignment); 2656 AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr( 2657 llvm::Align(AlignmentInt))); 2658 } 2659 } 2660 } 2661 2662 // Set 'noalias' if an argument type has the `restrict` qualifier. 2663 if (Arg->getType().isRestrictQualified()) 2664 AI->addAttr(llvm::Attribute::NoAlias); 2665 } 2666 2667 // Prepare the argument value. If we have the trivial case, handle it 2668 // with no muss and fuss. 2669 if (!isa<llvm::StructType>(ArgI.getCoerceToType()) && 2670 ArgI.getCoerceToType() == ConvertType(Ty) && 2671 ArgI.getDirectOffset() == 0) { 2672 assert(NumIRArgs == 1); 2673 2674 // LLVM expects swifterror parameters to be used in very restricted 2675 // ways. Copy the value into a less-restricted temporary. 2676 llvm::Value *V = AI; 2677 if (FI.getExtParameterInfo(ArgNo).getABI() 2678 == ParameterABI::SwiftErrorResult) { 2679 QualType pointeeTy = Ty->getPointeeType(); 2680 assert(pointeeTy->isPointerType()); 2681 Address temp = 2682 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); 2683 Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy)); 2684 llvm::Value *incomingErrorValue = Builder.CreateLoad(arg); 2685 Builder.CreateStore(incomingErrorValue, temp); 2686 V = temp.getPointer(); 2687 2688 // Push a cleanup to copy the value back at the end of the function. 2689 // The convention does not guarantee that the value will be written 2690 // back if the function exits with an unwind exception. 2691 EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg); 2692 } 2693 2694 // Ensure the argument is the correct type. 2695 if (V->getType() != ArgI.getCoerceToType()) 2696 V = Builder.CreateBitCast(V, ArgI.getCoerceToType()); 2697 2698 if (isPromoted) 2699 V = emitArgumentDemotion(*this, Arg, V); 2700 2701 // Because of merging of function types from multiple decls it is 2702 // possible for the type of an argument to not match the corresponding 2703 // type in the function type. Since we are codegening the callee 2704 // in here, add a cast to the argument type. 2705 llvm::Type *LTy = ConvertType(Arg->getType()); 2706 if (V->getType() != LTy) 2707 V = Builder.CreateBitCast(V, LTy); 2708 2709 ArgVals.push_back(ParamValue::forDirect(V)); 2710 break; 2711 } 2712 2713 // VLST arguments are coerced to VLATs at the function boundary for 2714 // ABI consistency. If this is a VLST that was coerced to 2715 // a VLAT at the function boundary and the types match up, use 2716 // llvm.experimental.vector.extract to convert back to the original 2717 // VLST. 2718 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) { 2719 auto *Coerced = Fn->getArg(FirstIRArg); 2720 if (auto *VecTyFrom = 2721 dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) { 2722 if (VecTyFrom->getElementType() == VecTyTo->getElementType()) { 2723 llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty); 2724 2725 assert(NumIRArgs == 1); 2726 Coerced->setName(Arg->getName() + ".coerce"); 2727 ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector( 2728 VecTyTo, Coerced, Zero, "castFixedSve"))); 2729 break; 2730 } 2731 } 2732 } 2733 2734 Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg), 2735 Arg->getName()); 2736 2737 // Pointer to store into. 2738 Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI); 2739 2740 // Fast-isel and the optimizer generally like scalar values better than 2741 // FCAs, so we flatten them if this is safe to do for this argument. 2742 llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType()); 2743 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy && 2744 STy->getNumElements() > 1) { 2745 uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy); 2746 llvm::Type *DstTy = Ptr.getElementType(); 2747 uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy); 2748 2749 Address AddrToStoreInto = Address::invalid(); 2750 if (SrcSize <= DstSize) { 2751 AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy); 2752 } else { 2753 AddrToStoreInto = 2754 CreateTempAlloca(STy, Alloca.getAlignment(), "coerce"); 2755 } 2756 2757 assert(STy->getNumElements() == NumIRArgs); 2758 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 2759 auto AI = Fn->getArg(FirstIRArg + i); 2760 AI->setName(Arg->getName() + ".coerce" + Twine(i)); 2761 Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i); 2762 Builder.CreateStore(AI, EltPtr); 2763 } 2764 2765 if (SrcSize > DstSize) { 2766 Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize); 2767 } 2768 2769 } else { 2770 // Simple case, just do a coerced store of the argument into the alloca. 2771 assert(NumIRArgs == 1); 2772 auto AI = Fn->getArg(FirstIRArg); 2773 AI->setName(Arg->getName() + ".coerce"); 2774 CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this); 2775 } 2776 2777 // Match to what EmitParmDecl is expecting for this type. 2778 if (CodeGenFunction::hasScalarEvaluationKind(Ty)) { 2779 llvm::Value *V = 2780 EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc()); 2781 if (isPromoted) 2782 V = emitArgumentDemotion(*this, Arg, V); 2783 ArgVals.push_back(ParamValue::forDirect(V)); 2784 } else { 2785 ArgVals.push_back(ParamValue::forIndirect(Alloca)); 2786 } 2787 break; 2788 } 2789 2790 case ABIArgInfo::CoerceAndExpand: { 2791 // Reconstruct into a temporary. 2792 Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg)); 2793 ArgVals.push_back(ParamValue::forIndirect(alloca)); 2794 2795 auto coercionType = ArgI.getCoerceAndExpandType(); 2796 alloca = Builder.CreateElementBitCast(alloca, coercionType); 2797 2798 unsigned argIndex = FirstIRArg; 2799 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) { 2800 llvm::Type *eltType = coercionType->getElementType(i); 2801 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 2802 continue; 2803 2804 auto eltAddr = Builder.CreateStructGEP(alloca, i); 2805 auto elt = Fn->getArg(argIndex++); 2806 Builder.CreateStore(elt, eltAddr); 2807 } 2808 assert(argIndex == FirstIRArg + NumIRArgs); 2809 break; 2810 } 2811 2812 case ABIArgInfo::Expand: { 2813 // If this structure was expanded into multiple arguments then 2814 // we need to create a temporary and reconstruct it from the 2815 // arguments. 2816 Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg)); 2817 LValue LV = MakeAddrLValue(Alloca, Ty); 2818 ArgVals.push_back(ParamValue::forIndirect(Alloca)); 2819 2820 auto FnArgIter = Fn->arg_begin() + FirstIRArg; 2821 ExpandTypeFromArgs(Ty, LV, FnArgIter); 2822 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs); 2823 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) { 2824 auto AI = Fn->getArg(FirstIRArg + i); 2825 AI->setName(Arg->getName() + "." + Twine(i)); 2826 } 2827 break; 2828 } 2829 2830 case ABIArgInfo::Ignore: 2831 assert(NumIRArgs == 0); 2832 // Initialize the local variable appropriately. 2833 if (!hasScalarEvaluationKind(Ty)) { 2834 ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty))); 2835 } else { 2836 llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType())); 2837 ArgVals.push_back(ParamValue::forDirect(U)); 2838 } 2839 break; 2840 } 2841 } 2842 2843 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) { 2844 for (int I = Args.size() - 1; I >= 0; --I) 2845 EmitParmDecl(*Args[I], ArgVals[I], I + 1); 2846 } else { 2847 for (unsigned I = 0, E = Args.size(); I != E; ++I) 2848 EmitParmDecl(*Args[I], ArgVals[I], I + 1); 2849 } 2850 } 2851 2852 static void eraseUnusedBitCasts(llvm::Instruction *insn) { 2853 while (insn->use_empty()) { 2854 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn); 2855 if (!bitcast) return; 2856 2857 // This is "safe" because we would have used a ConstantExpr otherwise. 2858 insn = cast<llvm::Instruction>(bitcast->getOperand(0)); 2859 bitcast->eraseFromParent(); 2860 } 2861 } 2862 2863 /// Try to emit a fused autorelease of a return result. 2864 static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, 2865 llvm::Value *result) { 2866 // We must be immediately followed the cast. 2867 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock(); 2868 if (BB->empty()) return nullptr; 2869 if (&BB->back() != result) return nullptr; 2870 2871 llvm::Type *resultType = result->getType(); 2872 2873 // result is in a BasicBlock and is therefore an Instruction. 2874 llvm::Instruction *generator = cast<llvm::Instruction>(result); 2875 2876 SmallVector<llvm::Instruction *, 4> InstsToKill; 2877 2878 // Look for: 2879 // %generator = bitcast %type1* %generator2 to %type2* 2880 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) { 2881 // We would have emitted this as a constant if the operand weren't 2882 // an Instruction. 2883 generator = cast<llvm::Instruction>(bitcast->getOperand(0)); 2884 2885 // Require the generator to be immediately followed by the cast. 2886 if (generator->getNextNode() != bitcast) 2887 return nullptr; 2888 2889 InstsToKill.push_back(bitcast); 2890 } 2891 2892 // Look for: 2893 // %generator = call i8* @objc_retain(i8* %originalResult) 2894 // or 2895 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult) 2896 llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator); 2897 if (!call) return nullptr; 2898 2899 bool doRetainAutorelease; 2900 2901 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) { 2902 doRetainAutorelease = true; 2903 } else if (call->getCalledOperand() == 2904 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) { 2905 doRetainAutorelease = false; 2906 2907 // If we emitted an assembly marker for this call (and the 2908 // ARCEntrypoints field should have been set if so), go looking 2909 // for that call. If we can't find it, we can't do this 2910 // optimization. But it should always be the immediately previous 2911 // instruction, unless we needed bitcasts around the call. 2912 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) { 2913 llvm::Instruction *prev = call->getPrevNode(); 2914 assert(prev); 2915 if (isa<llvm::BitCastInst>(prev)) { 2916 prev = prev->getPrevNode(); 2917 assert(prev); 2918 } 2919 assert(isa<llvm::CallInst>(prev)); 2920 assert(cast<llvm::CallInst>(prev)->getCalledOperand() == 2921 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker); 2922 InstsToKill.push_back(prev); 2923 } 2924 } else { 2925 return nullptr; 2926 } 2927 2928 result = call->getArgOperand(0); 2929 InstsToKill.push_back(call); 2930 2931 // Keep killing bitcasts, for sanity. Note that we no longer care 2932 // about precise ordering as long as there's exactly one use. 2933 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) { 2934 if (!bitcast->hasOneUse()) break; 2935 InstsToKill.push_back(bitcast); 2936 result = bitcast->getOperand(0); 2937 } 2938 2939 // Delete all the unnecessary instructions, from latest to earliest. 2940 for (auto *I : InstsToKill) 2941 I->eraseFromParent(); 2942 2943 // Do the fused retain/autorelease if we were asked to. 2944 if (doRetainAutorelease) 2945 result = CGF.EmitARCRetainAutoreleaseReturnValue(result); 2946 2947 // Cast back to the result type. 2948 return CGF.Builder.CreateBitCast(result, resultType); 2949 } 2950 2951 /// If this is a +1 of the value of an immutable 'self', remove it. 2952 static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF, 2953 llvm::Value *result) { 2954 // This is only applicable to a method with an immutable 'self'. 2955 const ObjCMethodDecl *method = 2956 dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl); 2957 if (!method) return nullptr; 2958 const VarDecl *self = method->getSelfDecl(); 2959 if (!self->getType().isConstQualified()) return nullptr; 2960 2961 // Look for a retain call. 2962 llvm::CallInst *retainCall = 2963 dyn_cast<llvm::CallInst>(result->stripPointerCasts()); 2964 if (!retainCall || retainCall->getCalledOperand() != 2965 CGF.CGM.getObjCEntrypoints().objc_retain) 2966 return nullptr; 2967 2968 // Look for an ordinary load of 'self'. 2969 llvm::Value *retainedValue = retainCall->getArgOperand(0); 2970 llvm::LoadInst *load = 2971 dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts()); 2972 if (!load || load->isAtomic() || load->isVolatile() || 2973 load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer()) 2974 return nullptr; 2975 2976 // Okay! Burn it all down. This relies for correctness on the 2977 // assumption that the retain is emitted as part of the return and 2978 // that thereafter everything is used "linearly". 2979 llvm::Type *resultType = result->getType(); 2980 eraseUnusedBitCasts(cast<llvm::Instruction>(result)); 2981 assert(retainCall->use_empty()); 2982 retainCall->eraseFromParent(); 2983 eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue)); 2984 2985 return CGF.Builder.CreateBitCast(load, resultType); 2986 } 2987 2988 /// Emit an ARC autorelease of the result of a function. 2989 /// 2990 /// \return the value to actually return from the function 2991 static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF, 2992 llvm::Value *result) { 2993 // If we're returning 'self', kill the initial retain. This is a 2994 // heuristic attempt to "encourage correctness" in the really unfortunate 2995 // case where we have a return of self during a dealloc and we desperately 2996 // need to avoid the possible autorelease. 2997 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result)) 2998 return self; 2999 3000 // At -O0, try to emit a fused retain/autorelease. 3001 if (CGF.shouldUseFusedARCCalls()) 3002 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result)) 3003 return fused; 3004 3005 return CGF.EmitARCAutoreleaseReturnValue(result); 3006 } 3007 3008 /// Heuristically search for a dominating store to the return-value slot. 3009 static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { 3010 // Check if a User is a store which pointerOperand is the ReturnValue. 3011 // We are looking for stores to the ReturnValue, not for stores of the 3012 // ReturnValue to some other location. 3013 auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * { 3014 auto *SI = dyn_cast<llvm::StoreInst>(U); 3015 if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer()) 3016 return nullptr; 3017 // These aren't actually possible for non-coerced returns, and we 3018 // only care about non-coerced returns on this code path. 3019 assert(!SI->isAtomic() && !SI->isVolatile()); 3020 return SI; 3021 }; 3022 // If there are multiple uses of the return-value slot, just check 3023 // for something immediately preceding the IP. Sometimes this can 3024 // happen with how we generate implicit-returns; it can also happen 3025 // with noreturn cleanups. 3026 if (!CGF.ReturnValue.getPointer()->hasOneUse()) { 3027 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); 3028 if (IP->empty()) return nullptr; 3029 llvm::Instruction *I = &IP->back(); 3030 3031 // Skip lifetime markers 3032 for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(), 3033 IE = IP->rend(); 3034 II != IE; ++II) { 3035 if (llvm::IntrinsicInst *Intrinsic = 3036 dyn_cast<llvm::IntrinsicInst>(&*II)) { 3037 if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) { 3038 const llvm::Value *CastAddr = Intrinsic->getArgOperand(1); 3039 ++II; 3040 if (II == IE) 3041 break; 3042 if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II)) 3043 continue; 3044 } 3045 } 3046 I = &*II; 3047 break; 3048 } 3049 3050 return GetStoreIfValid(I); 3051 } 3052 3053 llvm::StoreInst *store = 3054 GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back()); 3055 if (!store) return nullptr; 3056 3057 // Now do a first-and-dirty dominance check: just walk up the 3058 // single-predecessors chain from the current insertion point. 3059 llvm::BasicBlock *StoreBB = store->getParent(); 3060 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); 3061 while (IP != StoreBB) { 3062 if (!(IP = IP->getSinglePredecessor())) 3063 return nullptr; 3064 } 3065 3066 // Okay, the store's basic block dominates the insertion point; we 3067 // can do our thing. 3068 return store; 3069 } 3070 3071 // Helper functions for EmitCMSEClearRecord 3072 3073 // Set the bits corresponding to a field having width `BitWidth` and located at 3074 // offset `BitOffset` (from the least significant bit) within a storage unit of 3075 // `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte. 3076 // Use little-endian layout, i.e.`Bits[0]` is the LSB. 3077 static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset, 3078 int BitWidth, int CharWidth) { 3079 assert(CharWidth <= 64); 3080 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth); 3081 3082 int Pos = 0; 3083 if (BitOffset >= CharWidth) { 3084 Pos += BitOffset / CharWidth; 3085 BitOffset = BitOffset % CharWidth; 3086 } 3087 3088 const uint64_t Used = (uint64_t(1) << CharWidth) - 1; 3089 if (BitOffset + BitWidth >= CharWidth) { 3090 Bits[Pos++] |= (Used << BitOffset) & Used; 3091 BitWidth -= CharWidth - BitOffset; 3092 BitOffset = 0; 3093 } 3094 3095 while (BitWidth >= CharWidth) { 3096 Bits[Pos++] = Used; 3097 BitWidth -= CharWidth; 3098 } 3099 3100 if (BitWidth > 0) 3101 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset; 3102 } 3103 3104 // Set the bits corresponding to a field having width `BitWidth` and located at 3105 // offset `BitOffset` (from the least significant bit) within a storage unit of 3106 // `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of 3107 // `Bits` corresponds to one target byte. Use target endian layout. 3108 static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset, 3109 int StorageSize, int BitOffset, int BitWidth, 3110 int CharWidth, bool BigEndian) { 3111 3112 SmallVector<uint64_t, 8> TmpBits(StorageSize); 3113 setBitRange(TmpBits, BitOffset, BitWidth, CharWidth); 3114 3115 if (BigEndian) 3116 std::reverse(TmpBits.begin(), TmpBits.end()); 3117 3118 for (uint64_t V : TmpBits) 3119 Bits[StorageOffset++] |= V; 3120 } 3121 3122 static void setUsedBits(CodeGenModule &, QualType, int, 3123 SmallVectorImpl<uint64_t> &); 3124 3125 // Set the bits in `Bits`, which correspond to the value representations of 3126 // the actual members of the record type `RTy`. Note that this function does 3127 // not handle base classes, virtual tables, etc, since they cannot happen in 3128 // CMSE function arguments or return. The bit mask corresponds to the target 3129 // memory layout, i.e. it's endian dependent. 3130 static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset, 3131 SmallVectorImpl<uint64_t> &Bits) { 3132 ASTContext &Context = CGM.getContext(); 3133 int CharWidth = Context.getCharWidth(); 3134 const RecordDecl *RD = RTy->getDecl()->getDefinition(); 3135 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD); 3136 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD); 3137 3138 int Idx = 0; 3139 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) { 3140 const FieldDecl *F = *I; 3141 3142 if (F->isUnnamedBitfield() || F->isZeroLengthBitField(Context) || 3143 F->getType()->isIncompleteArrayType()) 3144 continue; 3145 3146 if (F->isBitField()) { 3147 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F); 3148 setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(), 3149 BFI.StorageSize / CharWidth, BFI.Offset, 3150 BFI.Size, CharWidth, 3151 CGM.getDataLayout().isBigEndian()); 3152 continue; 3153 } 3154 3155 setUsedBits(CGM, F->getType(), 3156 Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits); 3157 } 3158 } 3159 3160 // Set the bits in `Bits`, which correspond to the value representations of 3161 // the elements of an array type `ATy`. 3162 static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy, 3163 int Offset, SmallVectorImpl<uint64_t> &Bits) { 3164 const ASTContext &Context = CGM.getContext(); 3165 3166 QualType ETy = Context.getBaseElementType(ATy); 3167 int Size = Context.getTypeSizeInChars(ETy).getQuantity(); 3168 SmallVector<uint64_t, 4> TmpBits(Size); 3169 setUsedBits(CGM, ETy, 0, TmpBits); 3170 3171 for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; ++I) { 3172 auto Src = TmpBits.begin(); 3173 auto Dst = Bits.begin() + Offset + I * Size; 3174 for (int J = 0; J < Size; ++J) 3175 *Dst++ |= *Src++; 3176 } 3177 } 3178 3179 // Set the bits in `Bits`, which correspond to the value representations of 3180 // the type `QTy`. 3181 static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset, 3182 SmallVectorImpl<uint64_t> &Bits) { 3183 if (const auto *RTy = QTy->getAs<RecordType>()) 3184 return setUsedBits(CGM, RTy, Offset, Bits); 3185 3186 ASTContext &Context = CGM.getContext(); 3187 if (const auto *ATy = Context.getAsConstantArrayType(QTy)) 3188 return setUsedBits(CGM, ATy, Offset, Bits); 3189 3190 int Size = Context.getTypeSizeInChars(QTy).getQuantity(); 3191 if (Size <= 0) 3192 return; 3193 3194 std::fill_n(Bits.begin() + Offset, Size, 3195 (uint64_t(1) << Context.getCharWidth()) - 1); 3196 } 3197 3198 static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits, 3199 int Pos, int Size, int CharWidth, 3200 bool BigEndian) { 3201 assert(Size > 0); 3202 uint64_t Mask = 0; 3203 if (BigEndian) { 3204 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E; 3205 ++P) 3206 Mask = (Mask << CharWidth) | *P; 3207 } else { 3208 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos; 3209 do 3210 Mask = (Mask << CharWidth) | *--P; 3211 while (P != End); 3212 } 3213 return Mask; 3214 } 3215 3216 // Emit code to clear the bits in a record, which aren't a part of any user 3217 // declared member, when the record is a function return. 3218 llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src, 3219 llvm::IntegerType *ITy, 3220 QualType QTy) { 3221 assert(Src->getType() == ITy); 3222 assert(ITy->getScalarSizeInBits() <= 64); 3223 3224 const llvm::DataLayout &DataLayout = CGM.getDataLayout(); 3225 int Size = DataLayout.getTypeStoreSize(ITy); 3226 SmallVector<uint64_t, 4> Bits(Size); 3227 setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits); 3228 3229 int CharWidth = CGM.getContext().getCharWidth(); 3230 uint64_t Mask = 3231 buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian()); 3232 3233 return Builder.CreateAnd(Src, Mask, "cmse.clear"); 3234 } 3235 3236 // Emit code to clear the bits in a record, which aren't a part of any user 3237 // declared member, when the record is a function argument. 3238 llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src, 3239 llvm::ArrayType *ATy, 3240 QualType QTy) { 3241 const llvm::DataLayout &DataLayout = CGM.getDataLayout(); 3242 int Size = DataLayout.getTypeStoreSize(ATy); 3243 SmallVector<uint64_t, 16> Bits(Size); 3244 setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits); 3245 3246 // Clear each element of the LLVM array. 3247 int CharWidth = CGM.getContext().getCharWidth(); 3248 int CharsPerElt = 3249 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth; 3250 int MaskIndex = 0; 3251 llvm::Value *R = llvm::UndefValue::get(ATy); 3252 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) { 3253 uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth, 3254 DataLayout.isBigEndian()); 3255 MaskIndex += CharsPerElt; 3256 llvm::Value *T0 = Builder.CreateExtractValue(Src, I); 3257 llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear"); 3258 R = Builder.CreateInsertValue(R, T1, I); 3259 } 3260 3261 return R; 3262 } 3263 3264 void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, 3265 bool EmitRetDbgLoc, 3266 SourceLocation EndLoc) { 3267 if (FI.isNoReturn()) { 3268 // Noreturn functions don't return. 3269 EmitUnreachable(EndLoc); 3270 return; 3271 } 3272 3273 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) { 3274 // Naked functions don't have epilogues. 3275 Builder.CreateUnreachable(); 3276 return; 3277 } 3278 3279 // Functions with no result always return void. 3280 if (!ReturnValue.isValid()) { 3281 Builder.CreateRetVoid(); 3282 return; 3283 } 3284 3285 llvm::DebugLoc RetDbgLoc; 3286 llvm::Value *RV = nullptr; 3287 QualType RetTy = FI.getReturnType(); 3288 const ABIArgInfo &RetAI = FI.getReturnInfo(); 3289 3290 switch (RetAI.getKind()) { 3291 case ABIArgInfo::InAlloca: 3292 // Aggregrates get evaluated directly into the destination. Sometimes we 3293 // need to return the sret value in a register, though. 3294 assert(hasAggregateEvaluationKind(RetTy)); 3295 if (RetAI.getInAllocaSRet()) { 3296 llvm::Function::arg_iterator EI = CurFn->arg_end(); 3297 --EI; 3298 llvm::Value *ArgStruct = &*EI; 3299 llvm::Value *SRet = Builder.CreateStructGEP( 3300 nullptr, ArgStruct, RetAI.getInAllocaFieldIndex()); 3301 RV = Builder.CreateAlignedLoad(SRet, getPointerAlign(), "sret"); 3302 } 3303 break; 3304 3305 case ABIArgInfo::Indirect: { 3306 auto AI = CurFn->arg_begin(); 3307 if (RetAI.isSRetAfterThis()) 3308 ++AI; 3309 switch (getEvaluationKind(RetTy)) { 3310 case TEK_Complex: { 3311 ComplexPairTy RT = 3312 EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc); 3313 EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy), 3314 /*isInit*/ true); 3315 break; 3316 } 3317 case TEK_Aggregate: 3318 // Do nothing; aggregrates get evaluated directly into the destination. 3319 break; 3320 case TEK_Scalar: 3321 EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), 3322 MakeNaturalAlignAddrLValue(&*AI, RetTy), 3323 /*isInit*/ true); 3324 break; 3325 } 3326 break; 3327 } 3328 3329 case ABIArgInfo::Extend: 3330 case ABIArgInfo::Direct: 3331 if (RetAI.getCoerceToType() == ConvertType(RetTy) && 3332 RetAI.getDirectOffset() == 0) { 3333 // The internal return value temp always will have pointer-to-return-type 3334 // type, just do a load. 3335 3336 // If there is a dominating store to ReturnValue, we can elide 3337 // the load, zap the store, and usually zap the alloca. 3338 if (llvm::StoreInst *SI = 3339 findDominatingStoreToReturnValue(*this)) { 3340 // Reuse the debug location from the store unless there is 3341 // cleanup code to be emitted between the store and return 3342 // instruction. 3343 if (EmitRetDbgLoc && !AutoreleaseResult) 3344 RetDbgLoc = SI->getDebugLoc(); 3345 // Get the stored value and nuke the now-dead store. 3346 RV = SI->getValueOperand(); 3347 SI->eraseFromParent(); 3348 3349 // Otherwise, we have to do a simple load. 3350 } else { 3351 RV = Builder.CreateLoad(ReturnValue); 3352 } 3353 } else { 3354 // If the value is offset in memory, apply the offset now. 3355 Address V = emitAddressAtOffset(*this, ReturnValue, RetAI); 3356 3357 RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this); 3358 } 3359 3360 // In ARC, end functions that return a retainable type with a call 3361 // to objc_autoreleaseReturnValue. 3362 if (AutoreleaseResult) { 3363 #ifndef NDEBUG 3364 // Type::isObjCRetainabletype has to be called on a QualType that hasn't 3365 // been stripped of the typedefs, so we cannot use RetTy here. Get the 3366 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from 3367 // CurCodeDecl or BlockInfo. 3368 QualType RT; 3369 3370 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl)) 3371 RT = FD->getReturnType(); 3372 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl)) 3373 RT = MD->getReturnType(); 3374 else if (isa<BlockDecl>(CurCodeDecl)) 3375 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType(); 3376 else 3377 llvm_unreachable("Unexpected function/method type"); 3378 3379 assert(getLangOpts().ObjCAutoRefCount && 3380 !FI.isReturnsRetained() && 3381 RT->isObjCRetainableType()); 3382 #endif 3383 RV = emitAutoreleaseOfResult(*this, RV); 3384 } 3385 3386 break; 3387 3388 case ABIArgInfo::Ignore: 3389 break; 3390 3391 case ABIArgInfo::CoerceAndExpand: { 3392 auto coercionType = RetAI.getCoerceAndExpandType(); 3393 3394 // Load all of the coerced elements out into results. 3395 llvm::SmallVector<llvm::Value*, 4> results; 3396 Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType); 3397 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) { 3398 auto coercedEltType = coercionType->getElementType(i); 3399 if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType)) 3400 continue; 3401 3402 auto eltAddr = Builder.CreateStructGEP(addr, i); 3403 auto elt = Builder.CreateLoad(eltAddr); 3404 results.push_back(elt); 3405 } 3406 3407 // If we have one result, it's the single direct result type. 3408 if (results.size() == 1) { 3409 RV = results[0]; 3410 3411 // Otherwise, we need to make a first-class aggregate. 3412 } else { 3413 // Construct a return type that lacks padding elements. 3414 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType(); 3415 3416 RV = llvm::UndefValue::get(returnType); 3417 for (unsigned i = 0, e = results.size(); i != e; ++i) { 3418 RV = Builder.CreateInsertValue(RV, results[i], i); 3419 } 3420 } 3421 break; 3422 } 3423 case ABIArgInfo::Expand: 3424 case ABIArgInfo::IndirectAliased: 3425 llvm_unreachable("Invalid ABI kind for return argument"); 3426 } 3427 3428 llvm::Instruction *Ret; 3429 if (RV) { 3430 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) { 3431 // For certain return types, clear padding bits, as they may reveal 3432 // sensitive information. 3433 // Small struct/union types are passed as integers. 3434 auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType()); 3435 if (ITy != nullptr && isa<RecordType>(RetTy.getCanonicalType())) 3436 RV = EmitCMSEClearRecord(RV, ITy, RetTy); 3437 } 3438 EmitReturnValueCheck(RV); 3439 Ret = Builder.CreateRet(RV); 3440 } else { 3441 Ret = Builder.CreateRetVoid(); 3442 } 3443 3444 if (RetDbgLoc) 3445 Ret->setDebugLoc(std::move(RetDbgLoc)); 3446 } 3447 3448 void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) { 3449 // A current decl may not be available when emitting vtable thunks. 3450 if (!CurCodeDecl) 3451 return; 3452 3453 // If the return block isn't reachable, neither is this check, so don't emit 3454 // it. 3455 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) 3456 return; 3457 3458 ReturnsNonNullAttr *RetNNAttr = nullptr; 3459 if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute)) 3460 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>(); 3461 3462 if (!RetNNAttr && !requiresReturnValueNullabilityCheck()) 3463 return; 3464 3465 // Prefer the returns_nonnull attribute if it's present. 3466 SourceLocation AttrLoc; 3467 SanitizerMask CheckKind; 3468 SanitizerHandler Handler; 3469 if (RetNNAttr) { 3470 assert(!requiresReturnValueNullabilityCheck() && 3471 "Cannot check nullability and the nonnull attribute"); 3472 AttrLoc = RetNNAttr->getLocation(); 3473 CheckKind = SanitizerKind::ReturnsNonnullAttribute; 3474 Handler = SanitizerHandler::NonnullReturn; 3475 } else { 3476 if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl)) 3477 if (auto *TSI = DD->getTypeSourceInfo()) 3478 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>()) 3479 AttrLoc = FTL.getReturnLoc().findNullabilityLoc(); 3480 CheckKind = SanitizerKind::NullabilityReturn; 3481 Handler = SanitizerHandler::NullabilityReturn; 3482 } 3483 3484 SanitizerScope SanScope(this); 3485 3486 // Make sure the "return" source location is valid. If we're checking a 3487 // nullability annotation, make sure the preconditions for the check are met. 3488 llvm::BasicBlock *Check = createBasicBlock("nullcheck"); 3489 llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck"); 3490 llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load"); 3491 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr); 3492 if (requiresReturnValueNullabilityCheck()) 3493 CanNullCheck = 3494 Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition); 3495 Builder.CreateCondBr(CanNullCheck, Check, NoCheck); 3496 EmitBlock(Check); 3497 3498 // Now do the null check. 3499 llvm::Value *Cond = Builder.CreateIsNotNull(RV); 3500 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)}; 3501 llvm::Value *DynamicData[] = {SLocPtr}; 3502 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData); 3503 3504 EmitBlock(NoCheck); 3505 3506 #ifndef NDEBUG 3507 // The return location should not be used after the check has been emitted. 3508 ReturnLocation = Address::invalid(); 3509 #endif 3510 } 3511 3512 static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) { 3513 const CXXRecordDecl *RD = type->getAsCXXRecordDecl(); 3514 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory; 3515 } 3516 3517 static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, 3518 QualType Ty) { 3519 // FIXME: Generate IR in one pass, rather than going back and fixing up these 3520 // placeholders. 3521 llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty); 3522 llvm::Type *IRPtrTy = IRTy->getPointerTo(); 3523 llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo()); 3524 3525 // FIXME: When we generate this IR in one pass, we shouldn't need 3526 // this win32-specific alignment hack. 3527 CharUnits Align = CharUnits::fromQuantity(4); 3528 Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align); 3529 3530 return AggValueSlot::forAddr(Address(Placeholder, Align), 3531 Ty.getQualifiers(), 3532 AggValueSlot::IsNotDestructed, 3533 AggValueSlot::DoesNotNeedGCBarriers, 3534 AggValueSlot::IsNotAliased, 3535 AggValueSlot::DoesNotOverlap); 3536 } 3537 3538 void CodeGenFunction::EmitDelegateCallArg(CallArgList &args, 3539 const VarDecl *param, 3540 SourceLocation loc) { 3541 // StartFunction converted the ABI-lowered parameter(s) into a 3542 // local alloca. We need to turn that into an r-value suitable 3543 // for EmitCall. 3544 Address local = GetAddrOfLocalVar(param); 3545 3546 QualType type = param->getType(); 3547 3548 if (isInAllocaArgument(CGM.getCXXABI(), type)) { 3549 CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter"); 3550 } 3551 3552 // GetAddrOfLocalVar returns a pointer-to-pointer for references, 3553 // but the argument needs to be the original pointer. 3554 if (type->isReferenceType()) { 3555 args.add(RValue::get(Builder.CreateLoad(local)), type); 3556 3557 // In ARC, move out of consumed arguments so that the release cleanup 3558 // entered by StartFunction doesn't cause an over-release. This isn't 3559 // optimal -O0 code generation, but it should get cleaned up when 3560 // optimization is enabled. This also assumes that delegate calls are 3561 // performed exactly once for a set of arguments, but that should be safe. 3562 } else if (getLangOpts().ObjCAutoRefCount && 3563 param->hasAttr<NSConsumedAttr>() && 3564 type->isObjCRetainableType()) { 3565 llvm::Value *ptr = Builder.CreateLoad(local); 3566 auto null = 3567 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType())); 3568 Builder.CreateStore(null, local); 3569 args.add(RValue::get(ptr), type); 3570 3571 // For the most part, we just need to load the alloca, except that 3572 // aggregate r-values are actually pointers to temporaries. 3573 } else { 3574 args.add(convertTempToRValue(local, type, loc), type); 3575 } 3576 3577 // Deactivate the cleanup for the callee-destructed param that was pushed. 3578 if (hasAggregateEvaluationKind(type) && !CurFuncIsThunk && 3579 type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() && 3580 param->needsDestruction(getContext())) { 3581 EHScopeStack::stable_iterator cleanup = 3582 CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param)); 3583 assert(cleanup.isValid() && 3584 "cleanup for callee-destructed param not recorded"); 3585 // This unreachable is a temporary marker which will be removed later. 3586 llvm::Instruction *isActive = Builder.CreateUnreachable(); 3587 args.addArgCleanupDeactivation(cleanup, isActive); 3588 } 3589 } 3590 3591 static bool isProvablyNull(llvm::Value *addr) { 3592 return isa<llvm::ConstantPointerNull>(addr); 3593 } 3594 3595 /// Emit the actual writing-back of a writeback. 3596 static void emitWriteback(CodeGenFunction &CGF, 3597 const CallArgList::Writeback &writeback) { 3598 const LValue &srcLV = writeback.Source; 3599 Address srcAddr = srcLV.getAddress(CGF); 3600 assert(!isProvablyNull(srcAddr.getPointer()) && 3601 "shouldn't have writeback for provably null argument"); 3602 3603 llvm::BasicBlock *contBB = nullptr; 3604 3605 // If the argument wasn't provably non-null, we need to null check 3606 // before doing the store. 3607 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), 3608 CGF.CGM.getDataLayout()); 3609 if (!provablyNonNull) { 3610 llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback"); 3611 contBB = CGF.createBasicBlock("icr.done"); 3612 3613 llvm::Value *isNull = 3614 CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); 3615 CGF.Builder.CreateCondBr(isNull, contBB, writebackBB); 3616 CGF.EmitBlock(writebackBB); 3617 } 3618 3619 // Load the value to writeback. 3620 llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary); 3621 3622 // Cast it back, in case we're writing an id to a Foo* or something. 3623 value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(), 3624 "icr.writeback-cast"); 3625 3626 // Perform the writeback. 3627 3628 // If we have a "to use" value, it's something we need to emit a use 3629 // of. This has to be carefully threaded in: if it's done after the 3630 // release it's potentially undefined behavior (and the optimizer 3631 // will ignore it), and if it happens before the retain then the 3632 // optimizer could move the release there. 3633 if (writeback.ToUse) { 3634 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong); 3635 3636 // Retain the new value. No need to block-copy here: the block's 3637 // being passed up the stack. 3638 value = CGF.EmitARCRetainNonBlock(value); 3639 3640 // Emit the intrinsic use here. 3641 CGF.EmitARCIntrinsicUse(writeback.ToUse); 3642 3643 // Load the old value (primitively). 3644 llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation()); 3645 3646 // Put the new value in place (primitively). 3647 CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false); 3648 3649 // Release the old value. 3650 CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime()); 3651 3652 // Otherwise, we can just do a normal lvalue store. 3653 } else { 3654 CGF.EmitStoreThroughLValue(RValue::get(value), srcLV); 3655 } 3656 3657 // Jump to the continuation block. 3658 if (!provablyNonNull) 3659 CGF.EmitBlock(contBB); 3660 } 3661 3662 static void emitWritebacks(CodeGenFunction &CGF, 3663 const CallArgList &args) { 3664 for (const auto &I : args.writebacks()) 3665 emitWriteback(CGF, I); 3666 } 3667 3668 static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF, 3669 const CallArgList &CallArgs) { 3670 ArrayRef<CallArgList::CallArgCleanup> Cleanups = 3671 CallArgs.getCleanupsToDeactivate(); 3672 // Iterate in reverse to increase the likelihood of popping the cleanup. 3673 for (const auto &I : llvm::reverse(Cleanups)) { 3674 CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP); 3675 I.IsActiveIP->eraseFromParent(); 3676 } 3677 } 3678 3679 static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) { 3680 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens())) 3681 if (uop->getOpcode() == UO_AddrOf) 3682 return uop->getSubExpr(); 3683 return nullptr; 3684 } 3685 3686 /// Emit an argument that's being passed call-by-writeback. That is, 3687 /// we are passing the address of an __autoreleased temporary; it 3688 /// might be copy-initialized with the current value of the given 3689 /// address, but it will definitely be copied out of after the call. 3690 static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, 3691 const ObjCIndirectCopyRestoreExpr *CRE) { 3692 LValue srcLV; 3693 3694 // Make an optimistic effort to emit the address as an l-value. 3695 // This can fail if the argument expression is more complicated. 3696 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) { 3697 srcLV = CGF.EmitLValue(lvExpr); 3698 3699 // Otherwise, just emit it as a scalar. 3700 } else { 3701 Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr()); 3702 3703 QualType srcAddrType = 3704 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType(); 3705 srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType); 3706 } 3707 Address srcAddr = srcLV.getAddress(CGF); 3708 3709 // The dest and src types don't necessarily match in LLVM terms 3710 // because of the crazy ObjC compatibility rules. 3711 3712 llvm::PointerType *destType = 3713 cast<llvm::PointerType>(CGF.ConvertType(CRE->getType())); 3714 3715 // If the address is a constant null, just pass the appropriate null. 3716 if (isProvablyNull(srcAddr.getPointer())) { 3717 args.add(RValue::get(llvm::ConstantPointerNull::get(destType)), 3718 CRE->getType()); 3719 return; 3720 } 3721 3722 // Create the temporary. 3723 Address temp = CGF.CreateTempAlloca(destType->getElementType(), 3724 CGF.getPointerAlign(), 3725 "icr.temp"); 3726 // Loading an l-value can introduce a cleanup if the l-value is __weak, 3727 // and that cleanup will be conditional if we can't prove that the l-value 3728 // isn't null, so we need to register a dominating point so that the cleanups 3729 // system will make valid IR. 3730 CodeGenFunction::ConditionalEvaluation condEval(CGF); 3731 3732 // Zero-initialize it if we're not doing a copy-initialization. 3733 bool shouldCopy = CRE->shouldCopy(); 3734 if (!shouldCopy) { 3735 llvm::Value *null = 3736 llvm::ConstantPointerNull::get( 3737 cast<llvm::PointerType>(destType->getElementType())); 3738 CGF.Builder.CreateStore(null, temp); 3739 } 3740 3741 llvm::BasicBlock *contBB = nullptr; 3742 llvm::BasicBlock *originBB = nullptr; 3743 3744 // If the address is *not* known to be non-null, we need to switch. 3745 llvm::Value *finalArgument; 3746 3747 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), 3748 CGF.CGM.getDataLayout()); 3749 if (provablyNonNull) { 3750 finalArgument = temp.getPointer(); 3751 } else { 3752 llvm::Value *isNull = 3753 CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); 3754 3755 finalArgument = CGF.Builder.CreateSelect(isNull, 3756 llvm::ConstantPointerNull::get(destType), 3757 temp.getPointer(), "icr.argument"); 3758 3759 // If we need to copy, then the load has to be conditional, which 3760 // means we need control flow. 3761 if (shouldCopy) { 3762 originBB = CGF.Builder.GetInsertBlock(); 3763 contBB = CGF.createBasicBlock("icr.cont"); 3764 llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy"); 3765 CGF.Builder.CreateCondBr(isNull, contBB, copyBB); 3766 CGF.EmitBlock(copyBB); 3767 condEval.begin(CGF); 3768 } 3769 } 3770 3771 llvm::Value *valueToUse = nullptr; 3772 3773 // Perform a copy if necessary. 3774 if (shouldCopy) { 3775 RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation()); 3776 assert(srcRV.isScalar()); 3777 3778 llvm::Value *src = srcRV.getScalarVal(); 3779 src = CGF.Builder.CreateBitCast(src, destType->getElementType(), 3780 "icr.cast"); 3781 3782 // Use an ordinary store, not a store-to-lvalue. 3783 CGF.Builder.CreateStore(src, temp); 3784 3785 // If optimization is enabled, and the value was held in a 3786 // __strong variable, we need to tell the optimizer that this 3787 // value has to stay alive until we're doing the store back. 3788 // This is because the temporary is effectively unretained, 3789 // and so otherwise we can violate the high-level semantics. 3790 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 && 3791 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) { 3792 valueToUse = src; 3793 } 3794 } 3795 3796 // Finish the control flow if we needed it. 3797 if (shouldCopy && !provablyNonNull) { 3798 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock(); 3799 CGF.EmitBlock(contBB); 3800 3801 // Make a phi for the value to intrinsically use. 3802 if (valueToUse) { 3803 llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2, 3804 "icr.to-use"); 3805 phiToUse->addIncoming(valueToUse, copyBB); 3806 phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()), 3807 originBB); 3808 valueToUse = phiToUse; 3809 } 3810 3811 condEval.end(CGF); 3812 } 3813 3814 args.addWriteback(srcLV, temp, valueToUse); 3815 args.add(RValue::get(finalArgument), CRE->getType()); 3816 } 3817 3818 void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) { 3819 assert(!StackBase); 3820 3821 // Save the stack. 3822 llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave); 3823 StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save"); 3824 } 3825 3826 void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const { 3827 if (StackBase) { 3828 // Restore the stack after the call. 3829 llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore); 3830 CGF.Builder.CreateCall(F, StackBase); 3831 } 3832 } 3833 3834 void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, 3835 SourceLocation ArgLoc, 3836 AbstractCallee AC, 3837 unsigned ParmNum) { 3838 if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) || 3839 SanOpts.has(SanitizerKind::NullabilityArg))) 3840 return; 3841 3842 // The param decl may be missing in a variadic function. 3843 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum) : nullptr; 3844 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum; 3845 3846 // Prefer the nonnull attribute if it's present. 3847 const NonNullAttr *NNAttr = nullptr; 3848 if (SanOpts.has(SanitizerKind::NonnullAttribute)) 3849 NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo); 3850 3851 bool CanCheckNullability = false; 3852 if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD) { 3853 auto Nullability = PVD->getType()->getNullability(getContext()); 3854 CanCheckNullability = Nullability && 3855 *Nullability == NullabilityKind::NonNull && 3856 PVD->getTypeSourceInfo(); 3857 } 3858 3859 if (!NNAttr && !CanCheckNullability) 3860 return; 3861 3862 SourceLocation AttrLoc; 3863 SanitizerMask CheckKind; 3864 SanitizerHandler Handler; 3865 if (NNAttr) { 3866 AttrLoc = NNAttr->getLocation(); 3867 CheckKind = SanitizerKind::NonnullAttribute; 3868 Handler = SanitizerHandler::NonnullArg; 3869 } else { 3870 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc(); 3871 CheckKind = SanitizerKind::NullabilityArg; 3872 Handler = SanitizerHandler::NullabilityArg; 3873 } 3874 3875 SanitizerScope SanScope(this); 3876 llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType); 3877 llvm::Constant *StaticData[] = { 3878 EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc), 3879 llvm::ConstantInt::get(Int32Ty, ArgNo + 1), 3880 }; 3881 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None); 3882 } 3883 3884 // Check if the call is going to use the inalloca convention. This needs to 3885 // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged 3886 // later, so we can't check it directly. 3887 static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC, 3888 ArrayRef<QualType> ArgTypes) { 3889 // The Swift calling convention doesn't go through the target-specific 3890 // argument classification, so it never uses inalloca. 3891 // TODO: Consider limiting inalloca use to only calling conventions supported 3892 // by MSVC. 3893 if (ExplicitCC == CC_Swift) 3894 return false; 3895 if (!CGM.getTarget().getCXXABI().isMicrosoft()) 3896 return false; 3897 return llvm::any_of(ArgTypes, [&](QualType Ty) { 3898 return isInAllocaArgument(CGM.getCXXABI(), Ty); 3899 }); 3900 } 3901 3902 #ifndef NDEBUG 3903 // Determine whether the given argument is an Objective-C method 3904 // that may have type parameters in its signature. 3905 static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) { 3906 const DeclContext *dc = method->getDeclContext(); 3907 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) { 3908 return classDecl->getTypeParamListAsWritten(); 3909 } 3910 3911 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) { 3912 return catDecl->getTypeParamList(); 3913 } 3914 3915 return false; 3916 } 3917 #endif 3918 3919 /// EmitCallArgs - Emit call arguments for a function. 3920 void CodeGenFunction::EmitCallArgs( 3921 CallArgList &Args, PrototypeWrapper Prototype, 3922 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange, 3923 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) { 3924 SmallVector<QualType, 16> ArgTypes; 3925 3926 assert((ParamsToSkip == 0 || Prototype.P) && 3927 "Can't skip parameters if type info is not provided"); 3928 3929 // This variable only captures *explicitly* written conventions, not those 3930 // applied by default via command line flags or target defaults, such as 3931 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would 3932 // require knowing if this is a C++ instance method or being able to see 3933 // unprototyped FunctionTypes. 3934 CallingConv ExplicitCC = CC_C; 3935 3936 // First, if a prototype was provided, use those argument types. 3937 bool IsVariadic = false; 3938 if (Prototype.P) { 3939 const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>(); 3940 if (MD) { 3941 IsVariadic = MD->isVariadic(); 3942 ExplicitCC = getCallingConventionForDecl( 3943 MD, CGM.getTarget().getTriple().isOSWindows()); 3944 ArgTypes.assign(MD->param_type_begin() + ParamsToSkip, 3945 MD->param_type_end()); 3946 } else { 3947 const auto *FPT = Prototype.P.get<const FunctionProtoType *>(); 3948 IsVariadic = FPT->isVariadic(); 3949 ExplicitCC = FPT->getExtInfo().getCC(); 3950 ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip, 3951 FPT->param_type_end()); 3952 } 3953 3954 #ifndef NDEBUG 3955 // Check that the prototyped types match the argument expression types. 3956 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD); 3957 CallExpr::const_arg_iterator Arg = ArgRange.begin(); 3958 for (QualType Ty : ArgTypes) { 3959 assert(Arg != ArgRange.end() && "Running over edge of argument list!"); 3960 assert( 3961 (isGenericMethod || Ty->isVariablyModifiedType() || 3962 Ty.getNonReferenceType()->isObjCRetainableType() || 3963 getContext() 3964 .getCanonicalType(Ty.getNonReferenceType()) 3965 .getTypePtr() == 3966 getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) && 3967 "type mismatch in call argument!"); 3968 ++Arg; 3969 } 3970 3971 // Either we've emitted all the call args, or we have a call to variadic 3972 // function. 3973 assert((Arg == ArgRange.end() || IsVariadic) && 3974 "Extra arguments in non-variadic function!"); 3975 #endif 3976 } 3977 3978 // If we still have any arguments, emit them using the type of the argument. 3979 for (auto *A : llvm::make_range(std::next(ArgRange.begin(), ArgTypes.size()), 3980 ArgRange.end())) 3981 ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType()); 3982 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin())); 3983 3984 // We must evaluate arguments from right to left in the MS C++ ABI, 3985 // because arguments are destroyed left to right in the callee. As a special 3986 // case, there are certain language constructs that require left-to-right 3987 // evaluation, and in those cases we consider the evaluation order requirement 3988 // to trump the "destruction order is reverse construction order" guarantee. 3989 bool LeftToRight = 3990 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee() 3991 ? Order == EvaluationOrder::ForceLeftToRight 3992 : Order != EvaluationOrder::ForceRightToLeft; 3993 3994 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg, 3995 RValue EmittedArg) { 3996 if (!AC.hasFunctionDecl() || I >= AC.getNumParams()) 3997 return; 3998 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>(); 3999 if (PS == nullptr) 4000 return; 4001 4002 const auto &Context = getContext(); 4003 auto SizeTy = Context.getSizeType(); 4004 auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy)); 4005 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?"); 4006 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T, 4007 EmittedArg.getScalarVal(), 4008 PS->isDynamic()); 4009 Args.add(RValue::get(V), SizeTy); 4010 // If we're emitting args in reverse, be sure to do so with 4011 // pass_object_size, as well. 4012 if (!LeftToRight) 4013 std::swap(Args.back(), *(&Args.back() - 1)); 4014 }; 4015 4016 // Insert a stack save if we're going to need any inalloca args. 4017 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { 4018 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 && 4019 "inalloca only supported on x86"); 4020 Args.allocateArgumentMemory(*this); 4021 } 4022 4023 // Evaluate each argument in the appropriate order. 4024 size_t CallArgsStart = Args.size(); 4025 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) { 4026 unsigned Idx = LeftToRight ? I : E - I - 1; 4027 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx; 4028 unsigned InitialArgSize = Args.size(); 4029 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of 4030 // the argument and parameter match or the objc method is parameterized. 4031 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) || 4032 getContext().hasSameUnqualifiedType((*Arg)->getType(), 4033 ArgTypes[Idx]) || 4034 (isa<ObjCMethodDecl>(AC.getDecl()) && 4035 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) && 4036 "Argument and parameter types don't match"); 4037 EmitCallArg(Args, *Arg, ArgTypes[Idx]); 4038 // In particular, we depend on it being the last arg in Args, and the 4039 // objectsize bits depend on there only being one arg if !LeftToRight. 4040 assert(InitialArgSize + 1 == Args.size() && 4041 "The code below depends on only adding one arg per EmitCallArg"); 4042 (void)InitialArgSize; 4043 // Since pointer argument are never emitted as LValue, it is safe to emit 4044 // non-null argument check for r-value only. 4045 if (!Args.back().hasLValue()) { 4046 RValue RVArg = Args.back().getKnownRValue(); 4047 EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC, 4048 ParamsToSkip + Idx); 4049 // @llvm.objectsize should never have side-effects and shouldn't need 4050 // destruction/cleanups, so we can safely "emit" it after its arg, 4051 // regardless of right-to-leftness 4052 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg); 4053 } 4054 } 4055 4056 if (!LeftToRight) { 4057 // Un-reverse the arguments we just evaluated so they match up with the LLVM 4058 // IR function. 4059 std::reverse(Args.begin() + CallArgsStart, Args.end()); 4060 } 4061 } 4062 4063 namespace { 4064 4065 struct DestroyUnpassedArg final : EHScopeStack::Cleanup { 4066 DestroyUnpassedArg(Address Addr, QualType Ty) 4067 : Addr(Addr), Ty(Ty) {} 4068 4069 Address Addr; 4070 QualType Ty; 4071 4072 void Emit(CodeGenFunction &CGF, Flags flags) override { 4073 QualType::DestructionKind DtorKind = Ty.isDestructedType(); 4074 if (DtorKind == QualType::DK_cxx_destructor) { 4075 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor(); 4076 assert(!Dtor->isTrivial()); 4077 CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false, 4078 /*Delegating=*/false, Addr, Ty); 4079 } else { 4080 CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty)); 4081 } 4082 } 4083 }; 4084 4085 struct DisableDebugLocationUpdates { 4086 CodeGenFunction &CGF; 4087 bool disabledDebugInfo; 4088 DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) { 4089 if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo())) 4090 CGF.disableDebugInfo(); 4091 } 4092 ~DisableDebugLocationUpdates() { 4093 if (disabledDebugInfo) 4094 CGF.enableDebugInfo(); 4095 } 4096 }; 4097 4098 } // end anonymous namespace 4099 4100 RValue CallArg::getRValue(CodeGenFunction &CGF) const { 4101 if (!HasLV) 4102 return RV; 4103 LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty); 4104 CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap, 4105 LV.isVolatile()); 4106 IsUsed = true; 4107 return RValue::getAggregate(Copy.getAddress(CGF)); 4108 } 4109 4110 void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const { 4111 LValue Dst = CGF.MakeAddrLValue(Addr, Ty); 4112 if (!HasLV && RV.isScalar()) 4113 CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true); 4114 else if (!HasLV && RV.isComplex()) 4115 CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true); 4116 else { 4117 auto Addr = HasLV ? LV.getAddress(CGF) : RV.getAggregateAddress(); 4118 LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty); 4119 // We assume that call args are never copied into subobjects. 4120 CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap, 4121 HasLV ? LV.isVolatileQualified() 4122 : RV.isVolatileQualified()); 4123 } 4124 IsUsed = true; 4125 } 4126 4127 void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, 4128 QualType type) { 4129 DisableDebugLocationUpdates Dis(*this, E); 4130 if (const ObjCIndirectCopyRestoreExpr *CRE 4131 = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) { 4132 assert(getLangOpts().ObjCAutoRefCount); 4133 return emitWritebackArg(*this, args, CRE); 4134 } 4135 4136 assert(type->isReferenceType() == E->isGLValue() && 4137 "reference binding to unmaterialized r-value!"); 4138 4139 if (E->isGLValue()) { 4140 assert(E->getObjectKind() == OK_Ordinary); 4141 return args.add(EmitReferenceBindingToExpr(E), type); 4142 } 4143 4144 bool HasAggregateEvalKind = hasAggregateEvaluationKind(type); 4145 4146 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee. 4147 // However, we still have to push an EH-only cleanup in case we unwind before 4148 // we make it to the call. 4149 if (HasAggregateEvalKind && 4150 type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) { 4151 // If we're using inalloca, use the argument memory. Otherwise, use a 4152 // temporary. 4153 AggValueSlot Slot; 4154 if (args.isUsingInAlloca()) 4155 Slot = createPlaceholderSlot(*this, type); 4156 else 4157 Slot = CreateAggTemp(type, "agg.tmp"); 4158 4159 bool DestroyedInCallee = true, NeedsEHCleanup = true; 4160 if (const auto *RD = type->getAsCXXRecordDecl()) 4161 DestroyedInCallee = RD->hasNonTrivialDestructor(); 4162 else 4163 NeedsEHCleanup = needsEHCleanup(type.isDestructedType()); 4164 4165 if (DestroyedInCallee) 4166 Slot.setExternallyDestructed(); 4167 4168 EmitAggExpr(E, Slot); 4169 RValue RV = Slot.asRValue(); 4170 args.add(RV, type); 4171 4172 if (DestroyedInCallee && NeedsEHCleanup) { 4173 // Create a no-op GEP between the placeholder and the cleanup so we can 4174 // RAUW it successfully. It also serves as a marker of the first 4175 // instruction where the cleanup is active. 4176 pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(), 4177 type); 4178 // This unreachable is a temporary marker which will be removed later. 4179 llvm::Instruction *IsActive = Builder.CreateUnreachable(); 4180 args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive); 4181 } 4182 return; 4183 } 4184 4185 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) && 4186 cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) { 4187 LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr()); 4188 assert(L.isSimple()); 4189 args.addUncopiedAggregate(L, type); 4190 return; 4191 } 4192 4193 args.add(EmitAnyExprToTemp(E), type); 4194 } 4195 4196 QualType CodeGenFunction::getVarArgType(const Expr *Arg) { 4197 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC 4198 // implicitly widens null pointer constants that are arguments to varargs 4199 // functions to pointer-sized ints. 4200 if (!getTarget().getTriple().isOSWindows()) 4201 return Arg->getType(); 4202 4203 if (Arg->getType()->isIntegerType() && 4204 getContext().getTypeSize(Arg->getType()) < 4205 getContext().getTargetInfo().getPointerWidth(0) && 4206 Arg->isNullPointerConstant(getContext(), 4207 Expr::NPC_ValueDependentIsNotNull)) { 4208 return getContext().getIntPtrType(); 4209 } 4210 4211 return Arg->getType(); 4212 } 4213 4214 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC 4215 // optimizer it can aggressively ignore unwind edges. 4216 void 4217 CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) { 4218 if (CGM.getCodeGenOpts().OptimizationLevel != 0 && 4219 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions) 4220 Inst->setMetadata("clang.arc.no_objc_arc_exceptions", 4221 CGM.getNoObjCARCExceptionsMetadata()); 4222 } 4223 4224 /// Emits a call to the given no-arguments nounwind runtime function. 4225 llvm::CallInst * 4226 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, 4227 const llvm::Twine &name) { 4228 return EmitNounwindRuntimeCall(callee, None, name); 4229 } 4230 4231 /// Emits a call to the given nounwind runtime function. 4232 llvm::CallInst * 4233 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, 4234 ArrayRef<llvm::Value *> args, 4235 const llvm::Twine &name) { 4236 llvm::CallInst *call = EmitRuntimeCall(callee, args, name); 4237 call->setDoesNotThrow(); 4238 return call; 4239 } 4240 4241 /// Emits a simple call (never an invoke) to the given no-arguments 4242 /// runtime function. 4243 llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee, 4244 const llvm::Twine &name) { 4245 return EmitRuntimeCall(callee, None, name); 4246 } 4247 4248 // Calls which may throw must have operand bundles indicating which funclet 4249 // they are nested within. 4250 SmallVector<llvm::OperandBundleDef, 1> 4251 CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) { 4252 SmallVector<llvm::OperandBundleDef, 1> BundleList; 4253 // There is no need for a funclet operand bundle if we aren't inside a 4254 // funclet. 4255 if (!CurrentFuncletPad) 4256 return BundleList; 4257 4258 // Skip intrinsics which cannot throw. 4259 auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts()); 4260 if (CalleeFn && CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) 4261 return BundleList; 4262 4263 BundleList.emplace_back("funclet", CurrentFuncletPad); 4264 return BundleList; 4265 } 4266 4267 /// Emits a simple call (never an invoke) to the given runtime function. 4268 llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee, 4269 ArrayRef<llvm::Value *> args, 4270 const llvm::Twine &name) { 4271 llvm::CallInst *call = Builder.CreateCall( 4272 callee, args, getBundlesForFunclet(callee.getCallee()), name); 4273 call->setCallingConv(getRuntimeCC()); 4274 return call; 4275 } 4276 4277 /// Emits a call or invoke to the given noreturn runtime function. 4278 void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke( 4279 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) { 4280 SmallVector<llvm::OperandBundleDef, 1> BundleList = 4281 getBundlesForFunclet(callee.getCallee()); 4282 4283 if (getInvokeDest()) { 4284 llvm::InvokeInst *invoke = 4285 Builder.CreateInvoke(callee, 4286 getUnreachableBlock(), 4287 getInvokeDest(), 4288 args, 4289 BundleList); 4290 invoke->setDoesNotReturn(); 4291 invoke->setCallingConv(getRuntimeCC()); 4292 } else { 4293 llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList); 4294 call->setDoesNotReturn(); 4295 call->setCallingConv(getRuntimeCC()); 4296 Builder.CreateUnreachable(); 4297 } 4298 } 4299 4300 /// Emits a call or invoke instruction to the given nullary runtime function. 4301 llvm::CallBase * 4302 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, 4303 const Twine &name) { 4304 return EmitRuntimeCallOrInvoke(callee, None, name); 4305 } 4306 4307 /// Emits a call or invoke instruction to the given runtime function. 4308 llvm::CallBase * 4309 CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, 4310 ArrayRef<llvm::Value *> args, 4311 const Twine &name) { 4312 llvm::CallBase *call = EmitCallOrInvoke(callee, args, name); 4313 call->setCallingConv(getRuntimeCC()); 4314 return call; 4315 } 4316 4317 /// Emits a call or invoke instruction to the given function, depending 4318 /// on the current state of the EH stack. 4319 llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee, 4320 ArrayRef<llvm::Value *> Args, 4321 const Twine &Name) { 4322 llvm::BasicBlock *InvokeDest = getInvokeDest(); 4323 SmallVector<llvm::OperandBundleDef, 1> BundleList = 4324 getBundlesForFunclet(Callee.getCallee()); 4325 4326 llvm::CallBase *Inst; 4327 if (!InvokeDest) 4328 Inst = Builder.CreateCall(Callee, Args, BundleList, Name); 4329 else { 4330 llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont"); 4331 Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList, 4332 Name); 4333 EmitBlock(ContBB); 4334 } 4335 4336 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC 4337 // optimizer it can aggressively ignore unwind edges. 4338 if (CGM.getLangOpts().ObjCAutoRefCount) 4339 AddObjCARCExceptionMetadata(Inst); 4340 4341 return Inst; 4342 } 4343 4344 void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old, 4345 llvm::Value *New) { 4346 DeferredReplacements.push_back(std::make_pair(Old, New)); 4347 } 4348 4349 namespace { 4350 4351 /// Specify given \p NewAlign as the alignment of return value attribute. If 4352 /// such attribute already exists, re-set it to the maximal one of two options. 4353 LLVM_NODISCARD llvm::AttributeList 4354 maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx, 4355 const llvm::AttributeList &Attrs, 4356 llvm::Align NewAlign) { 4357 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne(); 4358 if (CurAlign >= NewAlign) 4359 return Attrs; 4360 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign); 4361 return Attrs 4362 .removeAttribute(Ctx, llvm::AttributeList::ReturnIndex, 4363 llvm::Attribute::AttrKind::Alignment) 4364 .addAttribute(Ctx, llvm::AttributeList::ReturnIndex, AlignAttr); 4365 } 4366 4367 template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter { 4368 protected: 4369 CodeGenFunction &CGF; 4370 4371 /// We do nothing if this is, or becomes, nullptr. 4372 const AlignedAttrTy *AA = nullptr; 4373 4374 llvm::Value *Alignment = nullptr; // May or may not be a constant. 4375 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero. 4376 4377 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl) 4378 : CGF(CGF_) { 4379 if (!FuncDecl) 4380 return; 4381 AA = FuncDecl->getAttr<AlignedAttrTy>(); 4382 } 4383 4384 public: 4385 /// If we can, materialize the alignment as an attribute on return value. 4386 LLVM_NODISCARD llvm::AttributeList 4387 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) { 4388 if (!AA || OffsetCI || CGF.SanOpts.has(SanitizerKind::Alignment)) 4389 return Attrs; 4390 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment); 4391 if (!AlignmentCI) 4392 return Attrs; 4393 // We may legitimately have non-power-of-2 alignment here. 4394 // If so, this is UB land, emit it via `@llvm.assume` instead. 4395 if (!AlignmentCI->getValue().isPowerOf2()) 4396 return Attrs; 4397 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute( 4398 CGF.getLLVMContext(), Attrs, 4399 llvm::Align( 4400 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment))); 4401 AA = nullptr; // We're done. Disallow doing anything else. 4402 return NewAttrs; 4403 } 4404 4405 /// Emit alignment assumption. 4406 /// This is a general fallback that we take if either there is an offset, 4407 /// or the alignment is variable or we are sanitizing for alignment. 4408 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) { 4409 if (!AA) 4410 return; 4411 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, 4412 AA->getLocation(), Alignment, OffsetCI); 4413 AA = nullptr; // We're done. Disallow doing anything else. 4414 } 4415 }; 4416 4417 /// Helper data structure to emit `AssumeAlignedAttr`. 4418 class AssumeAlignedAttrEmitter final 4419 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> { 4420 public: 4421 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl) 4422 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) { 4423 if (!AA) 4424 return; 4425 // It is guaranteed that the alignment/offset are constants. 4426 Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment())); 4427 if (Expr *Offset = AA->getOffset()) { 4428 OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset)); 4429 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset. 4430 OffsetCI = nullptr; 4431 } 4432 } 4433 }; 4434 4435 /// Helper data structure to emit `AllocAlignAttr`. 4436 class AllocAlignAttrEmitter final 4437 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> { 4438 public: 4439 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl, 4440 const CallArgList &CallArgs) 4441 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) { 4442 if (!AA) 4443 return; 4444 // Alignment may or may not be a constant, and that is okay. 4445 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()] 4446 .getRValue(CGF) 4447 .getScalarVal(); 4448 } 4449 }; 4450 4451 } // namespace 4452 4453 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, 4454 const CGCallee &Callee, 4455 ReturnValueSlot ReturnValue, 4456 const CallArgList &CallArgs, 4457 llvm::CallBase **callOrInvoke, 4458 SourceLocation Loc) { 4459 // FIXME: We no longer need the types from CallArgs; lift up and simplify. 4460 4461 assert(Callee.isOrdinary() || Callee.isVirtual()); 4462 4463 // Handle struct-return functions by passing a pointer to the 4464 // location that we would like to return into. 4465 QualType RetTy = CallInfo.getReturnType(); 4466 const ABIArgInfo &RetAI = CallInfo.getReturnInfo(); 4467 4468 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo); 4469 4470 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl(); 4471 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) { 4472 // We can only guarantee that a function is called from the correct 4473 // context/function based on the appropriate target attributes, 4474 // so only check in the case where we have both always_inline and target 4475 // since otherwise we could be making a conditional call after a check for 4476 // the proper cpu features (and it won't cause code generation issues due to 4477 // function based code generation). 4478 if (TargetDecl->hasAttr<AlwaysInlineAttr>() && 4479 TargetDecl->hasAttr<TargetAttr>()) 4480 checkTargetFeatures(Loc, FD); 4481 4482 // Some architectures (such as x86-64) have the ABI changed based on 4483 // attribute-target/features. Give them a chance to diagnose. 4484 CGM.getTargetCodeGenInfo().checkFunctionCallABI( 4485 CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs); 4486 } 4487 4488 #ifndef NDEBUG 4489 if (!(CallInfo.isVariadic() && CallInfo.getArgStruct())) { 4490 // For an inalloca varargs function, we don't expect CallInfo to match the 4491 // function pointer's type, because the inalloca struct a will have extra 4492 // fields in it for the varargs parameters. Code later in this function 4493 // bitcasts the function pointer to the type derived from CallInfo. 4494 // 4495 // In other cases, we assert that the types match up (until pointers stop 4496 // having pointee types). 4497 llvm::Type *TypeFromVal; 4498 if (Callee.isVirtual()) 4499 TypeFromVal = Callee.getVirtualFunctionType(); 4500 else 4501 TypeFromVal = 4502 Callee.getFunctionPointer()->getType()->getPointerElementType(); 4503 assert(IRFuncTy == TypeFromVal); 4504 } 4505 #endif 4506 4507 // 1. Set up the arguments. 4508 4509 // If we're using inalloca, insert the allocation after the stack save. 4510 // FIXME: Do this earlier rather than hacking it in here! 4511 Address ArgMemory = Address::invalid(); 4512 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { 4513 const llvm::DataLayout &DL = CGM.getDataLayout(); 4514 llvm::Instruction *IP = CallArgs.getStackBase(); 4515 llvm::AllocaInst *AI; 4516 if (IP) { 4517 IP = IP->getNextNode(); 4518 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), 4519 "argmem", IP); 4520 } else { 4521 AI = CreateTempAlloca(ArgStruct, "argmem"); 4522 } 4523 auto Align = CallInfo.getArgStructAlignment(); 4524 AI->setAlignment(Align.getAsAlign()); 4525 AI->setUsedWithInAlloca(true); 4526 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); 4527 ArgMemory = Address(AI, Align); 4528 } 4529 4530 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); 4531 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs()); 4532 4533 // If the call returns a temporary with struct return, create a temporary 4534 // alloca to hold the result, unless one is given to us. 4535 Address SRetPtr = Address::invalid(); 4536 Address SRetAlloca = Address::invalid(); 4537 llvm::Value *UnusedReturnSizePtr = nullptr; 4538 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) { 4539 if (!ReturnValue.isNull()) { 4540 SRetPtr = ReturnValue.getValue(); 4541 } else { 4542 SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca); 4543 if (HaveInsertPoint() && ReturnValue.isUnused()) { 4544 uint64_t size = 4545 CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy)); 4546 UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer()); 4547 } 4548 } 4549 if (IRFunctionArgs.hasSRetArg()) { 4550 IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer(); 4551 } else if (RetAI.isInAlloca()) { 4552 Address Addr = 4553 Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex()); 4554 Builder.CreateStore(SRetPtr.getPointer(), Addr); 4555 } 4556 } 4557 4558 Address swiftErrorTemp = Address::invalid(); 4559 Address swiftErrorArg = Address::invalid(); 4560 4561 // When passing arguments using temporary allocas, we need to add the 4562 // appropriate lifetime markers. This vector keeps track of all the lifetime 4563 // markers that need to be ended right after the call. 4564 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall; 4565 4566 // Translate all of the arguments as necessary to match the IR lowering. 4567 assert(CallInfo.arg_size() == CallArgs.size() && 4568 "Mismatch between function signature & arguments."); 4569 unsigned ArgNo = 0; 4570 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin(); 4571 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end(); 4572 I != E; ++I, ++info_it, ++ArgNo) { 4573 const ABIArgInfo &ArgInfo = info_it->info; 4574 4575 // Insert a padding argument to ensure proper alignment. 4576 if (IRFunctionArgs.hasPaddingArg(ArgNo)) 4577 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] = 4578 llvm::UndefValue::get(ArgInfo.getPaddingType()); 4579 4580 unsigned FirstIRArg, NumIRArgs; 4581 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo); 4582 4583 switch (ArgInfo.getKind()) { 4584 case ABIArgInfo::InAlloca: { 4585 assert(NumIRArgs == 0); 4586 assert(getTarget().getTriple().getArch() == llvm::Triple::x86); 4587 if (I->isAggregate()) { 4588 Address Addr = I->hasLValue() 4589 ? I->getKnownLValue().getAddress(*this) 4590 : I->getKnownRValue().getAggregateAddress(); 4591 llvm::Instruction *Placeholder = 4592 cast<llvm::Instruction>(Addr.getPointer()); 4593 4594 if (!ArgInfo.getInAllocaIndirect()) { 4595 // Replace the placeholder with the appropriate argument slot GEP. 4596 CGBuilderTy::InsertPoint IP = Builder.saveIP(); 4597 Builder.SetInsertPoint(Placeholder); 4598 Addr = Builder.CreateStructGEP(ArgMemory, 4599 ArgInfo.getInAllocaFieldIndex()); 4600 Builder.restoreIP(IP); 4601 } else { 4602 // For indirect things such as overaligned structs, replace the 4603 // placeholder with a regular aggregate temporary alloca. Store the 4604 // address of this alloca into the struct. 4605 Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp"); 4606 Address ArgSlot = Builder.CreateStructGEP( 4607 ArgMemory, ArgInfo.getInAllocaFieldIndex()); 4608 Builder.CreateStore(Addr.getPointer(), ArgSlot); 4609 } 4610 deferPlaceholderReplacement(Placeholder, Addr.getPointer()); 4611 } else if (ArgInfo.getInAllocaIndirect()) { 4612 // Make a temporary alloca and store the address of it into the argument 4613 // struct. 4614 Address Addr = CreateMemTempWithoutCast( 4615 I->Ty, getContext().getTypeAlignInChars(I->Ty), 4616 "indirect-arg-temp"); 4617 I->copyInto(*this, Addr); 4618 Address ArgSlot = 4619 Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex()); 4620 Builder.CreateStore(Addr.getPointer(), ArgSlot); 4621 } else { 4622 // Store the RValue into the argument struct. 4623 Address Addr = 4624 Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex()); 4625 unsigned AS = Addr.getType()->getPointerAddressSpace(); 4626 llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS); 4627 // There are some cases where a trivial bitcast is not avoidable. The 4628 // definition of a type later in a translation unit may change it's type 4629 // from {}* to (%struct.foo*)*. 4630 if (Addr.getType() != MemType) 4631 Addr = Builder.CreateBitCast(Addr, MemType); 4632 I->copyInto(*this, Addr); 4633 } 4634 break; 4635 } 4636 4637 case ABIArgInfo::Indirect: 4638 case ABIArgInfo::IndirectAliased: { 4639 assert(NumIRArgs == 1); 4640 if (!I->isAggregate()) { 4641 // Make a temporary alloca to pass the argument. 4642 Address Addr = CreateMemTempWithoutCast( 4643 I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp"); 4644 IRCallArgs[FirstIRArg] = Addr.getPointer(); 4645 4646 I->copyInto(*this, Addr); 4647 } else { 4648 // We want to avoid creating an unnecessary temporary+copy here; 4649 // however, we need one in three cases: 4650 // 1. If the argument is not byval, and we are required to copy the 4651 // source. (This case doesn't occur on any common architecture.) 4652 // 2. If the argument is byval, RV is not sufficiently aligned, and 4653 // we cannot force it to be sufficiently aligned. 4654 // 3. If the argument is byval, but RV is not located in default 4655 // or alloca address space. 4656 Address Addr = I->hasLValue() 4657 ? I->getKnownLValue().getAddress(*this) 4658 : I->getKnownRValue().getAggregateAddress(); 4659 llvm::Value *V = Addr.getPointer(); 4660 CharUnits Align = ArgInfo.getIndirectAlign(); 4661 const llvm::DataLayout *TD = &CGM.getDataLayout(); 4662 4663 assert((FirstIRArg >= IRFuncTy->getNumParams() || 4664 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() == 4665 TD->getAllocaAddrSpace()) && 4666 "indirect argument must be in alloca address space"); 4667 4668 bool NeedCopy = false; 4669 4670 if (Addr.getAlignment() < Align && 4671 llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) < 4672 Align.getAsAlign()) { 4673 NeedCopy = true; 4674 } else if (I->hasLValue()) { 4675 auto LV = I->getKnownLValue(); 4676 auto AS = LV.getAddressSpace(); 4677 4678 if (!ArgInfo.getIndirectByVal() || 4679 (LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) { 4680 NeedCopy = true; 4681 } 4682 if (!getLangOpts().OpenCL) { 4683 if ((ArgInfo.getIndirectByVal() && 4684 (AS != LangAS::Default && 4685 AS != CGM.getASTAllocaAddressSpace()))) { 4686 NeedCopy = true; 4687 } 4688 } 4689 // For OpenCL even if RV is located in default or alloca address space 4690 // we don't want to perform address space cast for it. 4691 else if ((ArgInfo.getIndirectByVal() && 4692 Addr.getType()->getAddressSpace() != IRFuncTy-> 4693 getParamType(FirstIRArg)->getPointerAddressSpace())) { 4694 NeedCopy = true; 4695 } 4696 } 4697 4698 if (NeedCopy) { 4699 // Create an aligned temporary, and copy to it. 4700 Address AI = CreateMemTempWithoutCast( 4701 I->Ty, ArgInfo.getIndirectAlign(), "byval-temp"); 4702 IRCallArgs[FirstIRArg] = AI.getPointer(); 4703 4704 // Emit lifetime markers for the temporary alloca. 4705 uint64_t ByvalTempElementSize = 4706 CGM.getDataLayout().getTypeAllocSize(AI.getElementType()); 4707 llvm::Value *LifetimeSize = 4708 EmitLifetimeStart(ByvalTempElementSize, AI.getPointer()); 4709 4710 // Add cleanup code to emit the end lifetime marker after the call. 4711 if (LifetimeSize) // In case we disabled lifetime markers. 4712 CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize); 4713 4714 // Generate the copy. 4715 I->copyInto(*this, AI); 4716 } else { 4717 // Skip the extra memcpy call. 4718 auto *T = V->getType()->getPointerElementType()->getPointerTo( 4719 CGM.getDataLayout().getAllocaAddrSpace()); 4720 IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast( 4721 *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T, 4722 true); 4723 } 4724 } 4725 break; 4726 } 4727 4728 case ABIArgInfo::Ignore: 4729 assert(NumIRArgs == 0); 4730 break; 4731 4732 case ABIArgInfo::Extend: 4733 case ABIArgInfo::Direct: { 4734 if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) && 4735 ArgInfo.getCoerceToType() == ConvertType(info_it->type) && 4736 ArgInfo.getDirectOffset() == 0) { 4737 assert(NumIRArgs == 1); 4738 llvm::Value *V; 4739 if (!I->isAggregate()) 4740 V = I->getKnownRValue().getScalarVal(); 4741 else 4742 V = Builder.CreateLoad( 4743 I->hasLValue() ? I->getKnownLValue().getAddress(*this) 4744 : I->getKnownRValue().getAggregateAddress()); 4745 4746 // Implement swifterror by copying into a new swifterror argument. 4747 // We'll write back in the normal path out of the call. 4748 if (CallInfo.getExtParameterInfo(ArgNo).getABI() 4749 == ParameterABI::SwiftErrorResult) { 4750 assert(!swiftErrorTemp.isValid() && "multiple swifterror args"); 4751 4752 QualType pointeeTy = I->Ty->getPointeeType(); 4753 swiftErrorArg = 4754 Address(V, getContext().getTypeAlignInChars(pointeeTy)); 4755 4756 swiftErrorTemp = 4757 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); 4758 V = swiftErrorTemp.getPointer(); 4759 cast<llvm::AllocaInst>(V)->setSwiftError(true); 4760 4761 llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg); 4762 Builder.CreateStore(errorValue, swiftErrorTemp); 4763 } 4764 4765 // We might have to widen integers, but we should never truncate. 4766 if (ArgInfo.getCoerceToType() != V->getType() && 4767 V->getType()->isIntegerTy()) 4768 V = Builder.CreateZExt(V, ArgInfo.getCoerceToType()); 4769 4770 // If the argument doesn't match, perform a bitcast to coerce it. This 4771 // can happen due to trivial type mismatches. 4772 if (FirstIRArg < IRFuncTy->getNumParams() && 4773 V->getType() != IRFuncTy->getParamType(FirstIRArg)) 4774 V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg)); 4775 4776 IRCallArgs[FirstIRArg] = V; 4777 break; 4778 } 4779 4780 // FIXME: Avoid the conversion through memory if possible. 4781 Address Src = Address::invalid(); 4782 if (!I->isAggregate()) { 4783 Src = CreateMemTemp(I->Ty, "coerce"); 4784 I->copyInto(*this, Src); 4785 } else { 4786 Src = I->hasLValue() ? I->getKnownLValue().getAddress(*this) 4787 : I->getKnownRValue().getAggregateAddress(); 4788 } 4789 4790 // If the value is offset in memory, apply the offset now. 4791 Src = emitAddressAtOffset(*this, Src, ArgInfo); 4792 4793 // Fast-isel and the optimizer generally like scalar values better than 4794 // FCAs, so we flatten them if this is safe to do for this argument. 4795 llvm::StructType *STy = 4796 dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType()); 4797 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) { 4798 llvm::Type *SrcTy = Src.getElementType(); 4799 uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy); 4800 uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy); 4801 4802 // If the source type is smaller than the destination type of the 4803 // coerce-to logic, copy the source value into a temp alloca the size 4804 // of the destination type to allow loading all of it. The bits past 4805 // the source value are left undef. 4806 if (SrcSize < DstSize) { 4807 Address TempAlloca 4808 = CreateTempAlloca(STy, Src.getAlignment(), 4809 Src.getName() + ".coerce"); 4810 Builder.CreateMemCpy(TempAlloca, Src, SrcSize); 4811 Src = TempAlloca; 4812 } else { 4813 Src = Builder.CreateBitCast(Src, 4814 STy->getPointerTo(Src.getAddressSpace())); 4815 } 4816 4817 assert(NumIRArgs == STy->getNumElements()); 4818 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { 4819 Address EltPtr = Builder.CreateStructGEP(Src, i); 4820 llvm::Value *LI = Builder.CreateLoad(EltPtr); 4821 IRCallArgs[FirstIRArg + i] = LI; 4822 } 4823 } else { 4824 // In the simple case, just pass the coerced loaded value. 4825 assert(NumIRArgs == 1); 4826 llvm::Value *Load = 4827 CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this); 4828 4829 if (CallInfo.isCmseNSCall()) { 4830 // For certain parameter types, clear padding bits, as they may reveal 4831 // sensitive information. 4832 // Small struct/union types are passed as integer arrays. 4833 auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType()); 4834 if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType())) 4835 Load = EmitCMSEClearRecord(Load, ATy, I->Ty); 4836 } 4837 IRCallArgs[FirstIRArg] = Load; 4838 } 4839 4840 break; 4841 } 4842 4843 case ABIArgInfo::CoerceAndExpand: { 4844 auto coercionType = ArgInfo.getCoerceAndExpandType(); 4845 auto layout = CGM.getDataLayout().getStructLayout(coercionType); 4846 4847 llvm::Value *tempSize = nullptr; 4848 Address addr = Address::invalid(); 4849 Address AllocaAddr = Address::invalid(); 4850 if (I->isAggregate()) { 4851 addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this) 4852 : I->getKnownRValue().getAggregateAddress(); 4853 4854 } else { 4855 RValue RV = I->getKnownRValue(); 4856 assert(RV.isScalar()); // complex should always just be direct 4857 4858 llvm::Type *scalarType = RV.getScalarVal()->getType(); 4859 auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType); 4860 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType); 4861 4862 // Materialize to a temporary. 4863 addr = CreateTempAlloca( 4864 RV.getScalarVal()->getType(), 4865 CharUnits::fromQuantity(std::max( 4866 (unsigned)layout->getAlignment().value(), scalarAlign)), 4867 "tmp", 4868 /*ArraySize=*/nullptr, &AllocaAddr); 4869 tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer()); 4870 4871 Builder.CreateStore(RV.getScalarVal(), addr); 4872 } 4873 4874 addr = Builder.CreateElementBitCast(addr, coercionType); 4875 4876 unsigned IRArgPos = FirstIRArg; 4877 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) { 4878 llvm::Type *eltType = coercionType->getElementType(i); 4879 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue; 4880 Address eltAddr = Builder.CreateStructGEP(addr, i); 4881 llvm::Value *elt = Builder.CreateLoad(eltAddr); 4882 IRCallArgs[IRArgPos++] = elt; 4883 } 4884 assert(IRArgPos == FirstIRArg + NumIRArgs); 4885 4886 if (tempSize) { 4887 EmitLifetimeEnd(tempSize, AllocaAddr.getPointer()); 4888 } 4889 4890 break; 4891 } 4892 4893 case ABIArgInfo::Expand: { 4894 unsigned IRArgPos = FirstIRArg; 4895 ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos); 4896 assert(IRArgPos == FirstIRArg + NumIRArgs); 4897 break; 4898 } 4899 } 4900 } 4901 4902 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this); 4903 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer(); 4904 4905 // If we're using inalloca, set up that argument. 4906 if (ArgMemory.isValid()) { 4907 llvm::Value *Arg = ArgMemory.getPointer(); 4908 if (CallInfo.isVariadic()) { 4909 // When passing non-POD arguments by value to variadic functions, we will 4910 // end up with a variadic prototype and an inalloca call site. In such 4911 // cases, we can't do any parameter mismatch checks. Give up and bitcast 4912 // the callee. 4913 unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace(); 4914 CalleePtr = 4915 Builder.CreateBitCast(CalleePtr, IRFuncTy->getPointerTo(CalleeAS)); 4916 } else { 4917 llvm::Type *LastParamTy = 4918 IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1); 4919 if (Arg->getType() != LastParamTy) { 4920 #ifndef NDEBUG 4921 // Assert that these structs have equivalent element types. 4922 llvm::StructType *FullTy = CallInfo.getArgStruct(); 4923 llvm::StructType *DeclaredTy = cast<llvm::StructType>( 4924 cast<llvm::PointerType>(LastParamTy)->getElementType()); 4925 assert(DeclaredTy->getNumElements() == FullTy->getNumElements()); 4926 for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(), 4927 DE = DeclaredTy->element_end(), 4928 FI = FullTy->element_begin(); 4929 DI != DE; ++DI, ++FI) 4930 assert(*DI == *FI); 4931 #endif 4932 Arg = Builder.CreateBitCast(Arg, LastParamTy); 4933 } 4934 } 4935 assert(IRFunctionArgs.hasInallocaArg()); 4936 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg; 4937 } 4938 4939 // 2. Prepare the function pointer. 4940 4941 // If the callee is a bitcast of a non-variadic function to have a 4942 // variadic function pointer type, check to see if we can remove the 4943 // bitcast. This comes up with unprototyped functions. 4944 // 4945 // This makes the IR nicer, but more importantly it ensures that we 4946 // can inline the function at -O0 if it is marked always_inline. 4947 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT, 4948 llvm::Value *Ptr) -> llvm::Function * { 4949 if (!CalleeFT->isVarArg()) 4950 return nullptr; 4951 4952 // Get underlying value if it's a bitcast 4953 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) { 4954 if (CE->getOpcode() == llvm::Instruction::BitCast) 4955 Ptr = CE->getOperand(0); 4956 } 4957 4958 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr); 4959 if (!OrigFn) 4960 return nullptr; 4961 4962 llvm::FunctionType *OrigFT = OrigFn->getFunctionType(); 4963 4964 // If the original type is variadic, or if any of the component types 4965 // disagree, we cannot remove the cast. 4966 if (OrigFT->isVarArg() || 4967 OrigFT->getNumParams() != CalleeFT->getNumParams() || 4968 OrigFT->getReturnType() != CalleeFT->getReturnType()) 4969 return nullptr; 4970 4971 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i) 4972 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i)) 4973 return nullptr; 4974 4975 return OrigFn; 4976 }; 4977 4978 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) { 4979 CalleePtr = OrigFn; 4980 IRFuncTy = OrigFn->getFunctionType(); 4981 } 4982 4983 // 3. Perform the actual call. 4984 4985 // Deactivate any cleanups that we're supposed to do immediately before 4986 // the call. 4987 if (!CallArgs.getCleanupsToDeactivate().empty()) 4988 deactivateArgCleanupsBeforeCall(*this, CallArgs); 4989 4990 // Assert that the arguments we computed match up. The IR verifier 4991 // will catch this, but this is a common enough source of problems 4992 // during IRGen changes that it's way better for debugging to catch 4993 // it ourselves here. 4994 #ifndef NDEBUG 4995 assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg()); 4996 for (unsigned i = 0; i < IRCallArgs.size(); ++i) { 4997 // Inalloca argument can have different type. 4998 if (IRFunctionArgs.hasInallocaArg() && 4999 i == IRFunctionArgs.getInallocaArgNo()) 5000 continue; 5001 if (i < IRFuncTy->getNumParams()) 5002 assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i)); 5003 } 5004 #endif 5005 5006 // Update the largest vector width if any arguments have vector types. 5007 for (unsigned i = 0; i < IRCallArgs.size(); ++i) { 5008 if (auto *VT = dyn_cast<llvm::VectorType>(IRCallArgs[i]->getType())) 5009 LargestVectorWidth = 5010 std::max((uint64_t)LargestVectorWidth, 5011 VT->getPrimitiveSizeInBits().getKnownMinSize()); 5012 } 5013 5014 // Compute the calling convention and attributes. 5015 unsigned CallingConv; 5016 llvm::AttributeList Attrs; 5017 CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo, 5018 Callee.getAbstractInfo(), Attrs, CallingConv, 5019 /*AttrOnCallSite=*/true); 5020 5021 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) 5022 if (FD->hasAttr<StrictFPAttr>()) 5023 // All calls within a strictfp function are marked strictfp 5024 Attrs = 5025 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex, 5026 llvm::Attribute::StrictFP); 5027 5028 // Add call-site nomerge attribute if exists. 5029 if (InNoMergeAttributedStmt) 5030 Attrs = 5031 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex, 5032 llvm::Attribute::NoMerge); 5033 5034 // Apply some call-site-specific attributes. 5035 // TODO: work this into building the attribute set. 5036 5037 // Apply always_inline to all calls within flatten functions. 5038 // FIXME: should this really take priority over __try, below? 5039 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() && 5040 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>())) { 5041 Attrs = 5042 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex, 5043 llvm::Attribute::AlwaysInline); 5044 } 5045 5046 // Disable inlining inside SEH __try blocks. 5047 if (isSEHTryScope()) { 5048 Attrs = 5049 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex, 5050 llvm::Attribute::NoInline); 5051 } 5052 5053 // Decide whether to use a call or an invoke. 5054 bool CannotThrow; 5055 if (currentFunctionUsesSEHTry()) { 5056 // SEH cares about asynchronous exceptions, so everything can "throw." 5057 CannotThrow = false; 5058 } else if (isCleanupPadScope() && 5059 EHPersonality::get(*this).isMSVCXXPersonality()) { 5060 // The MSVC++ personality will implicitly terminate the program if an 5061 // exception is thrown during a cleanup outside of a try/catch. 5062 // We don't need to model anything in IR to get this behavior. 5063 CannotThrow = true; 5064 } else { 5065 // Otherwise, nounwind call sites will never throw. 5066 CannotThrow = Attrs.hasFnAttribute(llvm::Attribute::NoUnwind); 5067 5068 if (auto *FPtr = dyn_cast<llvm::Function>(CalleePtr)) 5069 if (FPtr->hasFnAttribute(llvm::Attribute::NoUnwind)) 5070 CannotThrow = true; 5071 } 5072 5073 // If we made a temporary, be sure to clean up after ourselves. Note that we 5074 // can't depend on being inside of an ExprWithCleanups, so we need to manually 5075 // pop this cleanup later on. Being eager about this is OK, since this 5076 // temporary is 'invisible' outside of the callee. 5077 if (UnusedReturnSizePtr) 5078 pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetAlloca, 5079 UnusedReturnSizePtr); 5080 5081 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest(); 5082 5083 SmallVector<llvm::OperandBundleDef, 1> BundleList = 5084 getBundlesForFunclet(CalleePtr); 5085 5086 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) 5087 if (FD->hasAttr<StrictFPAttr>()) 5088 // All calls within a strictfp function are marked strictfp 5089 Attrs = 5090 Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex, 5091 llvm::Attribute::StrictFP); 5092 5093 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl); 5094 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs); 5095 5096 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs); 5097 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs); 5098 5099 // Emit the actual call/invoke instruction. 5100 llvm::CallBase *CI; 5101 if (!InvokeDest) { 5102 CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList); 5103 } else { 5104 llvm::BasicBlock *Cont = createBasicBlock("invoke.cont"); 5105 CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs, 5106 BundleList); 5107 EmitBlock(Cont); 5108 } 5109 if (callOrInvoke) 5110 *callOrInvoke = CI; 5111 5112 // If this is within a function that has the guard(nocf) attribute and is an 5113 // indirect call, add the "guard_nocf" attribute to this call to indicate that 5114 // Control Flow Guard checks should not be added, even if the call is inlined. 5115 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) { 5116 if (const auto *A = FD->getAttr<CFGuardAttr>()) { 5117 if (A->getGuard() == CFGuardAttr::GuardArg::nocf && !CI->getCalledFunction()) 5118 Attrs = Attrs.addAttribute( 5119 getLLVMContext(), llvm::AttributeList::FunctionIndex, "guard_nocf"); 5120 } 5121 } 5122 5123 // Apply the attributes and calling convention. 5124 CI->setAttributes(Attrs); 5125 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv)); 5126 5127 // Apply various metadata. 5128 5129 if (!CI->getType()->isVoidTy()) 5130 CI->setName("call"); 5131 5132 // Update largest vector width from the return type. 5133 if (auto *VT = dyn_cast<llvm::VectorType>(CI->getType())) 5134 LargestVectorWidth = 5135 std::max((uint64_t)LargestVectorWidth, 5136 VT->getPrimitiveSizeInBits().getKnownMinSize()); 5137 5138 // Insert instrumentation or attach profile metadata at indirect call sites. 5139 // For more details, see the comment before the definition of 5140 // IPVK_IndirectCallTarget in InstrProfData.inc. 5141 if (!CI->getCalledFunction()) 5142 PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget, 5143 CI, CalleePtr); 5144 5145 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC 5146 // optimizer it can aggressively ignore unwind edges. 5147 if (CGM.getLangOpts().ObjCAutoRefCount) 5148 AddObjCARCExceptionMetadata(CI); 5149 5150 // Suppress tail calls if requested. 5151 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) { 5152 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>()) 5153 Call->setTailCallKind(llvm::CallInst::TCK_NoTail); 5154 } 5155 5156 // Add metadata for calls to MSAllocator functions 5157 if (getDebugInfo() && TargetDecl && 5158 TargetDecl->hasAttr<MSAllocatorAttr>()) 5159 getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy->getPointeeType(), Loc); 5160 5161 // 4. Finish the call. 5162 5163 // If the call doesn't return, finish the basic block and clear the 5164 // insertion point; this allows the rest of IRGen to discard 5165 // unreachable code. 5166 if (CI->doesNotReturn()) { 5167 if (UnusedReturnSizePtr) 5168 PopCleanupBlock(); 5169 5170 // Strip away the noreturn attribute to better diagnose unreachable UB. 5171 if (SanOpts.has(SanitizerKind::Unreachable)) { 5172 // Also remove from function since CallBase::hasFnAttr additionally checks 5173 // attributes of the called function. 5174 if (auto *F = CI->getCalledFunction()) 5175 F->removeFnAttr(llvm::Attribute::NoReturn); 5176 CI->removeAttribute(llvm::AttributeList::FunctionIndex, 5177 llvm::Attribute::NoReturn); 5178 5179 // Avoid incompatibility with ASan which relies on the `noreturn` 5180 // attribute to insert handler calls. 5181 if (SanOpts.hasOneOf(SanitizerKind::Address | 5182 SanitizerKind::KernelAddress)) { 5183 SanitizerScope SanScope(this); 5184 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder); 5185 Builder.SetInsertPoint(CI); 5186 auto *FnType = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false); 5187 llvm::FunctionCallee Fn = 5188 CGM.CreateRuntimeFunction(FnType, "__asan_handle_no_return"); 5189 EmitNounwindRuntimeCall(Fn); 5190 } 5191 } 5192 5193 EmitUnreachable(Loc); 5194 Builder.ClearInsertionPoint(); 5195 5196 // FIXME: For now, emit a dummy basic block because expr emitters in 5197 // generally are not ready to handle emitting expressions at unreachable 5198 // points. 5199 EnsureInsertPoint(); 5200 5201 // Return a reasonable RValue. 5202 return GetUndefRValue(RetTy); 5203 } 5204 5205 // Perform the swifterror writeback. 5206 if (swiftErrorTemp.isValid()) { 5207 llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp); 5208 Builder.CreateStore(errorResult, swiftErrorArg); 5209 } 5210 5211 // Emit any call-associated writebacks immediately. Arguably this 5212 // should happen after any return-value munging. 5213 if (CallArgs.hasWritebacks()) 5214 emitWritebacks(*this, CallArgs); 5215 5216 // The stack cleanup for inalloca arguments has to run out of the normal 5217 // lexical order, so deactivate it and run it manually here. 5218 CallArgs.freeArgumentMemory(*this); 5219 5220 // Extract the return value. 5221 RValue Ret = [&] { 5222 switch (RetAI.getKind()) { 5223 case ABIArgInfo::CoerceAndExpand: { 5224 auto coercionType = RetAI.getCoerceAndExpandType(); 5225 5226 Address addr = SRetPtr; 5227 addr = Builder.CreateElementBitCast(addr, coercionType); 5228 5229 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType()); 5230 bool requiresExtract = isa<llvm::StructType>(CI->getType()); 5231 5232 unsigned unpaddedIndex = 0; 5233 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) { 5234 llvm::Type *eltType = coercionType->getElementType(i); 5235 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) continue; 5236 Address eltAddr = Builder.CreateStructGEP(addr, i); 5237 llvm::Value *elt = CI; 5238 if (requiresExtract) 5239 elt = Builder.CreateExtractValue(elt, unpaddedIndex++); 5240 else 5241 assert(unpaddedIndex == 0); 5242 Builder.CreateStore(elt, eltAddr); 5243 } 5244 // FALLTHROUGH 5245 LLVM_FALLTHROUGH; 5246 } 5247 5248 case ABIArgInfo::InAlloca: 5249 case ABIArgInfo::Indirect: { 5250 RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation()); 5251 if (UnusedReturnSizePtr) 5252 PopCleanupBlock(); 5253 return ret; 5254 } 5255 5256 case ABIArgInfo::Ignore: 5257 // If we are ignoring an argument that had a result, make sure to 5258 // construct the appropriate return value for our caller. 5259 return GetUndefRValue(RetTy); 5260 5261 case ABIArgInfo::Extend: 5262 case ABIArgInfo::Direct: { 5263 llvm::Type *RetIRTy = ConvertType(RetTy); 5264 if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) { 5265 switch (getEvaluationKind(RetTy)) { 5266 case TEK_Complex: { 5267 llvm::Value *Real = Builder.CreateExtractValue(CI, 0); 5268 llvm::Value *Imag = Builder.CreateExtractValue(CI, 1); 5269 return RValue::getComplex(std::make_pair(Real, Imag)); 5270 } 5271 case TEK_Aggregate: { 5272 Address DestPtr = ReturnValue.getValue(); 5273 bool DestIsVolatile = ReturnValue.isVolatile(); 5274 5275 if (!DestPtr.isValid()) { 5276 DestPtr = CreateMemTemp(RetTy, "agg.tmp"); 5277 DestIsVolatile = false; 5278 } 5279 EmitAggregateStore(CI, DestPtr, DestIsVolatile); 5280 return RValue::getAggregate(DestPtr); 5281 } 5282 case TEK_Scalar: { 5283 // If the argument doesn't match, perform a bitcast to coerce it. This 5284 // can happen due to trivial type mismatches. 5285 llvm::Value *V = CI; 5286 if (V->getType() != RetIRTy) 5287 V = Builder.CreateBitCast(V, RetIRTy); 5288 return RValue::get(V); 5289 } 5290 } 5291 llvm_unreachable("bad evaluation kind"); 5292 } 5293 5294 Address DestPtr = ReturnValue.getValue(); 5295 bool DestIsVolatile = ReturnValue.isVolatile(); 5296 5297 if (!DestPtr.isValid()) { 5298 DestPtr = CreateMemTemp(RetTy, "coerce"); 5299 DestIsVolatile = false; 5300 } 5301 5302 // If the value is offset in memory, apply the offset now. 5303 Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI); 5304 CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this); 5305 5306 return convertTempToRValue(DestPtr, RetTy, SourceLocation()); 5307 } 5308 5309 case ABIArgInfo::Expand: 5310 case ABIArgInfo::IndirectAliased: 5311 llvm_unreachable("Invalid ABI kind for return argument"); 5312 } 5313 5314 llvm_unreachable("Unhandled ABIArgInfo::Kind"); 5315 } (); 5316 5317 // Emit the assume_aligned check on the return value. 5318 if (Ret.isScalar() && TargetDecl) { 5319 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret); 5320 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret); 5321 } 5322 5323 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though 5324 // we can't use the full cleanup mechanism. 5325 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall) 5326 LifetimeEnd.Emit(*this, /*Flags=*/{}); 5327 5328 if (!ReturnValue.isExternallyDestructed() && 5329 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct) 5330 pushDestroy(QualType::DK_nontrivial_c_struct, Ret.getAggregateAddress(), 5331 RetTy); 5332 5333 return Ret; 5334 } 5335 5336 CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const { 5337 if (isVirtual()) { 5338 const CallExpr *CE = getVirtualCallExpr(); 5339 return CGF.CGM.getCXXABI().getVirtualFunctionPointer( 5340 CGF, getVirtualMethodDecl(), getThisAddress(), getVirtualFunctionType(), 5341 CE ? CE->getBeginLoc() : SourceLocation()); 5342 } 5343 5344 return *this; 5345 } 5346 5347 /* VarArg handling */ 5348 5349 Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) { 5350 VAListAddr = VE->isMicrosoftABI() 5351 ? EmitMSVAListRef(VE->getSubExpr()) 5352 : EmitVAListRef(VE->getSubExpr()); 5353 QualType Ty = VE->getType(); 5354 if (VE->isMicrosoftABI()) 5355 return CGM.getTypes().getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty); 5356 return CGM.getTypes().getABIInfo().EmitVAArg(*this, VAListAddr, Ty); 5357 } 5358