xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp (revision cb14a3fe5122c879eae1fb480ed7ce82a699ddb6)
1 //===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This coordinates the per-function state used while generating code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CodeGenFunction.h"
14 #include "CGBlocks.h"
15 #include "CGCUDARuntime.h"
16 #include "CGCXXABI.h"
17 #include "CGCleanup.h"
18 #include "CGDebugInfo.h"
19 #include "CGHLSLRuntime.h"
20 #include "CGOpenMPRuntime.h"
21 #include "CodeGenModule.h"
22 #include "CodeGenPGO.h"
23 #include "TargetInfo.h"
24 #include "clang/AST/ASTContext.h"
25 #include "clang/AST/ASTLambda.h"
26 #include "clang/AST/Attr.h"
27 #include "clang/AST/Decl.h"
28 #include "clang/AST/DeclCXX.h"
29 #include "clang/AST/Expr.h"
30 #include "clang/AST/StmtCXX.h"
31 #include "clang/AST/StmtObjC.h"
32 #include "clang/Basic/Builtins.h"
33 #include "clang/Basic/CodeGenOptions.h"
34 #include "clang/Basic/TargetInfo.h"
35 #include "clang/CodeGen/CGFunctionInfo.h"
36 #include "clang/Frontend/FrontendDiagnostic.h"
37 #include "llvm/ADT/ArrayRef.h"
38 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
39 #include "llvm/IR/DataLayout.h"
40 #include "llvm/IR/Dominators.h"
41 #include "llvm/IR/FPEnv.h"
42 #include "llvm/IR/IntrinsicInst.h"
43 #include "llvm/IR/Intrinsics.h"
44 #include "llvm/IR/MDBuilder.h"
45 #include "llvm/IR/Operator.h"
46 #include "llvm/Support/CRC.h"
47 #include "llvm/Support/xxhash.h"
48 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
49 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
50 #include <optional>
51 
52 using namespace clang;
53 using namespace CodeGen;
54 
55 /// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
56 /// markers.
57 static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
58                                       const LangOptions &LangOpts) {
59   if (CGOpts.DisableLifetimeMarkers)
60     return false;
61 
62   // Sanitizers may use markers.
63   if (CGOpts.SanitizeAddressUseAfterScope ||
64       LangOpts.Sanitize.has(SanitizerKind::HWAddress) ||
65       LangOpts.Sanitize.has(SanitizerKind::Memory))
66     return true;
67 
68   // For now, only in optimized builds.
69   return CGOpts.OptimizationLevel != 0;
70 }
71 
72 CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
73     : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
74       Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
75               CGBuilderInserterTy(this)),
76       SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
77       DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
78       ShouldEmitLifetimeMarkers(
79           shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {
80   if (!suppressNewContext)
81     CGM.getCXXABI().getMangleContext().startNewFunction();
82   EHStack.setCGF(this);
83 
84   SetFastMathFlags(CurFPFeatures);
85 }
86 
87 CodeGenFunction::~CodeGenFunction() {
88   assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
89 
90   if (getLangOpts().OpenMP && CurFn)
91     CGM.getOpenMPRuntime().functionFinished(*this);
92 
93   // If we have an OpenMPIRBuilder we want to finalize functions (incl.
94   // outlining etc) at some point. Doing it once the function codegen is done
95   // seems to be a reasonable spot. We do it here, as opposed to the deletion
96   // time of the CodeGenModule, because we have to ensure the IR has not yet
97   // been "emitted" to the outside, thus, modifications are still sensible.
98   if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
99     CGM.getOpenMPRuntime().getOMPBuilder().finalize(CurFn);
100 }
101 
102 // Map the LangOption for exception behavior into
103 // the corresponding enum in the IR.
104 llvm::fp::ExceptionBehavior
105 clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
106 
107   switch (Kind) {
108   case LangOptions::FPE_Ignore:  return llvm::fp::ebIgnore;
109   case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
110   case LangOptions::FPE_Strict:  return llvm::fp::ebStrict;
111   default:
112     llvm_unreachable("Unsupported FP Exception Behavior");
113   }
114 }
115 
116 void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
117   llvm::FastMathFlags FMF;
118   FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
119   FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
120   FMF.setNoInfs(FPFeatures.getNoHonorInfs());
121   FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
122   FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
123   FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
124   FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
125   Builder.setFastMathFlags(FMF);
126 }
127 
128 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
129                                                   const Expr *E)
130     : CGF(CGF) {
131   ConstructorHelper(E->getFPFeaturesInEffect(CGF.getLangOpts()));
132 }
133 
134 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
135                                                   FPOptions FPFeatures)
136     : CGF(CGF) {
137   ConstructorHelper(FPFeatures);
138 }
139 
140 void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
141   OldFPFeatures = CGF.CurFPFeatures;
142   CGF.CurFPFeatures = FPFeatures;
143 
144   OldExcept = CGF.Builder.getDefaultConstrainedExcept();
145   OldRounding = CGF.Builder.getDefaultConstrainedRounding();
146 
147   if (OldFPFeatures == FPFeatures)
148     return;
149 
150   FMFGuard.emplace(CGF.Builder);
151 
152   llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
153   CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
154   auto NewExceptionBehavior =
155       ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>(
156           FPFeatures.getExceptionMode()));
157   CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
158 
159   CGF.SetFastMathFlags(FPFeatures);
160 
161   assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
162           isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
163           isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
164           (NewExceptionBehavior == llvm::fp::ebIgnore &&
165            NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
166          "FPConstrained should be enabled on entire function");
167 
168   auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
169     auto OldValue =
170         CGF.CurFn->getFnAttribute(Name).getValueAsBool();
171     auto NewValue = OldValue & Value;
172     if (OldValue != NewValue)
173       CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));
174   };
175   mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
176   mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
177   mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
178   mergeFnAttrValue(
179       "unsafe-fp-math",
180       FPFeatures.getAllowFPReassociate() && FPFeatures.getAllowReciprocal() &&
181           FPFeatures.getAllowApproxFunc() && FPFeatures.getNoSignedZero() &&
182           FPFeatures.allowFPContractAcrossStatement());
183 }
184 
185 CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
186   CGF.CurFPFeatures = OldFPFeatures;
187   CGF.Builder.setDefaultConstrainedExcept(OldExcept);
188   CGF.Builder.setDefaultConstrainedRounding(OldRounding);
189 }
190 
191 LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
192   LValueBaseInfo BaseInfo;
193   TBAAAccessInfo TBAAInfo;
194   CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
195   Address Addr(V, ConvertTypeForMem(T), Alignment);
196   return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo);
197 }
198 
199 /// Given a value of type T* that may not be to a complete object,
200 /// construct an l-value with the natural pointee alignment of T.
201 LValue
202 CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
203   LValueBaseInfo BaseInfo;
204   TBAAAccessInfo TBAAInfo;
205   CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
206                                                 /* forPointeeType= */ true);
207   Address Addr(V, ConvertTypeForMem(T), Align);
208   return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
209 }
210 
211 
212 llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
213   return CGM.getTypes().ConvertTypeForMem(T);
214 }
215 
216 llvm::Type *CodeGenFunction::ConvertType(QualType T) {
217   return CGM.getTypes().ConvertType(T);
218 }
219 
220 TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
221   type = type.getCanonicalType();
222   while (true) {
223     switch (type->getTypeClass()) {
224 #define TYPE(name, parent)
225 #define ABSTRACT_TYPE(name, parent)
226 #define NON_CANONICAL_TYPE(name, parent) case Type::name:
227 #define DEPENDENT_TYPE(name, parent) case Type::name:
228 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
229 #include "clang/AST/TypeNodes.inc"
230       llvm_unreachable("non-canonical or dependent type in IR-generation");
231 
232     case Type::Auto:
233     case Type::DeducedTemplateSpecialization:
234       llvm_unreachable("undeduced type in IR-generation");
235 
236     // Various scalar types.
237     case Type::Builtin:
238     case Type::Pointer:
239     case Type::BlockPointer:
240     case Type::LValueReference:
241     case Type::RValueReference:
242     case Type::MemberPointer:
243     case Type::Vector:
244     case Type::ExtVector:
245     case Type::ConstantMatrix:
246     case Type::FunctionProto:
247     case Type::FunctionNoProto:
248     case Type::Enum:
249     case Type::ObjCObjectPointer:
250     case Type::Pipe:
251     case Type::BitInt:
252       return TEK_Scalar;
253 
254     // Complexes.
255     case Type::Complex:
256       return TEK_Complex;
257 
258     // Arrays, records, and Objective-C objects.
259     case Type::ConstantArray:
260     case Type::IncompleteArray:
261     case Type::VariableArray:
262     case Type::Record:
263     case Type::ObjCObject:
264     case Type::ObjCInterface:
265       return TEK_Aggregate;
266 
267     // We operate on atomic values according to their underlying type.
268     case Type::Atomic:
269       type = cast<AtomicType>(type)->getValueType();
270       continue;
271     }
272     llvm_unreachable("unknown type kind!");
273   }
274 }
275 
276 llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
277   // For cleanliness, we try to avoid emitting the return block for
278   // simple cases.
279   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
280 
281   if (CurBB) {
282     assert(!CurBB->getTerminator() && "Unexpected terminated block.");
283 
284     // We have a valid insert point, reuse it if it is empty or there are no
285     // explicit jumps to the return block.
286     if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
287       ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
288       delete ReturnBlock.getBlock();
289       ReturnBlock = JumpDest();
290     } else
291       EmitBlock(ReturnBlock.getBlock());
292     return llvm::DebugLoc();
293   }
294 
295   // Otherwise, if the return block is the target of a single direct
296   // branch then we can just put the code in that block instead. This
297   // cleans up functions which started with a unified return block.
298   if (ReturnBlock.getBlock()->hasOneUse()) {
299     llvm::BranchInst *BI =
300       dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());
301     if (BI && BI->isUnconditional() &&
302         BI->getSuccessor(0) == ReturnBlock.getBlock()) {
303       // Record/return the DebugLoc of the simple 'return' expression to be used
304       // later by the actual 'ret' instruction.
305       llvm::DebugLoc Loc = BI->getDebugLoc();
306       Builder.SetInsertPoint(BI->getParent());
307       BI->eraseFromParent();
308       delete ReturnBlock.getBlock();
309       ReturnBlock = JumpDest();
310       return Loc;
311     }
312   }
313 
314   // FIXME: We are at an unreachable point, there is no reason to emit the block
315   // unless it has uses. However, we still need a place to put the debug
316   // region.end for now.
317 
318   EmitBlock(ReturnBlock.getBlock());
319   return llvm::DebugLoc();
320 }
321 
322 static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
323   if (!BB) return;
324   if (!BB->use_empty()) {
325     CGF.CurFn->insert(CGF.CurFn->end(), BB);
326     return;
327   }
328   delete BB;
329 }
330 
331 void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
332   assert(BreakContinueStack.empty() &&
333          "mismatched push/pop in break/continue stack!");
334 
335   bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
336     && NumSimpleReturnExprs == NumReturnExprs
337     && ReturnBlock.getBlock()->use_empty();
338   // Usually the return expression is evaluated before the cleanup
339   // code.  If the function contains only a simple return statement,
340   // such as a constant, the location before the cleanup code becomes
341   // the last useful breakpoint in the function, because the simple
342   // return expression will be evaluated after the cleanup code. To be
343   // safe, set the debug location for cleanup code to the location of
344   // the return statement.  Otherwise the cleanup code should be at the
345   // end of the function's lexical scope.
346   //
347   // If there are multiple branches to the return block, the branch
348   // instructions will get the location of the return statements and
349   // all will be fine.
350   if (CGDebugInfo *DI = getDebugInfo()) {
351     if (OnlySimpleReturnStmts)
352       DI->EmitLocation(Builder, LastStopPoint);
353     else
354       DI->EmitLocation(Builder, EndLoc);
355   }
356 
357   // Pop any cleanups that might have been associated with the
358   // parameters.  Do this in whatever block we're currently in; it's
359   // important to do this before we enter the return block or return
360   // edges will be *really* confused.
361   bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
362   bool HasOnlyLifetimeMarkers =
363       HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth);
364   bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers;
365 
366   std::optional<ApplyDebugLocation> OAL;
367   if (HasCleanups) {
368     // Make sure the line table doesn't jump back into the body for
369     // the ret after it's been at EndLoc.
370     if (CGDebugInfo *DI = getDebugInfo()) {
371       if (OnlySimpleReturnStmts)
372         DI->EmitLocation(Builder, EndLoc);
373       else
374         // We may not have a valid end location. Try to apply it anyway, and
375         // fall back to an artificial location if needed.
376         OAL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc);
377     }
378 
379     PopCleanupBlocks(PrologueCleanupDepth);
380   }
381 
382   // Emit function epilog (to return).
383   llvm::DebugLoc Loc = EmitReturnBlock();
384 
385   if (ShouldInstrumentFunction()) {
386     if (CGM.getCodeGenOpts().InstrumentFunctions)
387       CurFn->addFnAttr("instrument-function-exit", "__cyg_profile_func_exit");
388     if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
389       CurFn->addFnAttr("instrument-function-exit-inlined",
390                        "__cyg_profile_func_exit");
391   }
392 
393   // Emit debug descriptor for function end.
394   if (CGDebugInfo *DI = getDebugInfo())
395     DI->EmitFunctionEnd(Builder, CurFn);
396 
397   // Reset the debug location to that of the simple 'return' expression, if any
398   // rather than that of the end of the function's scope '}'.
399   ApplyDebugLocation AL(*this, Loc);
400   EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc);
401   EmitEndEHSpec(CurCodeDecl);
402 
403   assert(EHStack.empty() &&
404          "did not remove all scopes from cleanup stack!");
405 
406   // If someone did an indirect goto, emit the indirect goto block at the end of
407   // the function.
408   if (IndirectBranch) {
409     EmitBlock(IndirectBranch->getParent());
410     Builder.ClearInsertionPoint();
411   }
412 
413   // If some of our locals escaped, insert a call to llvm.localescape in the
414   // entry block.
415   if (!EscapedLocals.empty()) {
416     // Invert the map from local to index into a simple vector. There should be
417     // no holes.
418     SmallVector<llvm::Value *, 4> EscapeArgs;
419     EscapeArgs.resize(EscapedLocals.size());
420     for (auto &Pair : EscapedLocals)
421       EscapeArgs[Pair.second] = Pair.first;
422     llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
423         &CGM.getModule(), llvm::Intrinsic::localescape);
424     CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
425   }
426 
427   // Remove the AllocaInsertPt instruction, which is just a convenience for us.
428   llvm::Instruction *Ptr = AllocaInsertPt;
429   AllocaInsertPt = nullptr;
430   Ptr->eraseFromParent();
431 
432   // PostAllocaInsertPt, if created, was lazily created when it was required,
433   // remove it now since it was just created for our own convenience.
434   if (PostAllocaInsertPt) {
435     llvm::Instruction *PostPtr = PostAllocaInsertPt;
436     PostAllocaInsertPt = nullptr;
437     PostPtr->eraseFromParent();
438   }
439 
440   // If someone took the address of a label but never did an indirect goto, we
441   // made a zero entry PHI node, which is illegal, zap it now.
442   if (IndirectBranch) {
443     llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());
444     if (PN->getNumIncomingValues() == 0) {
445       PN->replaceAllUsesWith(llvm::UndefValue::get(PN->getType()));
446       PN->eraseFromParent();
447     }
448   }
449 
450   EmitIfUsed(*this, EHResumeBlock);
451   EmitIfUsed(*this, TerminateLandingPad);
452   EmitIfUsed(*this, TerminateHandler);
453   EmitIfUsed(*this, UnreachableBlock);
454 
455   for (const auto &FuncletAndParent : TerminateFunclets)
456     EmitIfUsed(*this, FuncletAndParent.second);
457 
458   if (CGM.getCodeGenOpts().EmitDeclMetadata)
459     EmitDeclMetadata();
460 
461   for (const auto &R : DeferredReplacements) {
462     if (llvm::Value *Old = R.first) {
463       Old->replaceAllUsesWith(R.second);
464       cast<llvm::Instruction>(Old)->eraseFromParent();
465     }
466   }
467   DeferredReplacements.clear();
468 
469   // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
470   // PHIs if the current function is a coroutine. We don't do it for all
471   // functions as it may result in slight increase in numbers of instructions
472   // if compiled with no optimizations. We do it for coroutine as the lifetime
473   // of CleanupDestSlot alloca make correct coroutine frame building very
474   // difficult.
475   if (NormalCleanupDest.isValid() && isCoroutine()) {
476     llvm::DominatorTree DT(*CurFn);
477     llvm::PromoteMemToReg(
478         cast<llvm::AllocaInst>(NormalCleanupDest.getPointer()), DT);
479     NormalCleanupDest = Address::invalid();
480   }
481 
482   // Scan function arguments for vector width.
483   for (llvm::Argument &A : CurFn->args())
484     if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
485       LargestVectorWidth =
486           std::max((uint64_t)LargestVectorWidth,
487                    VT->getPrimitiveSizeInBits().getKnownMinValue());
488 
489   // Update vector width based on return type.
490   if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))
491     LargestVectorWidth =
492         std::max((uint64_t)LargestVectorWidth,
493                  VT->getPrimitiveSizeInBits().getKnownMinValue());
494 
495   if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
496     LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
497 
498   // Add the min-legal-vector-width attribute. This contains the max width from:
499   // 1. min-vector-width attribute used in the source program.
500   // 2. Any builtins used that have a vector width specified.
501   // 3. Values passed in and out of inline assembly.
502   // 4. Width of vector arguments and return types for this function.
503   // 5. Width of vector arguments and return types for functions called by this
504   //    function.
505   if (getContext().getTargetInfo().getTriple().isX86())
506     CurFn->addFnAttr("min-legal-vector-width",
507                      llvm::utostr(LargestVectorWidth));
508 
509   // Add vscale_range attribute if appropriate.
510   std::optional<std::pair<unsigned, unsigned>> VScaleRange =
511       getContext().getTargetInfo().getVScaleRange(getLangOpts());
512   if (VScaleRange) {
513     CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
514         getLLVMContext(), VScaleRange->first, VScaleRange->second));
515   }
516 
517   // If we generated an unreachable return block, delete it now.
518   if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
519     Builder.ClearInsertionPoint();
520     ReturnBlock.getBlock()->eraseFromParent();
521   }
522   if (ReturnValue.isValid()) {
523     auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
524     if (RetAlloca && RetAlloca->use_empty()) {
525       RetAlloca->eraseFromParent();
526       ReturnValue = Address::invalid();
527     }
528   }
529 }
530 
531 /// ShouldInstrumentFunction - Return true if the current function should be
532 /// instrumented with __cyg_profile_func_* calls
533 bool CodeGenFunction::ShouldInstrumentFunction() {
534   if (!CGM.getCodeGenOpts().InstrumentFunctions &&
535       !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
536       !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
537     return false;
538   if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
539     return false;
540   return true;
541 }
542 
543 bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
544   if (!CurFuncDecl)
545     return false;
546   return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
547 }
548 
549 /// ShouldXRayInstrument - Return true if the current function should be
550 /// instrumented with XRay nop sleds.
551 bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
552   return CGM.getCodeGenOpts().XRayInstrumentFunctions;
553 }
554 
555 /// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
556 /// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
557 bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
558   return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
559          (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
560           CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
561               XRayInstrKind::Custom);
562 }
563 
564 bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
565   return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
566          (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
567           CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
568               XRayInstrKind::Typed);
569 }
570 
571 llvm::ConstantInt *
572 CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
573   // Remove any (C++17) exception specifications, to allow calling e.g. a
574   // noexcept function through a non-noexcept pointer.
575   if (!Ty->isFunctionNoProtoType())
576     Ty = getContext().getFunctionTypeWithExceptionSpec(Ty, EST_None);
577   std::string Mangled;
578   llvm::raw_string_ostream Out(Mangled);
579   CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out, false);
580   return llvm::ConstantInt::get(
581       CGM.Int32Ty, static_cast<uint32_t>(llvm::xxh3_64bits(Mangled)));
582 }
583 
584 void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
585                                          llvm::Function *Fn) {
586   if (!FD->hasAttr<OpenCLKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
587     return;
588 
589   llvm::LLVMContext &Context = getLLVMContext();
590 
591   CGM.GenKernelArgMetadata(Fn, FD, this);
592 
593   if (!getLangOpts().OpenCL)
594     return;
595 
596   if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
597     QualType HintQTy = A->getTypeHint();
598     const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
599     bool IsSignedInteger =
600         HintQTy->isSignedIntegerType() ||
601         (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
602     llvm::Metadata *AttrMDArgs[] = {
603         llvm::ConstantAsMetadata::get(llvm::UndefValue::get(
604             CGM.getTypes().ConvertType(A->getTypeHint()))),
605         llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
606             llvm::IntegerType::get(Context, 32),
607             llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
608     Fn->setMetadata("vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));
609   }
610 
611   if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
612     llvm::Metadata *AttrMDArgs[] = {
613         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
614         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
615         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
616     Fn->setMetadata("work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));
617   }
618 
619   if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
620     llvm::Metadata *AttrMDArgs[] = {
621         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
622         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
623         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
624     Fn->setMetadata("reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));
625   }
626 
627   if (const OpenCLIntelReqdSubGroupSizeAttr *A =
628           FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
629     llvm::Metadata *AttrMDArgs[] = {
630         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getSubGroupSize()))};
631     Fn->setMetadata("intel_reqd_sub_group_size",
632                     llvm::MDNode::get(Context, AttrMDArgs));
633   }
634 }
635 
636 /// Determine whether the function F ends with a return stmt.
637 static bool endsWithReturn(const Decl* F) {
638   const Stmt *Body = nullptr;
639   if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))
640     Body = FD->getBody();
641   else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
642     Body = OMD->getBody();
643 
644   if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
645     auto LastStmt = CS->body_rbegin();
646     if (LastStmt != CS->body_rend())
647       return isa<ReturnStmt>(*LastStmt);
648   }
649   return false;
650 }
651 
652 void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
653   if (SanOpts.has(SanitizerKind::Thread)) {
654     Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
655     Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
656   }
657 }
658 
659 /// Check if the return value of this function requires sanitization.
660 bool CodeGenFunction::requiresReturnValueCheck() const {
661   return requiresReturnValueNullabilityCheck() ||
662          (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
663           CurCodeDecl->getAttr<ReturnsNonNullAttr>());
664 }
665 
666 static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
667   auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);
668   if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
669       !MD->getDeclName().getAsIdentifierInfo()->isStr("allocate") ||
670       (MD->getNumParams() != 1 && MD->getNumParams() != 2))
671     return false;
672 
673   if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.getSizeType())
674     return false;
675 
676   if (MD->getNumParams() == 2) {
677     auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
678     if (!PT || !PT->isVoidPointerType() ||
679         !PT->getPointeeType().isConstQualified())
680       return false;
681   }
682 
683   return true;
684 }
685 
686 bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
687   const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
688   return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
689 }
690 
691 bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
692   return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
693          getTarget().getCXXABI().isMicrosoft() &&
694          llvm::any_of(MD->parameters(), [&](ParmVarDecl *P) {
695            return isInAllocaArgument(CGM.getCXXABI(), P->getType());
696          });
697 }
698 
699 /// Return the UBSan prologue signature for \p FD if one is available.
700 static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
701                                             const FunctionDecl *FD) {
702   if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
703     if (!MD->isStatic())
704       return nullptr;
705   return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
706 }
707 
708 void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
709                                     llvm::Function *Fn,
710                                     const CGFunctionInfo &FnInfo,
711                                     const FunctionArgList &Args,
712                                     SourceLocation Loc,
713                                     SourceLocation StartLoc) {
714   assert(!CurFn &&
715          "Do not use a CodeGenFunction object for more than one function");
716 
717   const Decl *D = GD.getDecl();
718 
719   DidCallStackSave = false;
720   CurCodeDecl = D;
721   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
722   if (FD && FD->usesSEHTry())
723     CurSEHParent = GD;
724   CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
725   FnRetTy = RetTy;
726   CurFn = Fn;
727   CurFnInfo = &FnInfo;
728   assert(CurFn->isDeclaration() && "Function already has body?");
729 
730   // If this function is ignored for any of the enabled sanitizers,
731   // disable the sanitizer for the function.
732   do {
733 #define SANITIZER(NAME, ID)                                                    \
734   if (SanOpts.empty())                                                         \
735     break;                                                                     \
736   if (SanOpts.has(SanitizerKind::ID))                                          \
737     if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc))                    \
738       SanOpts.set(SanitizerKind::ID, false);
739 
740 #include "clang/Basic/Sanitizers.def"
741 #undef SANITIZER
742   } while (false);
743 
744   if (D) {
745     const bool SanitizeBounds = SanOpts.hasOneOf(SanitizerKind::Bounds);
746     SanitizerMask no_sanitize_mask;
747     bool NoSanitizeCoverage = false;
748 
749     for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
750       no_sanitize_mask |= Attr->getMask();
751       // SanitizeCoverage is not handled by SanOpts.
752       if (Attr->hasCoverage())
753         NoSanitizeCoverage = true;
754     }
755 
756     // Apply the no_sanitize* attributes to SanOpts.
757     SanOpts.Mask &= ~no_sanitize_mask;
758     if (no_sanitize_mask & SanitizerKind::Address)
759       SanOpts.set(SanitizerKind::KernelAddress, false);
760     if (no_sanitize_mask & SanitizerKind::KernelAddress)
761       SanOpts.set(SanitizerKind::Address, false);
762     if (no_sanitize_mask & SanitizerKind::HWAddress)
763       SanOpts.set(SanitizerKind::KernelHWAddress, false);
764     if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
765       SanOpts.set(SanitizerKind::HWAddress, false);
766 
767     if (SanitizeBounds && !SanOpts.hasOneOf(SanitizerKind::Bounds))
768       Fn->addFnAttr(llvm::Attribute::NoSanitizeBounds);
769 
770     if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
771       Fn->addFnAttr(llvm::Attribute::NoSanitizeCoverage);
772 
773     // Some passes need the non-negated no_sanitize attribute. Pass them on.
774     if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
775       if (no_sanitize_mask & SanitizerKind::Thread)
776         Fn->addFnAttr("no_sanitize_thread");
777     }
778   }
779 
780   if (ShouldSkipSanitizerInstrumentation()) {
781     CurFn->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);
782   } else {
783     // Apply sanitizer attributes to the function.
784     if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
785       Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
786     if (SanOpts.hasOneOf(SanitizerKind::HWAddress |
787                          SanitizerKind::KernelHWAddress))
788       Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
789     if (SanOpts.has(SanitizerKind::MemtagStack))
790       Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
791     if (SanOpts.has(SanitizerKind::Thread))
792       Fn->addFnAttr(llvm::Attribute::SanitizeThread);
793     if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
794       Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
795   }
796   if (SanOpts.has(SanitizerKind::SafeStack))
797     Fn->addFnAttr(llvm::Attribute::SafeStack);
798   if (SanOpts.has(SanitizerKind::ShadowCallStack))
799     Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
800 
801   // Apply fuzzing attribute to the function.
802   if (SanOpts.hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
803     Fn->addFnAttr(llvm::Attribute::OptForFuzzing);
804 
805   // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
806   // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
807   if (SanOpts.has(SanitizerKind::Thread)) {
808     if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {
809       IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);
810       if (OMD->getMethodFamily() == OMF_dealloc ||
811           OMD->getMethodFamily() == OMF_initialize ||
812           (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) {
813         markAsIgnoreThreadCheckingAtRuntime(Fn);
814       }
815     }
816   }
817 
818   // Ignore unrelated casts in STL allocate() since the allocator must cast
819   // from void* to T* before object initialization completes. Don't match on the
820   // namespace because not all allocators are in std::
821   if (D && SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
822     if (matchesStlAllocatorFn(D, getContext()))
823       SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
824   }
825 
826   // Ignore null checks in coroutine functions since the coroutines passes
827   // are not aware of how to move the extra UBSan instructions across the split
828   // coroutine boundaries.
829   if (D && SanOpts.has(SanitizerKind::Null))
830     if (FD && FD->getBody() &&
831         FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
832       SanOpts.Mask &= ~SanitizerKind::Null;
833 
834   // Apply xray attributes to the function (as a string, for now)
835   bool AlwaysXRayAttr = false;
836   if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
837     if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
838             XRayInstrKind::FunctionEntry) ||
839         CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
840             XRayInstrKind::FunctionExit)) {
841       if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
842         Fn->addFnAttr("function-instrument", "xray-always");
843         AlwaysXRayAttr = true;
844       }
845       if (XRayAttr->neverXRayInstrument())
846         Fn->addFnAttr("function-instrument", "xray-never");
847       if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
848         if (ShouldXRayInstrumentFunction())
849           Fn->addFnAttr("xray-log-args",
850                         llvm::utostr(LogArgs->getArgumentCount()));
851     }
852   } else {
853     if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
854       Fn->addFnAttr(
855           "xray-instruction-threshold",
856           llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
857   }
858 
859   if (ShouldXRayInstrumentFunction()) {
860     if (CGM.getCodeGenOpts().XRayIgnoreLoops)
861       Fn->addFnAttr("xray-ignore-loops");
862 
863     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
864             XRayInstrKind::FunctionExit))
865       Fn->addFnAttr("xray-skip-exit");
866 
867     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
868             XRayInstrKind::FunctionEntry))
869       Fn->addFnAttr("xray-skip-entry");
870 
871     auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
872     if (FuncGroups > 1) {
873       auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
874                                               CurFn->getName().bytes_end());
875       auto Group = crc32(FuncName) % FuncGroups;
876       if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
877           !AlwaysXRayAttr)
878         Fn->addFnAttr("function-instrument", "xray-never");
879     }
880   }
881 
882   if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone) {
883     switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
884     case ProfileList::Skip:
885       Fn->addFnAttr(llvm::Attribute::SkipProfile);
886       break;
887     case ProfileList::Forbid:
888       Fn->addFnAttr(llvm::Attribute::NoProfile);
889       break;
890     case ProfileList::Allow:
891       break;
892     }
893   }
894 
895   unsigned Count, Offset;
896   if (const auto *Attr =
897           D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
898     Count = Attr->getCount();
899     Offset = Attr->getOffset();
900   } else {
901     Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
902     Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
903   }
904   if (Count && Offset <= Count) {
905     Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
906     if (Offset)
907       Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
908   }
909   // Instruct that functions for COFF/CodeView targets should start with a
910   // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
911   // backends as they don't need it -- instructions on these architectures are
912   // always atomically patchable at runtime.
913   if (CGM.getCodeGenOpts().HotPatch &&
914       getContext().getTargetInfo().getTriple().isX86() &&
915       getContext().getTargetInfo().getTriple().getEnvironment() !=
916           llvm::Triple::CODE16)
917     Fn->addFnAttr("patchable-function", "prologue-short-redirect");
918 
919   // Add no-jump-tables value.
920   if (CGM.getCodeGenOpts().NoUseJumpTables)
921     Fn->addFnAttr("no-jump-tables", "true");
922 
923   // Add no-inline-line-tables value.
924   if (CGM.getCodeGenOpts().NoInlineLineTables)
925     Fn->addFnAttr("no-inline-line-tables");
926 
927   // Add profile-sample-accurate value.
928   if (CGM.getCodeGenOpts().ProfileSampleAccurate)
929     Fn->addFnAttr("profile-sample-accurate");
930 
931   if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
932     Fn->addFnAttr("use-sample-profile");
933 
934   if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
935     Fn->addFnAttr("cfi-canonical-jump-table");
936 
937   if (D && D->hasAttr<NoProfileFunctionAttr>())
938     Fn->addFnAttr(llvm::Attribute::NoProfile);
939 
940   if (D) {
941     // Function attributes take precedence over command line flags.
942     if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
943       switch (A->getThunkType()) {
944       case FunctionReturnThunksAttr::Kind::Keep:
945         break;
946       case FunctionReturnThunksAttr::Kind::Extern:
947         Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
948         break;
949       }
950     } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
951       Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
952   }
953 
954   if (FD && (getLangOpts().OpenCL ||
955              (getLangOpts().HIP && getLangOpts().CUDAIsDevice))) {
956     // Add metadata for a kernel function.
957     EmitKernelMetadata(FD, Fn);
958   }
959 
960   // If we are checking function types, emit a function type signature as
961   // prologue data.
962   if (FD && SanOpts.has(SanitizerKind::Function)) {
963     if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
964       llvm::LLVMContext &Ctx = Fn->getContext();
965       llvm::MDBuilder MDB(Ctx);
966       Fn->setMetadata(
967           llvm::LLVMContext::MD_func_sanitize,
968           MDB.createRTTIPointerPrologue(
969               PrologueSig, getUBSanFunctionTypeHash(FD->getType())));
970     }
971   }
972 
973   // If we're checking nullability, we need to know whether we can check the
974   // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
975   if (SanOpts.has(SanitizerKind::NullabilityReturn)) {
976     auto Nullability = FnRetTy->getNullability();
977     if (Nullability && *Nullability == NullabilityKind::NonNull) {
978       if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) &&
979             CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
980         RetValNullabilityPrecondition =
981             llvm::ConstantInt::getTrue(getLLVMContext());
982     }
983   }
984 
985   // If we're in C++ mode and the function name is "main", it is guaranteed
986   // to be norecurse by the standard (3.6.1.3 "The function main shall not be
987   // used within a program").
988   //
989   // OpenCL C 2.0 v2.2-11 s6.9.i:
990   //     Recursion is not supported.
991   //
992   // SYCL v1.2.1 s3.10:
993   //     kernels cannot include RTTI information, exception classes,
994   //     recursive code, virtual functions or make use of C++ libraries that
995   //     are not compiled for the device.
996   if (FD && ((getLangOpts().CPlusPlus && FD->isMain()) ||
997              getLangOpts().OpenCL || getLangOpts().SYCLIsDevice ||
998              (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
999     Fn->addFnAttr(llvm::Attribute::NoRecurse);
1000 
1001   llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1002   llvm::fp::ExceptionBehavior FPExceptionBehavior =
1003       ToConstrainedExceptMD(getLangOpts().getDefaultExceptionMode());
1004   Builder.setDefaultConstrainedRounding(RM);
1005   Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1006   if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1007       (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1008                RM != llvm::RoundingMode::NearestTiesToEven))) {
1009     Builder.setIsFPConstrained(true);
1010     Fn->addFnAttr(llvm::Attribute::StrictFP);
1011   }
1012 
1013   // If a custom alignment is used, force realigning to this alignment on
1014   // any main function which certainly will need it.
1015   if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1016              CGM.getCodeGenOpts().StackAlignment))
1017     Fn->addFnAttr("stackrealign");
1018 
1019   // "main" doesn't need to zero out call-used registers.
1020   if (FD && FD->isMain())
1021     Fn->removeFnAttr("zero-call-used-regs");
1022 
1023   llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
1024 
1025   // Create a marker to make it easy to insert allocas into the entryblock
1026   // later.  Don't create this with the builder, because we don't want it
1027   // folded.
1028   llvm::Value *Undef = llvm::UndefValue::get(Int32Ty);
1029   AllocaInsertPt = new llvm::BitCastInst(Undef, Int32Ty, "allocapt", EntryBB);
1030 
1031   ReturnBlock = getJumpDestInCurrentScope("return");
1032 
1033   Builder.SetInsertPoint(EntryBB);
1034 
1035   // If we're checking the return value, allocate space for a pointer to a
1036   // precise source location of the checked return statement.
1037   if (requiresReturnValueCheck()) {
1038     ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr");
1039     Builder.CreateStore(llvm::ConstantPointerNull::get(Int8PtrTy),
1040                         ReturnLocation);
1041   }
1042 
1043   // Emit subprogram debug descriptor.
1044   if (CGDebugInfo *DI = getDebugInfo()) {
1045     // Reconstruct the type from the argument list so that implicit parameters,
1046     // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1047     // convention.
1048     DI->emitFunctionStart(GD, Loc, StartLoc,
1049                           DI->getFunctionType(FD, RetTy, Args), CurFn,
1050                           CurFuncIsThunk);
1051   }
1052 
1053   if (ShouldInstrumentFunction()) {
1054     if (CGM.getCodeGenOpts().InstrumentFunctions)
1055       CurFn->addFnAttr("instrument-function-entry", "__cyg_profile_func_enter");
1056     if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1057       CurFn->addFnAttr("instrument-function-entry-inlined",
1058                        "__cyg_profile_func_enter");
1059     if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1060       CurFn->addFnAttr("instrument-function-entry-inlined",
1061                        "__cyg_profile_func_enter_bare");
1062   }
1063 
1064   // Since emitting the mcount call here impacts optimizations such as function
1065   // inlining, we just add an attribute to insert a mcount call in backend.
1066   // The attribute "counting-function" is set to mcount function name which is
1067   // architecture dependent.
1068   if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1069     // Calls to fentry/mcount should not be generated if function has
1070     // the no_instrument_function attribute.
1071     if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1072       if (CGM.getCodeGenOpts().CallFEntry)
1073         Fn->addFnAttr("fentry-call", "true");
1074       else {
1075         Fn->addFnAttr("instrument-function-entry-inlined",
1076                       getTarget().getMCountName());
1077       }
1078       if (CGM.getCodeGenOpts().MNopMCount) {
1079         if (!CGM.getCodeGenOpts().CallFEntry)
1080           CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1081             << "-mnop-mcount" << "-mfentry";
1082         Fn->addFnAttr("mnop-mcount");
1083       }
1084 
1085       if (CGM.getCodeGenOpts().RecordMCount) {
1086         if (!CGM.getCodeGenOpts().CallFEntry)
1087           CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1088             << "-mrecord-mcount" << "-mfentry";
1089         Fn->addFnAttr("mrecord-mcount");
1090       }
1091     }
1092   }
1093 
1094   if (CGM.getCodeGenOpts().PackedStack) {
1095     if (getContext().getTargetInfo().getTriple().getArch() !=
1096         llvm::Triple::systemz)
1097       CGM.getDiags().Report(diag::err_opt_not_valid_on_target)
1098         << "-mpacked-stack";
1099     Fn->addFnAttr("packed-stack");
1100   }
1101 
1102   if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1103       !CGM.getDiags().isIgnored(diag::warn_fe_backend_frame_larger_than, Loc))
1104     Fn->addFnAttr("warn-stack-size",
1105                   std::to_string(CGM.getCodeGenOpts().WarnStackSize));
1106 
1107   if (RetTy->isVoidType()) {
1108     // Void type; nothing to return.
1109     ReturnValue = Address::invalid();
1110 
1111     // Count the implicit return.
1112     if (!endsWithReturn(D))
1113       ++NumReturnExprs;
1114   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1115     // Indirect return; emit returned value directly into sret slot.
1116     // This reduces code size, and affects correctness in C++.
1117     auto AI = CurFn->arg_begin();
1118     if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1119       ++AI;
1120     ReturnValue =
1121         Address(&*AI, ConvertType(RetTy),
1122                 CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull);
1123     if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1124       ReturnValuePointer = CreateDefaultAlignTempAlloca(
1125           ReturnValue.getPointer()->getType(), "result.ptr");
1126       Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer);
1127     }
1128   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1129              !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
1130     // Load the sret pointer from the argument struct and return into that.
1131     unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1132     llvm::Function::arg_iterator EI = CurFn->arg_end();
1133     --EI;
1134     llvm::Value *Addr = Builder.CreateStructGEP(
1135         CurFnInfo->getArgStruct(), &*EI, Idx);
1136     llvm::Type *Ty =
1137         cast<llvm::GetElementPtrInst>(Addr)->getResultElementType();
1138     ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1139     Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result");
1140     ReturnValue = Address(Addr, ConvertType(RetTy),
1141                           CGM.getNaturalTypeAlignment(RetTy), KnownNonNull);
1142   } else {
1143     ReturnValue = CreateIRTemp(RetTy, "retval");
1144 
1145     // Tell the epilog emitter to autorelease the result.  We do this
1146     // now so that various specialized functions can suppress it
1147     // during their IR-generation.
1148     if (getLangOpts().ObjCAutoRefCount &&
1149         !CurFnInfo->isReturnsRetained() &&
1150         RetTy->isObjCRetainableType())
1151       AutoreleaseResult = true;
1152   }
1153 
1154   EmitStartEHSpec(CurCodeDecl);
1155 
1156   PrologueCleanupDepth = EHStack.stable_begin();
1157 
1158   // Emit OpenMP specific initialization of the device functions.
1159   if (getLangOpts().OpenMP && CurCodeDecl)
1160     CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl);
1161 
1162   // Handle emitting HLSL entry functions.
1163   if (D && D->hasAttr<HLSLShaderAttr>())
1164     CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1165 
1166   EmitFunctionProlog(*CurFnInfo, CurFn, Args);
1167 
1168   if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(D);
1169       MD && !MD->isStatic()) {
1170     bool IsInLambda =
1171         MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1172     if (MD->isImplicitObjectMemberFunction())
1173       CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
1174     if (IsInLambda) {
1175       // We're in a lambda; figure out the captures.
1176       MD->getParent()->getCaptureFields(LambdaCaptureFields,
1177                                         LambdaThisCaptureField);
1178       if (LambdaThisCaptureField) {
1179         // If the lambda captures the object referred to by '*this' - either by
1180         // value or by reference, make sure CXXThisValue points to the correct
1181         // object.
1182 
1183         // Get the lvalue for the field (which is a copy of the enclosing object
1184         // or contains the address of the enclosing object).
1185         LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
1186         if (!LambdaThisCaptureField->getType()->isPointerType()) {
1187           // If the enclosing object was captured by value, just use its address.
1188           CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
1189         } else {
1190           // Load the lvalue pointed to by the field, since '*this' was captured
1191           // by reference.
1192           CXXThisValue =
1193               EmitLoadOfLValue(ThisFieldLValue, SourceLocation()).getScalarVal();
1194         }
1195       }
1196       for (auto *FD : MD->getParent()->fields()) {
1197         if (FD->hasCapturedVLAType()) {
1198           auto *ExprArg = EmitLoadOfLValue(EmitLValueForLambdaField(FD),
1199                                            SourceLocation()).getScalarVal();
1200           auto VAT = FD->getCapturedVLAType();
1201           VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1202         }
1203       }
1204     } else if (MD->isImplicitObjectMemberFunction()) {
1205       // Not in a lambda; just use 'this' from the method.
1206       // FIXME: Should we generate a new load for each use of 'this'?  The
1207       // fast register allocator would be happier...
1208       CXXThisValue = CXXABIThisValue;
1209     }
1210 
1211     // Check the 'this' pointer once per function, if it's available.
1212     if (CXXABIThisValue) {
1213       SanitizerSet SkippedChecks;
1214       SkippedChecks.set(SanitizerKind::ObjectSize, true);
1215       QualType ThisTy = MD->getThisType();
1216 
1217       // If this is the call operator of a lambda with no captures, it
1218       // may have a static invoker function, which may call this operator with
1219       // a null 'this' pointer.
1220       if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1221         SkippedChecks.set(SanitizerKind::Null, true);
1222 
1223       EmitTypeCheck(
1224           isa<CXXConstructorDecl>(MD) ? TCK_ConstructorCall : TCK_MemberCall,
1225           Loc, CXXABIThisValue, ThisTy, CXXABIThisAlignment, SkippedChecks);
1226     }
1227   }
1228 
1229   // If any of the arguments have a variably modified type, make sure to
1230   // emit the type size, but only if the function is not naked. Naked functions
1231   // have no prolog to run this evaluation.
1232   if (!FD || !FD->hasAttr<NakedAttr>()) {
1233     for (const VarDecl *VD : Args) {
1234       // Dig out the type as written from ParmVarDecls; it's unclear whether
1235       // the standard (C99 6.9.1p10) requires this, but we're following the
1236       // precedent set by gcc.
1237       QualType Ty;
1238       if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
1239         Ty = PVD->getOriginalType();
1240       else
1241         Ty = VD->getType();
1242 
1243       if (Ty->isVariablyModifiedType())
1244         EmitVariablyModifiedType(Ty);
1245     }
1246   }
1247   // Emit a location at the end of the prologue.
1248   if (CGDebugInfo *DI = getDebugInfo())
1249     DI->EmitLocation(Builder, StartLoc);
1250   // TODO: Do we need to handle this in two places like we do with
1251   // target-features/target-cpu?
1252   if (CurFuncDecl)
1253     if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1254       LargestVectorWidth = VecWidth->getVectorWidth();
1255 }
1256 
1257 void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1258   incrementProfileCounter(Body);
1259   if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
1260     EmitCompoundStmtWithoutScope(*S);
1261   else
1262     EmitStmt(Body);
1263 }
1264 
1265 /// When instrumenting to collect profile data, the counts for some blocks
1266 /// such as switch cases need to not include the fall-through counts, so
1267 /// emit a branch around the instrumentation code. When not instrumenting,
1268 /// this just calls EmitBlock().
1269 void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1270                                                const Stmt *S) {
1271   llvm::BasicBlock *SkipCountBB = nullptr;
1272   if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1273     // When instrumenting for profiling, the fallthrough to certain
1274     // statements needs to skip over the instrumentation code so that we
1275     // get an accurate count.
1276     SkipCountBB = createBasicBlock("skipcount");
1277     EmitBranch(SkipCountBB);
1278   }
1279   EmitBlock(BB);
1280   uint64_t CurrentCount = getCurrentProfileCount();
1281   incrementProfileCounter(S);
1282   setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1283   if (SkipCountBB)
1284     EmitBlock(SkipCountBB);
1285 }
1286 
1287 /// Tries to mark the given function nounwind based on the
1288 /// non-existence of any throwing calls within it.  We believe this is
1289 /// lightweight enough to do at -O0.
1290 static void TryMarkNoThrow(llvm::Function *F) {
1291   // LLVM treats 'nounwind' on a function as part of the type, so we
1292   // can't do this on functions that can be overwritten.
1293   if (F->isInterposable()) return;
1294 
1295   for (llvm::BasicBlock &BB : *F)
1296     for (llvm::Instruction &I : BB)
1297       if (I.mayThrow())
1298         return;
1299 
1300   F->setDoesNotThrow();
1301 }
1302 
1303 QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1304                                                FunctionArgList &Args) {
1305   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1306   QualType ResTy = FD->getReturnType();
1307 
1308   const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
1309   if (MD && MD->isImplicitObjectMemberFunction()) {
1310     if (CGM.getCXXABI().HasThisReturn(GD))
1311       ResTy = MD->getThisType();
1312     else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1313       ResTy = CGM.getContext().VoidPtrTy;
1314     CGM.getCXXABI().buildThisParam(*this, Args);
1315   }
1316 
1317   // The base version of an inheriting constructor whose constructed base is a
1318   // virtual base is not passed any arguments (because it doesn't actually call
1319   // the inherited constructor).
1320   bool PassedParams = true;
1321   if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
1322     if (auto Inherited = CD->getInheritedConstructor())
1323       PassedParams =
1324           getTypes().inheritingCtorHasParams(Inherited, GD.getCtorType());
1325 
1326   if (PassedParams) {
1327     for (auto *Param : FD->parameters()) {
1328       Args.push_back(Param);
1329       if (!Param->hasAttr<PassObjectSizeAttr>())
1330         continue;
1331 
1332       auto *Implicit = ImplicitParamDecl::Create(
1333           getContext(), Param->getDeclContext(), Param->getLocation(),
1334           /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamKind::Other);
1335       SizeArguments[Param] = Implicit;
1336       Args.push_back(Implicit);
1337     }
1338   }
1339 
1340   if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
1341     CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
1342 
1343   return ResTy;
1344 }
1345 
1346 void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1347                                    const CGFunctionInfo &FnInfo) {
1348   assert(Fn && "generating code for null Function");
1349   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1350   CurGD = GD;
1351 
1352   FunctionArgList Args;
1353   QualType ResTy = BuildFunctionArgList(GD, Args);
1354 
1355   if (FD->isInlineBuiltinDeclaration()) {
1356     // When generating code for a builtin with an inline declaration, use a
1357     // mangled name to hold the actual body, while keeping an external
1358     // definition in case the function pointer is referenced somewhere.
1359     std::string FDInlineName = (Fn->getName() + ".inline").str();
1360     llvm::Module *M = Fn->getParent();
1361     llvm::Function *Clone = M->getFunction(FDInlineName);
1362     if (!Clone) {
1363       Clone = llvm::Function::Create(Fn->getFunctionType(),
1364                                      llvm::GlobalValue::InternalLinkage,
1365                                      Fn->getAddressSpace(), FDInlineName, M);
1366       Clone->addFnAttr(llvm::Attribute::AlwaysInline);
1367     }
1368     Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1369     Fn = Clone;
1370   } else {
1371     // Detect the unusual situation where an inline version is shadowed by a
1372     // non-inline version. In that case we should pick the external one
1373     // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1374     // to detect that situation before we reach codegen, so do some late
1375     // replacement.
1376     for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1377          PD = PD->getPreviousDecl()) {
1378       if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1379         std::string FDInlineName = (Fn->getName() + ".inline").str();
1380         llvm::Module *M = Fn->getParent();
1381         if (llvm::Function *Clone = M->getFunction(FDInlineName)) {
1382           Clone->replaceAllUsesWith(Fn);
1383           Clone->eraseFromParent();
1384         }
1385         break;
1386       }
1387     }
1388   }
1389 
1390   // Check if we should generate debug info for this function.
1391   if (FD->hasAttr<NoDebugAttr>()) {
1392     // Clear non-distinct debug info that was possibly attached to the function
1393     // due to an earlier declaration without the nodebug attribute
1394     Fn->setSubprogram(nullptr);
1395     // Disable debug info indefinitely for this function
1396     DebugInfo = nullptr;
1397   }
1398 
1399   // The function might not have a body if we're generating thunks for a
1400   // function declaration.
1401   SourceRange BodyRange;
1402   if (Stmt *Body = FD->getBody())
1403     BodyRange = Body->getSourceRange();
1404   else
1405     BodyRange = FD->getLocation();
1406   CurEHLocation = BodyRange.getEnd();
1407 
1408   // Use the location of the start of the function to determine where
1409   // the function definition is located. By default use the location
1410   // of the declaration as the location for the subprogram. A function
1411   // may lack a declaration in the source code if it is created by code
1412   // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1413   SourceLocation Loc = FD->getLocation();
1414 
1415   // If this is a function specialization then use the pattern body
1416   // as the location for the function.
1417   if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1418     if (SpecDecl->hasBody(SpecDecl))
1419       Loc = SpecDecl->getLocation();
1420 
1421   Stmt *Body = FD->getBody();
1422 
1423   if (Body) {
1424     // Coroutines always emit lifetime markers.
1425     if (isa<CoroutineBodyStmt>(Body))
1426       ShouldEmitLifetimeMarkers = true;
1427 
1428     // Initialize helper which will detect jumps which can cause invalid
1429     // lifetime markers.
1430     if (ShouldEmitLifetimeMarkers)
1431       Bypasses.Init(Body);
1432   }
1433 
1434   // Emit the standard function prologue.
1435   StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
1436 
1437   // Save parameters for coroutine function.
1438   if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body))
1439     llvm::append_range(FnArgs, FD->parameters());
1440 
1441   // Ensure that the function adheres to the forward progress guarantee, which
1442   // is required by certain optimizations.
1443   if (checkIfFunctionMustProgress())
1444     CurFn->addFnAttr(llvm::Attribute::MustProgress);
1445 
1446   // Generate the body of the function.
1447   PGO.assignRegionCounters(GD, CurFn);
1448   if (isa<CXXDestructorDecl>(FD))
1449     EmitDestructorBody(Args);
1450   else if (isa<CXXConstructorDecl>(FD))
1451     EmitConstructorBody(Args);
1452   else if (getLangOpts().CUDA &&
1453            !getLangOpts().CUDAIsDevice &&
1454            FD->hasAttr<CUDAGlobalAttr>())
1455     CGM.getCUDARuntime().emitDeviceStub(*this, Args);
1456   else if (isa<CXXMethodDecl>(FD) &&
1457            cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()) {
1458     // The lambda static invoker function is special, because it forwards or
1459     // clones the body of the function call operator (but is actually static).
1460     EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD));
1461   } else if (isa<CXXMethodDecl>(FD) &&
1462              isLambdaCallOperator(cast<CXXMethodDecl>(FD)) &&
1463              !FnInfo.isDelegateCall() &&
1464              cast<CXXMethodDecl>(FD)->getParent()->getLambdaStaticInvoker() &&
1465              hasInAllocaArg(cast<CXXMethodDecl>(FD))) {
1466     // If emitting a lambda with static invoker on X86 Windows, change
1467     // the call operator body.
1468     // Make sure that this is a call operator with an inalloca arg and check
1469     // for delegate call to make sure this is the original call op and not the
1470     // new forwarding function for the static invoker.
1471     EmitLambdaInAllocaCallOpBody(cast<CXXMethodDecl>(FD));
1472   } else if (FD->isDefaulted() && isa<CXXMethodDecl>(FD) &&
1473              (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() ||
1474               cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator())) {
1475     // Implicit copy-assignment gets the same special treatment as implicit
1476     // copy-constructors.
1477     emitImplicitAssignmentOperatorBody(Args);
1478   } else if (Body) {
1479     EmitFunctionBody(Body);
1480   } else
1481     llvm_unreachable("no definition for emitted function");
1482 
1483   // C++11 [stmt.return]p2:
1484   //   Flowing off the end of a function [...] results in undefined behavior in
1485   //   a value-returning function.
1486   // C11 6.9.1p12:
1487   //   If the '}' that terminates a function is reached, and the value of the
1488   //   function call is used by the caller, the behavior is undefined.
1489   if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1490       !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1491     bool ShouldEmitUnreachable =
1492         CGM.getCodeGenOpts().StrictReturn ||
1493         !CGM.MayDropFunctionReturn(FD->getASTContext(), FD->getReturnType());
1494     if (SanOpts.has(SanitizerKind::Return)) {
1495       SanitizerScope SanScope(this);
1496       llvm::Value *IsFalse = Builder.getFalse();
1497       EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
1498                 SanitizerHandler::MissingReturn,
1499                 EmitCheckSourceLocation(FD->getLocation()), std::nullopt);
1500     } else if (ShouldEmitUnreachable) {
1501       if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1502         EmitTrapCall(llvm::Intrinsic::trap);
1503     }
1504     if (SanOpts.has(SanitizerKind::Return) || ShouldEmitUnreachable) {
1505       Builder.CreateUnreachable();
1506       Builder.ClearInsertionPoint();
1507     }
1508   }
1509 
1510   // Emit the standard function epilogue.
1511   FinishFunction(BodyRange.getEnd());
1512 
1513   // If we haven't marked the function nothrow through other means, do
1514   // a quick pass now to see if we can.
1515   if (!CurFn->doesNotThrow())
1516     TryMarkNoThrow(CurFn);
1517 }
1518 
1519 /// ContainsLabel - Return true if the statement contains a label in it.  If
1520 /// this statement is not executed normally, it not containing a label means
1521 /// that we can just remove the code.
1522 bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1523   // Null statement, not a label!
1524   if (!S) return false;
1525 
1526   // If this is a label, we have to emit the code, consider something like:
1527   // if (0) {  ...  foo:  bar(); }  goto foo;
1528   //
1529   // TODO: If anyone cared, we could track __label__'s, since we know that you
1530   // can't jump to one from outside their declared region.
1531   if (isa<LabelStmt>(S))
1532     return true;
1533 
1534   // If this is a case/default statement, and we haven't seen a switch, we have
1535   // to emit the code.
1536   if (isa<SwitchCase>(S) && !IgnoreCaseStmts)
1537     return true;
1538 
1539   // If this is a switch statement, we want to ignore cases below it.
1540   if (isa<SwitchStmt>(S))
1541     IgnoreCaseStmts = true;
1542 
1543   // Scan subexpressions for verboten labels.
1544   for (const Stmt *SubStmt : S->children())
1545     if (ContainsLabel(SubStmt, IgnoreCaseStmts))
1546       return true;
1547 
1548   return false;
1549 }
1550 
1551 /// containsBreak - Return true if the statement contains a break out of it.
1552 /// If the statement (recursively) contains a switch or loop with a break
1553 /// inside of it, this is fine.
1554 bool CodeGenFunction::containsBreak(const Stmt *S) {
1555   // Null statement, not a label!
1556   if (!S) return false;
1557 
1558   // If this is a switch or loop that defines its own break scope, then we can
1559   // include it and anything inside of it.
1560   if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||
1561       isa<ForStmt>(S))
1562     return false;
1563 
1564   if (isa<BreakStmt>(S))
1565     return true;
1566 
1567   // Scan subexpressions for verboten breaks.
1568   for (const Stmt *SubStmt : S->children())
1569     if (containsBreak(SubStmt))
1570       return true;
1571 
1572   return false;
1573 }
1574 
1575 bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1576   if (!S) return false;
1577 
1578   // Some statement kinds add a scope and thus never add a decl to the current
1579   // scope. Note, this list is longer than the list of statements that might
1580   // have an unscoped decl nested within them, but this way is conservatively
1581   // correct even if more statement kinds are added.
1582   if (isa<IfStmt>(S) || isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
1583       isa<DoStmt>(S) || isa<ForStmt>(S) || isa<CompoundStmt>(S) ||
1584       isa<CXXForRangeStmt>(S) || isa<CXXTryStmt>(S) ||
1585       isa<ObjCForCollectionStmt>(S) || isa<ObjCAtTryStmt>(S))
1586     return false;
1587 
1588   if (isa<DeclStmt>(S))
1589     return true;
1590 
1591   for (const Stmt *SubStmt : S->children())
1592     if (mightAddDeclToScope(SubStmt))
1593       return true;
1594 
1595   return false;
1596 }
1597 
1598 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1599 /// to a constant, or if it does but contains a label, return false.  If it
1600 /// constant folds return true and set the boolean result in Result.
1601 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1602                                                    bool &ResultBool,
1603                                                    bool AllowLabels) {
1604   llvm::APSInt ResultInt;
1605   if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))
1606     return false;
1607 
1608   ResultBool = ResultInt.getBoolValue();
1609   return true;
1610 }
1611 
1612 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1613 /// to a constant, or if it does but contains a label, return false.  If it
1614 /// constant folds return true and set the folded value.
1615 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1616                                                    llvm::APSInt &ResultInt,
1617                                                    bool AllowLabels) {
1618   // FIXME: Rename and handle conversion of other evaluatable things
1619   // to bool.
1620   Expr::EvalResult Result;
1621   if (!Cond->EvaluateAsInt(Result, getContext()))
1622     return false;  // Not foldable, not integer or not fully evaluatable.
1623 
1624   llvm::APSInt Int = Result.Val.getInt();
1625   if (!AllowLabels && CodeGenFunction::ContainsLabel(Cond))
1626     return false;  // Contains a label.
1627 
1628   ResultInt = Int;
1629   return true;
1630 }
1631 
1632 /// Determine whether the given condition is an instrumentable condition
1633 /// (i.e. no "&&" or "||").
1634 bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1635   // Bypass simplistic logical-NOT operator before determining whether the
1636   // condition contains any other logical operator.
1637   if (const UnaryOperator *UnOp = dyn_cast<UnaryOperator>(C->IgnoreParens()))
1638     if (UnOp->getOpcode() == UO_LNot)
1639       C = UnOp->getSubExpr();
1640 
1641   const BinaryOperator *BOp = dyn_cast<BinaryOperator>(C->IgnoreParens());
1642   return (!BOp || !BOp->isLogicalOp());
1643 }
1644 
1645 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1646 /// increments a profile counter based on the semantics of the given logical
1647 /// operator opcode.  This is used to instrument branch condition coverage for
1648 /// logical operators.
1649 void CodeGenFunction::EmitBranchToCounterBlock(
1650     const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1651     llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1652     Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1653   // If not instrumenting, just emit a branch.
1654   bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1655   if (!InstrumentRegions || !isInstrumentedCondition(Cond))
1656     return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1657 
1658   llvm::BasicBlock *ThenBlock = nullptr;
1659   llvm::BasicBlock *ElseBlock = nullptr;
1660   llvm::BasicBlock *NextBlock = nullptr;
1661 
1662   // Create the block we'll use to increment the appropriate counter.
1663   llvm::BasicBlock *CounterIncrBlock = createBasicBlock("lop.rhscnt");
1664 
1665   // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1666   // means we need to evaluate the condition and increment the counter on TRUE:
1667   //
1668   // if (Cond)
1669   //   goto CounterIncrBlock;
1670   // else
1671   //   goto FalseBlock;
1672   //
1673   // CounterIncrBlock:
1674   //   Counter++;
1675   //   goto TrueBlock;
1676 
1677   if (LOp == BO_LAnd) {
1678     ThenBlock = CounterIncrBlock;
1679     ElseBlock = FalseBlock;
1680     NextBlock = TrueBlock;
1681   }
1682 
1683   // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1684   // we need to evaluate the condition and increment the counter on FALSE:
1685   //
1686   // if (Cond)
1687   //   goto TrueBlock;
1688   // else
1689   //   goto CounterIncrBlock;
1690   //
1691   // CounterIncrBlock:
1692   //   Counter++;
1693   //   goto FalseBlock;
1694 
1695   else if (LOp == BO_LOr) {
1696     ThenBlock = TrueBlock;
1697     ElseBlock = CounterIncrBlock;
1698     NextBlock = FalseBlock;
1699   } else {
1700     llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1701   }
1702 
1703   // Emit Branch based on condition.
1704   EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, TrueCount, LH);
1705 
1706   // Emit the block containing the counter increment(s).
1707   EmitBlock(CounterIncrBlock);
1708 
1709   // Increment corresponding counter; if index not provided, use Cond as index.
1710   incrementProfileCounter(CntrIdx ? CntrIdx : Cond);
1711 
1712   // Go to the next block.
1713   EmitBranch(NextBlock);
1714 }
1715 
1716 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1717 /// statement) to the specified blocks.  Based on the condition, this might try
1718 /// to simplify the codegen of the conditional based on the branch.
1719 /// \param LH The value of the likelihood attribute on the True branch.
1720 void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
1721                                            llvm::BasicBlock *TrueBlock,
1722                                            llvm::BasicBlock *FalseBlock,
1723                                            uint64_t TrueCount,
1724                                            Stmt::Likelihood LH) {
1725   Cond = Cond->IgnoreParens();
1726 
1727   if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
1728 
1729     // Handle X && Y in a condition.
1730     if (CondBOp->getOpcode() == BO_LAnd) {
1731       // If we have "1 && X", simplify the code.  "0 && X" would have constant
1732       // folded if the case was simple enough.
1733       bool ConstantBool = false;
1734       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1735           ConstantBool) {
1736         // br(1 && X) -> br(X).
1737         incrementProfileCounter(CondBOp);
1738         return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1739                                         FalseBlock, TrueCount, LH);
1740       }
1741 
1742       // If we have "X && 1", simplify the code to use an uncond branch.
1743       // "X && 0" would have been constant folded to 0.
1744       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1745           ConstantBool) {
1746         // br(X && 1) -> br(X).
1747         return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock,
1748                                         FalseBlock, TrueCount, LH, CondBOp);
1749       }
1750 
1751       // Emit the LHS as a conditional.  If the LHS conditional is false, we
1752       // want to jump to the FalseBlock.
1753       llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
1754       // The counter tells us how often we evaluate RHS, and all of TrueCount
1755       // can be propagated to that branch.
1756       uint64_t RHSCount = getProfileCount(CondBOp->getRHS());
1757 
1758       ConditionalEvaluation eval(*this);
1759       {
1760         ApplyDebugLocation DL(*this, Cond);
1761         // Propagate the likelihood attribute like __builtin_expect
1762         // __builtin_expect(X && Y, 1) -> X and Y are likely
1763         // __builtin_expect(X && Y, 0) -> only Y is unlikely
1764         EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount,
1765                              LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1766         EmitBlock(LHSTrue);
1767       }
1768 
1769       incrementProfileCounter(CondBOp);
1770       setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1771 
1772       // Any temporaries created here are conditional.
1773       eval.begin(*this);
1774       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1775                                FalseBlock, TrueCount, LH);
1776       eval.end(*this);
1777 
1778       return;
1779     }
1780 
1781     if (CondBOp->getOpcode() == BO_LOr) {
1782       // If we have "0 || X", simplify the code.  "1 || X" would have constant
1783       // folded if the case was simple enough.
1784       bool ConstantBool = false;
1785       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1786           !ConstantBool) {
1787         // br(0 || X) -> br(X).
1788         incrementProfileCounter(CondBOp);
1789         return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock,
1790                                         FalseBlock, TrueCount, LH);
1791       }
1792 
1793       // If we have "X || 0", simplify the code to use an uncond branch.
1794       // "X || 1" would have been constant folded to 1.
1795       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1796           !ConstantBool) {
1797         // br(X || 0) -> br(X).
1798         return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock,
1799                                         FalseBlock, TrueCount, LH, CondBOp);
1800       }
1801 
1802       // Emit the LHS as a conditional.  If the LHS conditional is true, we
1803       // want to jump to the TrueBlock.
1804       llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
1805       // We have the count for entry to the RHS and for the whole expression
1806       // being true, so we can divy up True count between the short circuit and
1807       // the RHS.
1808       uint64_t LHSCount =
1809           getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());
1810       uint64_t RHSCount = TrueCount - LHSCount;
1811 
1812       ConditionalEvaluation eval(*this);
1813       {
1814         // Propagate the likelihood attribute like __builtin_expect
1815         // __builtin_expect(X || Y, 1) -> only Y is likely
1816         // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
1817         ApplyDebugLocation DL(*this, Cond);
1818         EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount,
1819                              LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
1820         EmitBlock(LHSFalse);
1821       }
1822 
1823       incrementProfileCounter(CondBOp);
1824       setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1825 
1826       // Any temporaries created here are conditional.
1827       eval.begin(*this);
1828       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, FalseBlock,
1829                                RHSCount, LH);
1830 
1831       eval.end(*this);
1832 
1833       return;
1834     }
1835   }
1836 
1837   if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
1838     // br(!x, t, f) -> br(x, f, t)
1839     if (CondUOp->getOpcode() == UO_LNot) {
1840       // Negate the count.
1841       uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
1842       // The values of the enum are chosen to make this negation possible.
1843       LH = static_cast<Stmt::Likelihood>(-LH);
1844       // Negate the condition and swap the destination blocks.
1845       return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
1846                                   FalseCount, LH);
1847     }
1848   }
1849 
1850   if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {
1851     // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
1852     llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
1853     llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
1854 
1855     // The ConditionalOperator itself has no likelihood information for its
1856     // true and false branches. This matches the behavior of __builtin_expect.
1857     ConditionalEvaluation cond(*this);
1858     EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,
1859                          getProfileCount(CondOp), Stmt::LH_None);
1860 
1861     // When computing PGO branch weights, we only know the overall count for
1862     // the true block. This code is essentially doing tail duplication of the
1863     // naive code-gen, introducing new edges for which counts are not
1864     // available. Divide the counts proportionally between the LHS and RHS of
1865     // the conditional operator.
1866     uint64_t LHSScaledTrueCount = 0;
1867     if (TrueCount) {
1868       double LHSRatio =
1869           getProfileCount(CondOp) / (double)getCurrentProfileCount();
1870       LHSScaledTrueCount = TrueCount * LHSRatio;
1871     }
1872 
1873     cond.begin(*this);
1874     EmitBlock(LHSBlock);
1875     incrementProfileCounter(CondOp);
1876     {
1877       ApplyDebugLocation DL(*this, Cond);
1878       EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
1879                            LHSScaledTrueCount, LH);
1880     }
1881     cond.end(*this);
1882 
1883     cond.begin(*this);
1884     EmitBlock(RHSBlock);
1885     EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
1886                          TrueCount - LHSScaledTrueCount, LH);
1887     cond.end(*this);
1888 
1889     return;
1890   }
1891 
1892   if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {
1893     // Conditional operator handling can give us a throw expression as a
1894     // condition for a case like:
1895     //   br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
1896     // Fold this to:
1897     //   br(c, throw x, br(y, t, f))
1898     EmitCXXThrowExpr(Throw, /*KeepInsertionPoint*/false);
1899     return;
1900   }
1901 
1902   // Emit the code with the fully general case.
1903   llvm::Value *CondV;
1904   {
1905     ApplyDebugLocation DL(*this, Cond);
1906     CondV = EvaluateExprAsBool(Cond);
1907   }
1908 
1909   llvm::MDNode *Weights = nullptr;
1910   llvm::MDNode *Unpredictable = nullptr;
1911 
1912   // If the branch has a condition wrapped by __builtin_unpredictable,
1913   // create metadata that specifies that the branch is unpredictable.
1914   // Don't bother if not optimizing because that metadata would not be used.
1915   auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
1916   if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
1917     auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1918     if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1919       llvm::MDBuilder MDHelper(getLLVMContext());
1920       Unpredictable = MDHelper.createUnpredictable();
1921     }
1922   }
1923 
1924   // If there is a Likelihood knowledge for the cond, lower it.
1925   // Note that if not optimizing this won't emit anything.
1926   llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
1927   if (CondV != NewCondV)
1928     CondV = NewCondV;
1929   else {
1930     // Otherwise, lower profile counts. Note that we do this even at -O0.
1931     uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
1932     Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
1933   }
1934 
1935   Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
1936 }
1937 
1938 /// ErrorUnsupported - Print out an error that codegen doesn't support the
1939 /// specified stmt yet.
1940 void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
1941   CGM.ErrorUnsupported(S, Type);
1942 }
1943 
1944 /// emitNonZeroVLAInit - Emit the "zero" initialization of a
1945 /// variable-length array whose elements have a non-zero bit-pattern.
1946 ///
1947 /// \param baseType the inner-most element type of the array
1948 /// \param src - a char* pointing to the bit-pattern for a single
1949 /// base element of the array
1950 /// \param sizeInChars - the total size of the VLA, in chars
1951 static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
1952                                Address dest, Address src,
1953                                llvm::Value *sizeInChars) {
1954   CGBuilderTy &Builder = CGF.Builder;
1955 
1956   CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);
1957   llvm::Value *baseSizeInChars
1958     = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
1959 
1960   Address begin = dest.withElementType(CGF.Int8Ty);
1961   llvm::Value *end = Builder.CreateInBoundsGEP(
1962       begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end");
1963 
1964   llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
1965   llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
1966   llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");
1967 
1968   // Make a loop over the VLA.  C99 guarantees that the VLA element
1969   // count must be nonzero.
1970   CGF.EmitBlock(loopBB);
1971 
1972   llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
1973   cur->addIncoming(begin.getPointer(), originBB);
1974 
1975   CharUnits curAlign =
1976     dest.getAlignment().alignmentOfArrayElement(baseSize);
1977 
1978   // memcpy the individual element bit-pattern.
1979   Builder.CreateMemCpy(Address(cur, CGF.Int8Ty, curAlign), src, baseSizeInChars,
1980                        /*volatile*/ false);
1981 
1982   // Go to the next element.
1983   llvm::Value *next =
1984     Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");
1985 
1986   // Leave if that's the end of the VLA.
1987   llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
1988   Builder.CreateCondBr(done, contBB, loopBB);
1989   cur->addIncoming(next, loopBB);
1990 
1991   CGF.EmitBlock(contBB);
1992 }
1993 
1994 void
1995 CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
1996   // Ignore empty classes in C++.
1997   if (getLangOpts().CPlusPlus) {
1998     if (const RecordType *RT = Ty->getAs<RecordType>()) {
1999       if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
2000         return;
2001     }
2002   }
2003 
2004   if (DestPtr.getElementType() != Int8Ty)
2005     DestPtr = DestPtr.withElementType(Int8Ty);
2006 
2007   // Get size and alignment info for this aggregate.
2008   CharUnits size = getContext().getTypeSizeInChars(Ty);
2009 
2010   llvm::Value *SizeVal;
2011   const VariableArrayType *vla;
2012 
2013   // Don't bother emitting a zero-byte memset.
2014   if (size.isZero()) {
2015     // But note that getTypeInfo returns 0 for a VLA.
2016     if (const VariableArrayType *vlaType =
2017           dyn_cast_or_null<VariableArrayType>(
2018                                           getContext().getAsArrayType(Ty))) {
2019       auto VlaSize = getVLASize(vlaType);
2020       SizeVal = VlaSize.NumElts;
2021       CharUnits eltSize = getContext().getTypeSizeInChars(VlaSize.Type);
2022       if (!eltSize.isOne())
2023         SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(eltSize));
2024       vla = vlaType;
2025     } else {
2026       return;
2027     }
2028   } else {
2029     SizeVal = CGM.getSize(size);
2030     vla = nullptr;
2031   }
2032 
2033   // If the type contains a pointer to data member we can't memset it to zero.
2034   // Instead, create a null constant and copy it to the destination.
2035   // TODO: there are other patterns besides zero that we can usefully memset,
2036   // like -1, which happens to be the pattern used by member-pointers.
2037   if (!CGM.getTypes().isZeroInitializable(Ty)) {
2038     // For a VLA, emit a single element, then splat that over the VLA.
2039     if (vla) Ty = getContext().getBaseElementType(vla);
2040 
2041     llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
2042 
2043     llvm::GlobalVariable *NullVariable =
2044       new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2045                                /*isConstant=*/true,
2046                                llvm::GlobalVariable::PrivateLinkage,
2047                                NullConstant, Twine());
2048     CharUnits NullAlign = DestPtr.getAlignment();
2049     NullVariable->setAlignment(NullAlign.getAsAlign());
2050     Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2051 
2052     if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
2053 
2054     // Get and call the appropriate llvm.memcpy overload.
2055     Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
2056     return;
2057   }
2058 
2059   // Otherwise, just memset the whole thing to zero.  This is legal
2060   // because in LLVM, all default initializers (other than the ones we just
2061   // handled above) are guaranteed to have a bit pattern of all zeros.
2062   Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
2063 }
2064 
2065 llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2066   // Make sure that there is a block for the indirect goto.
2067   if (!IndirectBranch)
2068     GetIndirectGotoBlock();
2069 
2070   llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
2071 
2072   // Make sure the indirect branch includes all of the address-taken blocks.
2073   IndirectBranch->addDestination(BB);
2074   return llvm::BlockAddress::get(CurFn, BB);
2075 }
2076 
2077 llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2078   // If we already made the indirect branch for indirect goto, return its block.
2079   if (IndirectBranch) return IndirectBranch->getParent();
2080 
2081   CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));
2082 
2083   // Create the PHI node that indirect gotos will add entries to.
2084   llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,
2085                                               "indirect.goto.dest");
2086 
2087   // Create the indirect branch instruction.
2088   IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
2089   return IndirectBranch->getParent();
2090 }
2091 
2092 /// Computes the length of an array in elements, as well as the base
2093 /// element type and a properly-typed first element pointer.
2094 llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2095                                               QualType &baseType,
2096                                               Address &addr) {
2097   const ArrayType *arrayType = origArrayType;
2098 
2099   // If it's a VLA, we have to load the stored size.  Note that
2100   // this is the size of the VLA in bytes, not its size in elements.
2101   llvm::Value *numVLAElements = nullptr;
2102   if (isa<VariableArrayType>(arrayType)) {
2103     numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).NumElts;
2104 
2105     // Walk into all VLAs.  This doesn't require changes to addr,
2106     // which has type T* where T is the first non-VLA element type.
2107     do {
2108       QualType elementType = arrayType->getElementType();
2109       arrayType = getContext().getAsArrayType(elementType);
2110 
2111       // If we only have VLA components, 'addr' requires no adjustment.
2112       if (!arrayType) {
2113         baseType = elementType;
2114         return numVLAElements;
2115       }
2116     } while (isa<VariableArrayType>(arrayType));
2117 
2118     // We get out here only if we find a constant array type
2119     // inside the VLA.
2120   }
2121 
2122   // We have some number of constant-length arrays, so addr should
2123   // have LLVM type [M x [N x [...]]]*.  Build a GEP that walks
2124   // down to the first element of addr.
2125   SmallVector<llvm::Value*, 8> gepIndices;
2126 
2127   // GEP down to the array type.
2128   llvm::ConstantInt *zero = Builder.getInt32(0);
2129   gepIndices.push_back(zero);
2130 
2131   uint64_t countFromCLAs = 1;
2132   QualType eltType;
2133 
2134   llvm::ArrayType *llvmArrayType =
2135     dyn_cast<llvm::ArrayType>(addr.getElementType());
2136   while (llvmArrayType) {
2137     assert(isa<ConstantArrayType>(arrayType));
2138     assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
2139              == llvmArrayType->getNumElements());
2140 
2141     gepIndices.push_back(zero);
2142     countFromCLAs *= llvmArrayType->getNumElements();
2143     eltType = arrayType->getElementType();
2144 
2145     llvmArrayType =
2146       dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());
2147     arrayType = getContext().getAsArrayType(arrayType->getElementType());
2148     assert((!llvmArrayType || arrayType) &&
2149            "LLVM and Clang types are out-of-synch");
2150   }
2151 
2152   if (arrayType) {
2153     // From this point onwards, the Clang array type has been emitted
2154     // as some other type (probably a packed struct). Compute the array
2155     // size, and just emit the 'begin' expression as a bitcast.
2156     while (arrayType) {
2157       countFromCLAs *=
2158           cast<ConstantArrayType>(arrayType)->getSize().getZExtValue();
2159       eltType = arrayType->getElementType();
2160       arrayType = getContext().getAsArrayType(eltType);
2161     }
2162 
2163     llvm::Type *baseType = ConvertType(eltType);
2164     addr = addr.withElementType(baseType);
2165   } else {
2166     // Create the actual GEP.
2167     addr = Address(Builder.CreateInBoundsGEP(
2168         addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"),
2169         ConvertTypeForMem(eltType),
2170         addr.getAlignment());
2171   }
2172 
2173   baseType = eltType;
2174 
2175   llvm::Value *numElements
2176     = llvm::ConstantInt::get(SizeTy, countFromCLAs);
2177 
2178   // If we had any VLA dimensions, factor them in.
2179   if (numVLAElements)
2180     numElements = Builder.CreateNUWMul(numVLAElements, numElements);
2181 
2182   return numElements;
2183 }
2184 
2185 CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2186   const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2187   assert(vla && "type was not a variable array type!");
2188   return getVLASize(vla);
2189 }
2190 
2191 CodeGenFunction::VlaSizePair
2192 CodeGenFunction::getVLASize(const VariableArrayType *type) {
2193   // The number of elements so far; always size_t.
2194   llvm::Value *numElements = nullptr;
2195 
2196   QualType elementType;
2197   do {
2198     elementType = type->getElementType();
2199     llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2200     assert(vlaSize && "no size for VLA!");
2201     assert(vlaSize->getType() == SizeTy);
2202 
2203     if (!numElements) {
2204       numElements = vlaSize;
2205     } else {
2206       // It's undefined behavior if this wraps around, so mark it that way.
2207       // FIXME: Teach -fsanitize=undefined to trap this.
2208       numElements = Builder.CreateNUWMul(numElements, vlaSize);
2209     }
2210   } while ((type = getContext().getAsVariableArrayType(elementType)));
2211 
2212   return { numElements, elementType };
2213 }
2214 
2215 CodeGenFunction::VlaSizePair
2216 CodeGenFunction::getVLAElements1D(QualType type) {
2217   const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2218   assert(vla && "type was not a variable array type!");
2219   return getVLAElements1D(vla);
2220 }
2221 
2222 CodeGenFunction::VlaSizePair
2223 CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2224   llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2225   assert(VlaSize && "no size for VLA!");
2226   assert(VlaSize->getType() == SizeTy);
2227   return { VlaSize, Vla->getElementType() };
2228 }
2229 
2230 void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2231   assert(type->isVariablyModifiedType() &&
2232          "Must pass variably modified type to EmitVLASizes!");
2233 
2234   EnsureInsertPoint();
2235 
2236   // We're going to walk down into the type and look for VLA
2237   // expressions.
2238   do {
2239     assert(type->isVariablyModifiedType());
2240 
2241     const Type *ty = type.getTypePtr();
2242     switch (ty->getTypeClass()) {
2243 
2244 #define TYPE(Class, Base)
2245 #define ABSTRACT_TYPE(Class, Base)
2246 #define NON_CANONICAL_TYPE(Class, Base)
2247 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
2248 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2249 #include "clang/AST/TypeNodes.inc"
2250       llvm_unreachable("unexpected dependent type!");
2251 
2252     // These types are never variably-modified.
2253     case Type::Builtin:
2254     case Type::Complex:
2255     case Type::Vector:
2256     case Type::ExtVector:
2257     case Type::ConstantMatrix:
2258     case Type::Record:
2259     case Type::Enum:
2260     case Type::Using:
2261     case Type::TemplateSpecialization:
2262     case Type::ObjCTypeParam:
2263     case Type::ObjCObject:
2264     case Type::ObjCInterface:
2265     case Type::ObjCObjectPointer:
2266     case Type::BitInt:
2267       llvm_unreachable("type class is never variably-modified!");
2268 
2269     case Type::Elaborated:
2270       type = cast<ElaboratedType>(ty)->getNamedType();
2271       break;
2272 
2273     case Type::Adjusted:
2274       type = cast<AdjustedType>(ty)->getAdjustedType();
2275       break;
2276 
2277     case Type::Decayed:
2278       type = cast<DecayedType>(ty)->getPointeeType();
2279       break;
2280 
2281     case Type::Pointer:
2282       type = cast<PointerType>(ty)->getPointeeType();
2283       break;
2284 
2285     case Type::BlockPointer:
2286       type = cast<BlockPointerType>(ty)->getPointeeType();
2287       break;
2288 
2289     case Type::LValueReference:
2290     case Type::RValueReference:
2291       type = cast<ReferenceType>(ty)->getPointeeType();
2292       break;
2293 
2294     case Type::MemberPointer:
2295       type = cast<MemberPointerType>(ty)->getPointeeType();
2296       break;
2297 
2298     case Type::ConstantArray:
2299     case Type::IncompleteArray:
2300       // Losing element qualification here is fine.
2301       type = cast<ArrayType>(ty)->getElementType();
2302       break;
2303 
2304     case Type::VariableArray: {
2305       // Losing element qualification here is fine.
2306       const VariableArrayType *vat = cast<VariableArrayType>(ty);
2307 
2308       // Unknown size indication requires no size computation.
2309       // Otherwise, evaluate and record it.
2310       if (const Expr *sizeExpr = vat->getSizeExpr()) {
2311         // It's possible that we might have emitted this already,
2312         // e.g. with a typedef and a pointer to it.
2313         llvm::Value *&entry = VLASizeMap[sizeExpr];
2314         if (!entry) {
2315           llvm::Value *size = EmitScalarExpr(sizeExpr);
2316 
2317           // C11 6.7.6.2p5:
2318           //   If the size is an expression that is not an integer constant
2319           //   expression [...] each time it is evaluated it shall have a value
2320           //   greater than zero.
2321           if (SanOpts.has(SanitizerKind::VLABound)) {
2322             SanitizerScope SanScope(this);
2323             llvm::Value *Zero = llvm::Constant::getNullValue(size->getType());
2324             clang::QualType SEType = sizeExpr->getType();
2325             llvm::Value *CheckCondition =
2326                 SEType->isSignedIntegerType()
2327                     ? Builder.CreateICmpSGT(size, Zero)
2328                     : Builder.CreateICmpUGT(size, Zero);
2329             llvm::Constant *StaticArgs[] = {
2330                 EmitCheckSourceLocation(sizeExpr->getBeginLoc()),
2331                 EmitCheckTypeDescriptor(SEType)};
2332             EmitCheck(std::make_pair(CheckCondition, SanitizerKind::VLABound),
2333                       SanitizerHandler::VLABoundNotPositive, StaticArgs, size);
2334           }
2335 
2336           // Always zexting here would be wrong if it weren't
2337           // undefined behavior to have a negative bound.
2338           // FIXME: What about when size's type is larger than size_t?
2339           entry = Builder.CreateIntCast(size, SizeTy, /*signed*/ false);
2340         }
2341       }
2342       type = vat->getElementType();
2343       break;
2344     }
2345 
2346     case Type::FunctionProto:
2347     case Type::FunctionNoProto:
2348       type = cast<FunctionType>(ty)->getReturnType();
2349       break;
2350 
2351     case Type::Paren:
2352     case Type::TypeOf:
2353     case Type::UnaryTransform:
2354     case Type::Attributed:
2355     case Type::BTFTagAttributed:
2356     case Type::SubstTemplateTypeParm:
2357     case Type::MacroQualified:
2358       // Keep walking after single level desugaring.
2359       type = type.getSingleStepDesugaredType(getContext());
2360       break;
2361 
2362     case Type::Typedef:
2363     case Type::Decltype:
2364     case Type::Auto:
2365     case Type::DeducedTemplateSpecialization:
2366       // Stop walking: nothing to do.
2367       return;
2368 
2369     case Type::TypeOfExpr:
2370       // Stop walking: emit typeof expression.
2371       EmitIgnoredExpr(cast<TypeOfExprType>(ty)->getUnderlyingExpr());
2372       return;
2373 
2374     case Type::Atomic:
2375       type = cast<AtomicType>(ty)->getValueType();
2376       break;
2377 
2378     case Type::Pipe:
2379       type = cast<PipeType>(ty)->getElementType();
2380       break;
2381     }
2382   } while (type->isVariablyModifiedType());
2383 }
2384 
2385 Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2386   if (getContext().getBuiltinVaListType()->isArrayType())
2387     return EmitPointerWithAlignment(E);
2388   return EmitLValue(E).getAddress(*this);
2389 }
2390 
2391 Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2392   return EmitLValue(E).getAddress(*this);
2393 }
2394 
2395 void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2396                                               const APValue &Init) {
2397   assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2398   if (CGDebugInfo *Dbg = getDebugInfo())
2399     if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2400       Dbg->EmitGlobalVariable(E->getDecl(), Init);
2401 }
2402 
2403 CodeGenFunction::PeepholeProtection
2404 CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2405   // At the moment, the only aggressive peephole we do in IR gen
2406   // is trunc(zext) folding, but if we add more, we can easily
2407   // extend this protection.
2408 
2409   if (!rvalue.isScalar()) return PeepholeProtection();
2410   llvm::Value *value = rvalue.getScalarVal();
2411   if (!isa<llvm::ZExtInst>(value)) return PeepholeProtection();
2412 
2413   // Just make an extra bitcast.
2414   assert(HaveInsertPoint());
2415   llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2416                                                   Builder.GetInsertBlock());
2417 
2418   PeepholeProtection protection;
2419   protection.Inst = inst;
2420   return protection;
2421 }
2422 
2423 void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2424   if (!protection.Inst) return;
2425 
2426   // In theory, we could try to duplicate the peepholes now, but whatever.
2427   protection.Inst->eraseFromParent();
2428 }
2429 
2430 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2431                                               QualType Ty, SourceLocation Loc,
2432                                               SourceLocation AssumptionLoc,
2433                                               llvm::Value *Alignment,
2434                                               llvm::Value *OffsetValue) {
2435   if (Alignment->getType() != IntPtrTy)
2436     Alignment =
2437         Builder.CreateIntCast(Alignment, IntPtrTy, false, "casted.align");
2438   if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2439     OffsetValue =
2440         Builder.CreateIntCast(OffsetValue, IntPtrTy, true, "casted.offset");
2441   llvm::Value *TheCheck = nullptr;
2442   if (SanOpts.has(SanitizerKind::Alignment)) {
2443     llvm::Value *PtrIntValue =
2444         Builder.CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");
2445 
2446     if (OffsetValue) {
2447       bool IsOffsetZero = false;
2448       if (const auto *CI = dyn_cast<llvm::ConstantInt>(OffsetValue))
2449         IsOffsetZero = CI->isZero();
2450 
2451       if (!IsOffsetZero)
2452         PtrIntValue = Builder.CreateSub(PtrIntValue, OffsetValue, "offsetptr");
2453     }
2454 
2455     llvm::Value *Zero = llvm::ConstantInt::get(IntPtrTy, 0);
2456     llvm::Value *Mask =
2457         Builder.CreateSub(Alignment, llvm::ConstantInt::get(IntPtrTy, 1));
2458     llvm::Value *MaskedPtr = Builder.CreateAnd(PtrIntValue, Mask, "maskedptr");
2459     TheCheck = Builder.CreateICmpEQ(MaskedPtr, Zero, "maskcond");
2460   }
2461   llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2462       CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2463 
2464   if (!SanOpts.has(SanitizerKind::Alignment))
2465     return;
2466   emitAlignmentAssumptionCheck(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2467                                OffsetValue, TheCheck, Assumption);
2468 }
2469 
2470 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2471                                               const Expr *E,
2472                                               SourceLocation AssumptionLoc,
2473                                               llvm::Value *Alignment,
2474                                               llvm::Value *OffsetValue) {
2475   QualType Ty = E->getType();
2476   SourceLocation Loc = E->getExprLoc();
2477 
2478   emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2479                           OffsetValue);
2480 }
2481 
2482 llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2483                                                  llvm::Value *AnnotatedVal,
2484                                                  StringRef AnnotationStr,
2485                                                  SourceLocation Location,
2486                                                  const AnnotateAttr *Attr) {
2487   SmallVector<llvm::Value *, 5> Args = {
2488       AnnotatedVal,
2489       CGM.EmitAnnotationString(AnnotationStr),
2490       CGM.EmitAnnotationUnit(Location),
2491       CGM.EmitAnnotationLineNo(Location),
2492   };
2493   if (Attr)
2494     Args.push_back(CGM.EmitAnnotationArgs(Attr));
2495   return Builder.CreateCall(AnnotationFn, Args);
2496 }
2497 
2498 void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2499   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2500   for (const auto *I : D->specific_attrs<AnnotateAttr>())
2501     EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation,
2502                                         {V->getType(), CGM.ConstGlobalsPtrTy}),
2503                        V, I->getAnnotation(), D->getLocation(), I);
2504 }
2505 
2506 Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2507                                               Address Addr) {
2508   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2509   llvm::Value *V = Addr.getPointer();
2510   llvm::Type *VTy = V->getType();
2511   auto *PTy = dyn_cast<llvm::PointerType>(VTy);
2512   unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2513   llvm::PointerType *IntrinTy =
2514       llvm::PointerType::get(CGM.getLLVMContext(), AS);
2515   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
2516                                        {IntrinTy, CGM.ConstGlobalsPtrTy});
2517 
2518   for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2519     // FIXME Always emit the cast inst so we can differentiate between
2520     // annotation on the first field of a struct and annotation on the struct
2521     // itself.
2522     if (VTy != IntrinTy)
2523       V = Builder.CreateBitCast(V, IntrinTy);
2524     V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation(), I);
2525     V = Builder.CreateBitCast(V, VTy);
2526   }
2527 
2528   return Address(V, Addr.getElementType(), Addr.getAlignment());
2529 }
2530 
2531 CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2532 
2533 CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2534     : CGF(CGF) {
2535   assert(!CGF->IsSanitizerScope);
2536   CGF->IsSanitizerScope = true;
2537 }
2538 
2539 CodeGenFunction::SanitizerScope::~SanitizerScope() {
2540   CGF->IsSanitizerScope = false;
2541 }
2542 
2543 void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2544                                    const llvm::Twine &Name,
2545                                    llvm::BasicBlock *BB,
2546                                    llvm::BasicBlock::iterator InsertPt) const {
2547   LoopStack.InsertHelper(I);
2548   if (IsSanitizerScope)
2549     I->setNoSanitizeMetadata();
2550 }
2551 
2552 void CGBuilderInserter::InsertHelper(
2553     llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
2554     llvm::BasicBlock::iterator InsertPt) const {
2555   llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
2556   if (CGF)
2557     CGF->InsertHelper(I, Name, BB, InsertPt);
2558 }
2559 
2560 // Emits an error if we don't have a valid set of target features for the
2561 // called function.
2562 void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2563                                           const FunctionDecl *TargetDecl) {
2564   return checkTargetFeatures(E->getBeginLoc(), TargetDecl);
2565 }
2566 
2567 // Emits an error if we don't have a valid set of target features for the
2568 // called function.
2569 void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2570                                           const FunctionDecl *TargetDecl) {
2571   // Early exit if this is an indirect call.
2572   if (!TargetDecl)
2573     return;
2574 
2575   // Get the current enclosing function if it exists. If it doesn't
2576   // we can't check the target features anyhow.
2577   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);
2578   if (!FD)
2579     return;
2580 
2581   // Grab the required features for the call. For a builtin this is listed in
2582   // the td file with the default cpu, for an always_inline function this is any
2583   // listed cpu and any listed features.
2584   unsigned BuiltinID = TargetDecl->getBuiltinID();
2585   std::string MissingFeature;
2586   llvm::StringMap<bool> CallerFeatureMap;
2587   CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
2588   // When compiling in HipStdPar mode we have to be conservative in rejecting
2589   // target specific features in the FE, and defer the possible error to the
2590   // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
2591   // referenced by an accelerator executable function, we emit an error.
2592   bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2593   if (BuiltinID) {
2594     StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
2595     if (!Builtin::evaluateRequiredTargetFeatures(
2596         FeatureList, CallerFeatureMap) && !IsHipStdPar) {
2597       CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
2598           << TargetDecl->getDeclName()
2599           << FeatureList;
2600     }
2601   } else if (!TargetDecl->isMultiVersion() &&
2602              TargetDecl->hasAttr<TargetAttr>()) {
2603     // Get the required features for the callee.
2604 
2605     const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2606     ParsedTargetAttr ParsedAttr =
2607         CGM.getContext().filterFunctionTargetAttrs(TD);
2608 
2609     SmallVector<StringRef, 1> ReqFeatures;
2610     llvm::StringMap<bool> CalleeFeatureMap;
2611     CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
2612 
2613     for (const auto &F : ParsedAttr.Features) {
2614       if (F[0] == '+' && CalleeFeatureMap.lookup(F.substr(1)))
2615         ReqFeatures.push_back(StringRef(F).substr(1));
2616     }
2617 
2618     for (const auto &F : CalleeFeatureMap) {
2619       // Only positive features are "required".
2620       if (F.getValue())
2621         ReqFeatures.push_back(F.getKey());
2622     }
2623     if (!llvm::all_of(ReqFeatures, [&](StringRef Feature) {
2624       if (!CallerFeatureMap.lookup(Feature)) {
2625         MissingFeature = Feature.str();
2626         return false;
2627       }
2628       return true;
2629     }) && !IsHipStdPar)
2630       CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2631           << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2632   } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
2633     llvm::StringMap<bool> CalleeFeatureMap;
2634     CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
2635 
2636     for (const auto &F : CalleeFeatureMap) {
2637       if (F.getValue() && (!CallerFeatureMap.lookup(F.getKey()) ||
2638                            !CallerFeatureMap.find(F.getKey())->getValue()) &&
2639           !IsHipStdPar)
2640         CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2641             << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2642     }
2643   }
2644 }
2645 
2646 void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2647   if (!CGM.getCodeGenOpts().SanitizeStats)
2648     return;
2649 
2650   llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2651   IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2652   CGM.getSanStats().create(IRB, SSK);
2653 }
2654 
2655 void CodeGenFunction::EmitKCFIOperandBundle(
2656     const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2657   const FunctionProtoType *FP =
2658       Callee.getAbstractInfo().getCalleeFunctionProtoType();
2659   if (FP)
2660     Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar()));
2661 }
2662 
2663 llvm::Value *CodeGenFunction::FormAArch64ResolverCondition(
2664     const MultiVersionResolverOption &RO) {
2665   llvm::SmallVector<StringRef, 8> CondFeatures;
2666   for (const StringRef &Feature : RO.Conditions.Features) {
2667     // Form condition for features which are not yet enabled in target
2668     if (!getContext().getTargetInfo().hasFeature(Feature))
2669       CondFeatures.push_back(Feature);
2670   }
2671   if (!CondFeatures.empty()) {
2672     return EmitAArch64CpuSupports(CondFeatures);
2673   }
2674   return nullptr;
2675 }
2676 
2677 llvm::Value *CodeGenFunction::FormX86ResolverCondition(
2678     const MultiVersionResolverOption &RO) {
2679   llvm::Value *Condition = nullptr;
2680 
2681   if (!RO.Conditions.Architecture.empty()) {
2682     StringRef Arch = RO.Conditions.Architecture;
2683     // If arch= specifies an x86-64 micro-architecture level, test the feature
2684     // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
2685     if (Arch.starts_with("x86-64"))
2686       Condition = EmitX86CpuSupports({Arch});
2687     else
2688       Condition = EmitX86CpuIs(Arch);
2689   }
2690 
2691   if (!RO.Conditions.Features.empty()) {
2692     llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
2693     Condition =
2694         Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
2695   }
2696   return Condition;
2697 }
2698 
2699 static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2700                                              llvm::Function *Resolver,
2701                                              CGBuilderTy &Builder,
2702                                              llvm::Function *FuncToReturn,
2703                                              bool SupportsIFunc) {
2704   if (SupportsIFunc) {
2705     Builder.CreateRet(FuncToReturn);
2706     return;
2707   }
2708 
2709   llvm::SmallVector<llvm::Value *, 10> Args(
2710       llvm::make_pointer_range(Resolver->args()));
2711 
2712   llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
2713   Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2714 
2715   if (Resolver->getReturnType()->isVoidTy())
2716     Builder.CreateRetVoid();
2717   else
2718     Builder.CreateRet(Result);
2719 }
2720 
2721 void CodeGenFunction::EmitMultiVersionResolver(
2722     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2723 
2724   llvm::Triple::ArchType ArchType =
2725       getContext().getTargetInfo().getTriple().getArch();
2726 
2727   switch (ArchType) {
2728   case llvm::Triple::x86:
2729   case llvm::Triple::x86_64:
2730     EmitX86MultiVersionResolver(Resolver, Options);
2731     return;
2732   case llvm::Triple::aarch64:
2733     EmitAArch64MultiVersionResolver(Resolver, Options);
2734     return;
2735 
2736   default:
2737     assert(false && "Only implemented for x86 and AArch64 targets");
2738   }
2739 }
2740 
2741 void CodeGenFunction::EmitAArch64MultiVersionResolver(
2742     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2743   assert(!Options.empty() && "No multiversion resolver options found");
2744   assert(Options.back().Conditions.Features.size() == 0 &&
2745          "Default case must be last");
2746   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2747   assert(SupportsIFunc &&
2748          "Multiversion resolver requires target IFUNC support");
2749   bool AArch64CpuInitialized = false;
2750   llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2751 
2752   for (const MultiVersionResolverOption &RO : Options) {
2753     Builder.SetInsertPoint(CurBlock);
2754     llvm::Value *Condition = FormAArch64ResolverCondition(RO);
2755 
2756     // The 'default' or 'all features enabled' case.
2757     if (!Condition) {
2758       CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2759                                        SupportsIFunc);
2760       return;
2761     }
2762 
2763     if (!AArch64CpuInitialized) {
2764       Builder.SetInsertPoint(CurBlock, CurBlock->begin());
2765       EmitAArch64CpuInit();
2766       AArch64CpuInitialized = true;
2767       Builder.SetInsertPoint(CurBlock);
2768     }
2769 
2770     llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2771     CGBuilderTy RetBuilder(*this, RetBlock);
2772     CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2773                                      SupportsIFunc);
2774     CurBlock = createBasicBlock("resolver_else", Resolver);
2775     Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2776   }
2777 
2778   // If no default, emit an unreachable.
2779   Builder.SetInsertPoint(CurBlock);
2780   llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2781   TrapCall->setDoesNotReturn();
2782   TrapCall->setDoesNotThrow();
2783   Builder.CreateUnreachable();
2784   Builder.ClearInsertionPoint();
2785 }
2786 
2787 void CodeGenFunction::EmitX86MultiVersionResolver(
2788     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2789 
2790   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2791 
2792   // Main function's basic block.
2793   llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2794   Builder.SetInsertPoint(CurBlock);
2795   EmitX86CpuInit();
2796 
2797   for (const MultiVersionResolverOption &RO : Options) {
2798     Builder.SetInsertPoint(CurBlock);
2799     llvm::Value *Condition = FormX86ResolverCondition(RO);
2800 
2801     // The 'default' or 'generic' case.
2802     if (!Condition) {
2803       assert(&RO == Options.end() - 1 &&
2804              "Default or Generic case must be last");
2805       CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2806                                        SupportsIFunc);
2807       return;
2808     }
2809 
2810     llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2811     CGBuilderTy RetBuilder(*this, RetBlock);
2812     CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2813                                      SupportsIFunc);
2814     CurBlock = createBasicBlock("resolver_else", Resolver);
2815     Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2816   }
2817 
2818   // If no generic/default, emit an unreachable.
2819   Builder.SetInsertPoint(CurBlock);
2820   llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2821   TrapCall->setDoesNotReturn();
2822   TrapCall->setDoesNotThrow();
2823   Builder.CreateUnreachable();
2824   Builder.ClearInsertionPoint();
2825 }
2826 
2827 // Loc - where the diagnostic will point, where in the source code this
2828 //  alignment has failed.
2829 // SecondaryLoc - if present (will be present if sufficiently different from
2830 //  Loc), the diagnostic will additionally point a "Note:" to this location.
2831 //  It should be the location where the __attribute__((assume_aligned))
2832 //  was written e.g.
2833 void CodeGenFunction::emitAlignmentAssumptionCheck(
2834     llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
2835     SourceLocation SecondaryLoc, llvm::Value *Alignment,
2836     llvm::Value *OffsetValue, llvm::Value *TheCheck,
2837     llvm::Instruction *Assumption) {
2838   assert(Assumption && isa<llvm::CallInst>(Assumption) &&
2839          cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
2840              llvm::Intrinsic::getDeclaration(
2841                  Builder.GetInsertBlock()->getParent()->getParent(),
2842                  llvm::Intrinsic::assume) &&
2843          "Assumption should be a call to llvm.assume().");
2844   assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
2845          "Assumption should be the last instruction of the basic block, "
2846          "since the basic block is still being generated.");
2847 
2848   if (!SanOpts.has(SanitizerKind::Alignment))
2849     return;
2850 
2851   // Don't check pointers to volatile data. The behavior here is implementation-
2852   // defined.
2853   if (Ty->getPointeeType().isVolatileQualified())
2854     return;
2855 
2856   // We need to temorairly remove the assumption so we can insert the
2857   // sanitizer check before it, else the check will be dropped by optimizations.
2858   Assumption->removeFromParent();
2859 
2860   {
2861     SanitizerScope SanScope(this);
2862 
2863     if (!OffsetValue)
2864       OffsetValue = Builder.getInt1(false); // no offset.
2865 
2866     llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
2867                                     EmitCheckSourceLocation(SecondaryLoc),
2868                                     EmitCheckTypeDescriptor(Ty)};
2869     llvm::Value *DynamicData[] = {EmitCheckValue(Ptr),
2870                                   EmitCheckValue(Alignment),
2871                                   EmitCheckValue(OffsetValue)};
2872     EmitCheck({std::make_pair(TheCheck, SanitizerKind::Alignment)},
2873               SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
2874   }
2875 
2876   // We are now in the (new, empty) "cont" basic block.
2877   // Reintroduce the assumption.
2878   Builder.Insert(Assumption);
2879   // FIXME: Assumption still has it's original basic block as it's Parent.
2880 }
2881 
2882 llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
2883   if (CGDebugInfo *DI = getDebugInfo())
2884     return DI->SourceLocToDebugLoc(Location);
2885 
2886   return llvm::DebugLoc();
2887 }
2888 
2889 llvm::Value *
2890 CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
2891                                                       Stmt::Likelihood LH) {
2892   switch (LH) {
2893   case Stmt::LH_None:
2894     return Cond;
2895   case Stmt::LH_Likely:
2896   case Stmt::LH_Unlikely:
2897     // Don't generate llvm.expect on -O0 as the backend won't use it for
2898     // anything.
2899     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2900       return Cond;
2901     llvm::Type *CondTy = Cond->getType();
2902     assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
2903     llvm::Function *FnExpect =
2904         CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
2905     llvm::Value *ExpectedValueOfCond =
2906         llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
2907     return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
2908                               Cond->getName() + ".expval");
2909   }
2910   llvm_unreachable("Unknown Likelihood");
2911 }
2912 
2913 llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
2914                                                     unsigned NumElementsDst,
2915                                                     const llvm::Twine &Name) {
2916   auto *SrcTy = cast<llvm::FixedVectorType>(SrcVec->getType());
2917   unsigned NumElementsSrc = SrcTy->getNumElements();
2918   if (NumElementsSrc == NumElementsDst)
2919     return SrcVec;
2920 
2921   std::vector<int> ShuffleMask(NumElementsDst, -1);
2922   for (unsigned MaskIdx = 0;
2923        MaskIdx < std::min<>(NumElementsDst, NumElementsSrc); ++MaskIdx)
2924     ShuffleMask[MaskIdx] = MaskIdx;
2925 
2926   return Builder.CreateShuffleVector(SrcVec, ShuffleMask, Name);
2927 }
2928