xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp (revision 95eb4b873b6a8b527c5bd78d7191975dfca38998)
1 //===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This coordinates the per-function state used while generating code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CodeGenFunction.h"
14 #include "CGBlocks.h"
15 #include "CGCUDARuntime.h"
16 #include "CGCXXABI.h"
17 #include "CGCleanup.h"
18 #include "CGDebugInfo.h"
19 #include "CGHLSLRuntime.h"
20 #include "CGOpenMPRuntime.h"
21 #include "CodeGenModule.h"
22 #include "CodeGenPGO.h"
23 #include "TargetInfo.h"
24 #include "clang/AST/ASTContext.h"
25 #include "clang/AST/ASTLambda.h"
26 #include "clang/AST/Attr.h"
27 #include "clang/AST/Decl.h"
28 #include "clang/AST/DeclCXX.h"
29 #include "clang/AST/Expr.h"
30 #include "clang/AST/StmtCXX.h"
31 #include "clang/AST/StmtObjC.h"
32 #include "clang/Basic/Builtins.h"
33 #include "clang/Basic/CodeGenOptions.h"
34 #include "clang/Basic/TargetInfo.h"
35 #include "clang/CodeGen/CGFunctionInfo.h"
36 #include "clang/Frontend/FrontendDiagnostic.h"
37 #include "llvm/ADT/ArrayRef.h"
38 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
39 #include "llvm/IR/DataLayout.h"
40 #include "llvm/IR/Dominators.h"
41 #include "llvm/IR/FPEnv.h"
42 #include "llvm/IR/IntrinsicInst.h"
43 #include "llvm/IR/Intrinsics.h"
44 #include "llvm/IR/MDBuilder.h"
45 #include "llvm/IR/Operator.h"
46 #include "llvm/Support/CRC.h"
47 #include "llvm/Support/xxhash.h"
48 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
49 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
50 #include <optional>
51 
52 using namespace clang;
53 using namespace CodeGen;
54 
55 /// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
56 /// markers.
57 static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
58                                       const LangOptions &LangOpts) {
59   if (CGOpts.DisableLifetimeMarkers)
60     return false;
61 
62   // Sanitizers may use markers.
63   if (CGOpts.SanitizeAddressUseAfterScope ||
64       LangOpts.Sanitize.has(SanitizerKind::HWAddress) ||
65       LangOpts.Sanitize.has(SanitizerKind::Memory))
66     return true;
67 
68   // For now, only in optimized builds.
69   return CGOpts.OptimizationLevel != 0;
70 }
71 
72 CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
73     : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
74       Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
75               CGBuilderInserterTy(this)),
76       SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
77       DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
78       ShouldEmitLifetimeMarkers(
79           shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {
80   if (!suppressNewContext)
81     CGM.getCXXABI().getMangleContext().startNewFunction();
82   EHStack.setCGF(this);
83 
84   SetFastMathFlags(CurFPFeatures);
85 }
86 
87 CodeGenFunction::~CodeGenFunction() {
88   assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
89 
90   if (getLangOpts().OpenMP && CurFn)
91     CGM.getOpenMPRuntime().functionFinished(*this);
92 
93   // If we have an OpenMPIRBuilder we want to finalize functions (incl.
94   // outlining etc) at some point. Doing it once the function codegen is done
95   // seems to be a reasonable spot. We do it here, as opposed to the deletion
96   // time of the CodeGenModule, because we have to ensure the IR has not yet
97   // been "emitted" to the outside, thus, modifications are still sensible.
98   if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
99     CGM.getOpenMPRuntime().getOMPBuilder().finalize(CurFn);
100 }
101 
102 // Map the LangOption for exception behavior into
103 // the corresponding enum in the IR.
104 llvm::fp::ExceptionBehavior
105 clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
106 
107   switch (Kind) {
108   case LangOptions::FPE_Ignore:  return llvm::fp::ebIgnore;
109   case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
110   case LangOptions::FPE_Strict:  return llvm::fp::ebStrict;
111   default:
112     llvm_unreachable("Unsupported FP Exception Behavior");
113   }
114 }
115 
116 void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
117   llvm::FastMathFlags FMF;
118   FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
119   FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
120   FMF.setNoInfs(FPFeatures.getNoHonorInfs());
121   FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
122   FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
123   FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
124   FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
125   Builder.setFastMathFlags(FMF);
126 }
127 
128 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
129                                                   const Expr *E)
130     : CGF(CGF) {
131   ConstructorHelper(E->getFPFeaturesInEffect(CGF.getLangOpts()));
132 }
133 
134 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
135                                                   FPOptions FPFeatures)
136     : CGF(CGF) {
137   ConstructorHelper(FPFeatures);
138 }
139 
140 void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
141   OldFPFeatures = CGF.CurFPFeatures;
142   CGF.CurFPFeatures = FPFeatures;
143 
144   OldExcept = CGF.Builder.getDefaultConstrainedExcept();
145   OldRounding = CGF.Builder.getDefaultConstrainedRounding();
146 
147   if (OldFPFeatures == FPFeatures)
148     return;
149 
150   FMFGuard.emplace(CGF.Builder);
151 
152   llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
153   CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
154   auto NewExceptionBehavior =
155       ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>(
156           FPFeatures.getExceptionMode()));
157   CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
158 
159   CGF.SetFastMathFlags(FPFeatures);
160 
161   assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
162           isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
163           isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
164           (NewExceptionBehavior == llvm::fp::ebIgnore &&
165            NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
166          "FPConstrained should be enabled on entire function");
167 
168   auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
169     auto OldValue =
170         CGF.CurFn->getFnAttribute(Name).getValueAsBool();
171     auto NewValue = OldValue & Value;
172     if (OldValue != NewValue)
173       CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));
174   };
175   mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
176   mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
177   mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
178   mergeFnAttrValue(
179       "unsafe-fp-math",
180       FPFeatures.getAllowFPReassociate() && FPFeatures.getAllowReciprocal() &&
181           FPFeatures.getAllowApproxFunc() && FPFeatures.getNoSignedZero() &&
182           FPFeatures.allowFPContractAcrossStatement());
183 }
184 
185 CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
186   CGF.CurFPFeatures = OldFPFeatures;
187   CGF.Builder.setDefaultConstrainedExcept(OldExcept);
188   CGF.Builder.setDefaultConstrainedRounding(OldRounding);
189 }
190 
191 LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
192   LValueBaseInfo BaseInfo;
193   TBAAAccessInfo TBAAInfo;
194   CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
195   Address Addr(V, ConvertTypeForMem(T), Alignment);
196   return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo);
197 }
198 
199 /// Given a value of type T* that may not be to a complete object,
200 /// construct an l-value with the natural pointee alignment of T.
201 LValue
202 CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
203   LValueBaseInfo BaseInfo;
204   TBAAAccessInfo TBAAInfo;
205   CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
206                                                 /* forPointeeType= */ true);
207   Address Addr(V, ConvertTypeForMem(T), Align);
208   return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
209 }
210 
211 
212 llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
213   return CGM.getTypes().ConvertTypeForMem(T);
214 }
215 
216 llvm::Type *CodeGenFunction::ConvertType(QualType T) {
217   return CGM.getTypes().ConvertType(T);
218 }
219 
220 TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
221   type = type.getCanonicalType();
222   while (true) {
223     switch (type->getTypeClass()) {
224 #define TYPE(name, parent)
225 #define ABSTRACT_TYPE(name, parent)
226 #define NON_CANONICAL_TYPE(name, parent) case Type::name:
227 #define DEPENDENT_TYPE(name, parent) case Type::name:
228 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
229 #include "clang/AST/TypeNodes.inc"
230       llvm_unreachable("non-canonical or dependent type in IR-generation");
231 
232     case Type::Auto:
233     case Type::DeducedTemplateSpecialization:
234       llvm_unreachable("undeduced type in IR-generation");
235 
236     // Various scalar types.
237     case Type::Builtin:
238     case Type::Pointer:
239     case Type::BlockPointer:
240     case Type::LValueReference:
241     case Type::RValueReference:
242     case Type::MemberPointer:
243     case Type::Vector:
244     case Type::ExtVector:
245     case Type::ConstantMatrix:
246     case Type::FunctionProto:
247     case Type::FunctionNoProto:
248     case Type::Enum:
249     case Type::ObjCObjectPointer:
250     case Type::Pipe:
251     case Type::BitInt:
252       return TEK_Scalar;
253 
254     // Complexes.
255     case Type::Complex:
256       return TEK_Complex;
257 
258     // Arrays, records, and Objective-C objects.
259     case Type::ConstantArray:
260     case Type::IncompleteArray:
261     case Type::VariableArray:
262     case Type::Record:
263     case Type::ObjCObject:
264     case Type::ObjCInterface:
265       return TEK_Aggregate;
266 
267     // We operate on atomic values according to their underlying type.
268     case Type::Atomic:
269       type = cast<AtomicType>(type)->getValueType();
270       continue;
271     }
272     llvm_unreachable("unknown type kind!");
273   }
274 }
275 
276 llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
277   // For cleanliness, we try to avoid emitting the return block for
278   // simple cases.
279   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
280 
281   if (CurBB) {
282     assert(!CurBB->getTerminator() && "Unexpected terminated block.");
283 
284     // We have a valid insert point, reuse it if it is empty or there are no
285     // explicit jumps to the return block.
286     if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
287       ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
288       delete ReturnBlock.getBlock();
289       ReturnBlock = JumpDest();
290     } else
291       EmitBlock(ReturnBlock.getBlock());
292     return llvm::DebugLoc();
293   }
294 
295   // Otherwise, if the return block is the target of a single direct
296   // branch then we can just put the code in that block instead. This
297   // cleans up functions which started with a unified return block.
298   if (ReturnBlock.getBlock()->hasOneUse()) {
299     llvm::BranchInst *BI =
300       dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());
301     if (BI && BI->isUnconditional() &&
302         BI->getSuccessor(0) == ReturnBlock.getBlock()) {
303       // Record/return the DebugLoc of the simple 'return' expression to be used
304       // later by the actual 'ret' instruction.
305       llvm::DebugLoc Loc = BI->getDebugLoc();
306       Builder.SetInsertPoint(BI->getParent());
307       BI->eraseFromParent();
308       delete ReturnBlock.getBlock();
309       ReturnBlock = JumpDest();
310       return Loc;
311     }
312   }
313 
314   // FIXME: We are at an unreachable point, there is no reason to emit the block
315   // unless it has uses. However, we still need a place to put the debug
316   // region.end for now.
317 
318   EmitBlock(ReturnBlock.getBlock());
319   return llvm::DebugLoc();
320 }
321 
322 static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
323   if (!BB) return;
324   if (!BB->use_empty()) {
325     CGF.CurFn->insert(CGF.CurFn->end(), BB);
326     return;
327   }
328   delete BB;
329 }
330 
331 void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
332   assert(BreakContinueStack.empty() &&
333          "mismatched push/pop in break/continue stack!");
334 
335   bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
336     && NumSimpleReturnExprs == NumReturnExprs
337     && ReturnBlock.getBlock()->use_empty();
338   // Usually the return expression is evaluated before the cleanup
339   // code.  If the function contains only a simple return statement,
340   // such as a constant, the location before the cleanup code becomes
341   // the last useful breakpoint in the function, because the simple
342   // return expression will be evaluated after the cleanup code. To be
343   // safe, set the debug location for cleanup code to the location of
344   // the return statement.  Otherwise the cleanup code should be at the
345   // end of the function's lexical scope.
346   //
347   // If there are multiple branches to the return block, the branch
348   // instructions will get the location of the return statements and
349   // all will be fine.
350   if (CGDebugInfo *DI = getDebugInfo()) {
351     if (OnlySimpleReturnStmts)
352       DI->EmitLocation(Builder, LastStopPoint);
353     else
354       DI->EmitLocation(Builder, EndLoc);
355   }
356 
357   // Pop any cleanups that might have been associated with the
358   // parameters.  Do this in whatever block we're currently in; it's
359   // important to do this before we enter the return block or return
360   // edges will be *really* confused.
361   bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
362   bool HasOnlyLifetimeMarkers =
363       HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth);
364   bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers;
365 
366   std::optional<ApplyDebugLocation> OAL;
367   if (HasCleanups) {
368     // Make sure the line table doesn't jump back into the body for
369     // the ret after it's been at EndLoc.
370     if (CGDebugInfo *DI = getDebugInfo()) {
371       if (OnlySimpleReturnStmts)
372         DI->EmitLocation(Builder, EndLoc);
373       else
374         // We may not have a valid end location. Try to apply it anyway, and
375         // fall back to an artificial location if needed.
376         OAL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc);
377     }
378 
379     PopCleanupBlocks(PrologueCleanupDepth);
380   }
381 
382   // Emit function epilog (to return).
383   llvm::DebugLoc Loc = EmitReturnBlock();
384 
385   if (ShouldInstrumentFunction()) {
386     if (CGM.getCodeGenOpts().InstrumentFunctions)
387       CurFn->addFnAttr("instrument-function-exit", "__cyg_profile_func_exit");
388     if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
389       CurFn->addFnAttr("instrument-function-exit-inlined",
390                        "__cyg_profile_func_exit");
391   }
392 
393   // Emit debug descriptor for function end.
394   if (CGDebugInfo *DI = getDebugInfo())
395     DI->EmitFunctionEnd(Builder, CurFn);
396 
397   // Reset the debug location to that of the simple 'return' expression, if any
398   // rather than that of the end of the function's scope '}'.
399   ApplyDebugLocation AL(*this, Loc);
400   EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc);
401   EmitEndEHSpec(CurCodeDecl);
402 
403   assert(EHStack.empty() &&
404          "did not remove all scopes from cleanup stack!");
405 
406   // If someone did an indirect goto, emit the indirect goto block at the end of
407   // the function.
408   if (IndirectBranch) {
409     EmitBlock(IndirectBranch->getParent());
410     Builder.ClearInsertionPoint();
411   }
412 
413   // If some of our locals escaped, insert a call to llvm.localescape in the
414   // entry block.
415   if (!EscapedLocals.empty()) {
416     // Invert the map from local to index into a simple vector. There should be
417     // no holes.
418     SmallVector<llvm::Value *, 4> EscapeArgs;
419     EscapeArgs.resize(EscapedLocals.size());
420     for (auto &Pair : EscapedLocals)
421       EscapeArgs[Pair.second] = Pair.first;
422     llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
423         &CGM.getModule(), llvm::Intrinsic::localescape);
424     CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
425   }
426 
427   // Remove the AllocaInsertPt instruction, which is just a convenience for us.
428   llvm::Instruction *Ptr = AllocaInsertPt;
429   AllocaInsertPt = nullptr;
430   Ptr->eraseFromParent();
431 
432   // PostAllocaInsertPt, if created, was lazily created when it was required,
433   // remove it now since it was just created for our own convenience.
434   if (PostAllocaInsertPt) {
435     llvm::Instruction *PostPtr = PostAllocaInsertPt;
436     PostAllocaInsertPt = nullptr;
437     PostPtr->eraseFromParent();
438   }
439 
440   // If someone took the address of a label but never did an indirect goto, we
441   // made a zero entry PHI node, which is illegal, zap it now.
442   if (IndirectBranch) {
443     llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());
444     if (PN->getNumIncomingValues() == 0) {
445       PN->replaceAllUsesWith(llvm::UndefValue::get(PN->getType()));
446       PN->eraseFromParent();
447     }
448   }
449 
450   EmitIfUsed(*this, EHResumeBlock);
451   EmitIfUsed(*this, TerminateLandingPad);
452   EmitIfUsed(*this, TerminateHandler);
453   EmitIfUsed(*this, UnreachableBlock);
454 
455   for (const auto &FuncletAndParent : TerminateFunclets)
456     EmitIfUsed(*this, FuncletAndParent.second);
457 
458   if (CGM.getCodeGenOpts().EmitDeclMetadata)
459     EmitDeclMetadata();
460 
461   for (const auto &R : DeferredReplacements) {
462     if (llvm::Value *Old = R.first) {
463       Old->replaceAllUsesWith(R.second);
464       cast<llvm::Instruction>(Old)->eraseFromParent();
465     }
466   }
467   DeferredReplacements.clear();
468 
469   // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
470   // PHIs if the current function is a coroutine. We don't do it for all
471   // functions as it may result in slight increase in numbers of instructions
472   // if compiled with no optimizations. We do it for coroutine as the lifetime
473   // of CleanupDestSlot alloca make correct coroutine frame building very
474   // difficult.
475   if (NormalCleanupDest.isValid() && isCoroutine()) {
476     llvm::DominatorTree DT(*CurFn);
477     llvm::PromoteMemToReg(
478         cast<llvm::AllocaInst>(NormalCleanupDest.getPointer()), DT);
479     NormalCleanupDest = Address::invalid();
480   }
481 
482   // Scan function arguments for vector width.
483   for (llvm::Argument &A : CurFn->args())
484     if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
485       LargestVectorWidth =
486           std::max((uint64_t)LargestVectorWidth,
487                    VT->getPrimitiveSizeInBits().getKnownMinValue());
488 
489   // Update vector width based on return type.
490   if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))
491     LargestVectorWidth =
492         std::max((uint64_t)LargestVectorWidth,
493                  VT->getPrimitiveSizeInBits().getKnownMinValue());
494 
495   if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
496     LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
497 
498   // Add the min-legal-vector-width attribute. This contains the max width from:
499   // 1. min-vector-width attribute used in the source program.
500   // 2. Any builtins used that have a vector width specified.
501   // 3. Values passed in and out of inline assembly.
502   // 4. Width of vector arguments and return types for this function.
503   // 5. Width of vector arguments and return types for functions called by this
504   //    function.
505   if (getContext().getTargetInfo().getTriple().isX86())
506     CurFn->addFnAttr("min-legal-vector-width",
507                      llvm::utostr(LargestVectorWidth));
508 
509   // Add vscale_range attribute if appropriate.
510   std::optional<std::pair<unsigned, unsigned>> VScaleRange =
511       getContext().getTargetInfo().getVScaleRange(getLangOpts());
512   if (VScaleRange) {
513     CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
514         getLLVMContext(), VScaleRange->first, VScaleRange->second));
515   }
516 
517   // If we generated an unreachable return block, delete it now.
518   if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
519     Builder.ClearInsertionPoint();
520     ReturnBlock.getBlock()->eraseFromParent();
521   }
522   if (ReturnValue.isValid()) {
523     auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
524     if (RetAlloca && RetAlloca->use_empty()) {
525       RetAlloca->eraseFromParent();
526       ReturnValue = Address::invalid();
527     }
528   }
529 }
530 
531 /// ShouldInstrumentFunction - Return true if the current function should be
532 /// instrumented with __cyg_profile_func_* calls
533 bool CodeGenFunction::ShouldInstrumentFunction() {
534   if (!CGM.getCodeGenOpts().InstrumentFunctions &&
535       !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
536       !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
537     return false;
538   if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
539     return false;
540   return true;
541 }
542 
543 bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
544   if (!CurFuncDecl)
545     return false;
546   return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
547 }
548 
549 /// ShouldXRayInstrument - Return true if the current function should be
550 /// instrumented with XRay nop sleds.
551 bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
552   return CGM.getCodeGenOpts().XRayInstrumentFunctions;
553 }
554 
555 /// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
556 /// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
557 bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
558   return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
559          (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
560           CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
561               XRayInstrKind::Custom);
562 }
563 
564 bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
565   return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
566          (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
567           CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
568               XRayInstrKind::Typed);
569 }
570 
571 llvm::ConstantInt *
572 CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {
573   // Remove any (C++17) exception specifications, to allow calling e.g. a
574   // noexcept function through a non-noexcept pointer.
575   if (!Ty->isFunctionNoProtoType())
576     Ty = getContext().getFunctionTypeWithExceptionSpec(Ty, EST_None);
577   std::string Mangled;
578   llvm::raw_string_ostream Out(Mangled);
579   CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out, false);
580   return llvm::ConstantInt::get(
581       CGM.Int32Ty, static_cast<uint32_t>(llvm::xxh3_64bits(Mangled)));
582 }
583 
584 void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
585                                          llvm::Function *Fn) {
586   if (!FD->hasAttr<OpenCLKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
587     return;
588 
589   llvm::LLVMContext &Context = getLLVMContext();
590 
591   CGM.GenKernelArgMetadata(Fn, FD, this);
592 
593   if (!getLangOpts().OpenCL)
594     return;
595 
596   if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
597     QualType HintQTy = A->getTypeHint();
598     const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
599     bool IsSignedInteger =
600         HintQTy->isSignedIntegerType() ||
601         (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
602     llvm::Metadata *AttrMDArgs[] = {
603         llvm::ConstantAsMetadata::get(llvm::UndefValue::get(
604             CGM.getTypes().ConvertType(A->getTypeHint()))),
605         llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
606             llvm::IntegerType::get(Context, 32),
607             llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
608     Fn->setMetadata("vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));
609   }
610 
611   if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
612     llvm::Metadata *AttrMDArgs[] = {
613         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
614         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
615         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
616     Fn->setMetadata("work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));
617   }
618 
619   if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
620     llvm::Metadata *AttrMDArgs[] = {
621         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
622         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
623         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
624     Fn->setMetadata("reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));
625   }
626 
627   if (const OpenCLIntelReqdSubGroupSizeAttr *A =
628           FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
629     llvm::Metadata *AttrMDArgs[] = {
630         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getSubGroupSize()))};
631     Fn->setMetadata("intel_reqd_sub_group_size",
632                     llvm::MDNode::get(Context, AttrMDArgs));
633   }
634 }
635 
636 /// Determine whether the function F ends with a return stmt.
637 static bool endsWithReturn(const Decl* F) {
638   const Stmt *Body = nullptr;
639   if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))
640     Body = FD->getBody();
641   else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
642     Body = OMD->getBody();
643 
644   if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
645     auto LastStmt = CS->body_rbegin();
646     if (LastStmt != CS->body_rend())
647       return isa<ReturnStmt>(*LastStmt);
648   }
649   return false;
650 }
651 
652 void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
653   if (SanOpts.has(SanitizerKind::Thread)) {
654     Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
655     Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
656   }
657 }
658 
659 /// Check if the return value of this function requires sanitization.
660 bool CodeGenFunction::requiresReturnValueCheck() const {
661   return requiresReturnValueNullabilityCheck() ||
662          (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
663           CurCodeDecl->getAttr<ReturnsNonNullAttr>());
664 }
665 
666 static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
667   auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);
668   if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
669       !MD->getDeclName().getAsIdentifierInfo()->isStr("allocate") ||
670       (MD->getNumParams() != 1 && MD->getNumParams() != 2))
671     return false;
672 
673   if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.getSizeType())
674     return false;
675 
676   if (MD->getNumParams() == 2) {
677     auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
678     if (!PT || !PT->isVoidPointerType() ||
679         !PT->getPointeeType().isConstQualified())
680       return false;
681   }
682 
683   return true;
684 }
685 
686 bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {
687   const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
688   return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
689 }
690 
691 bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {
692   return getTarget().getTriple().getArch() == llvm::Triple::x86 &&
693          getTarget().getCXXABI().isMicrosoft() &&
694          llvm::any_of(MD->parameters(), [&](ParmVarDecl *P) {
695            return isInAllocaArgument(CGM.getCXXABI(), P->getType());
696          });
697 }
698 
699 /// Return the UBSan prologue signature for \p FD if one is available.
700 static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
701                                             const FunctionDecl *FD) {
702   if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
703     if (!MD->isStatic())
704       return nullptr;
705   return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
706 }
707 
708 void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
709                                     llvm::Function *Fn,
710                                     const CGFunctionInfo &FnInfo,
711                                     const FunctionArgList &Args,
712                                     SourceLocation Loc,
713                                     SourceLocation StartLoc) {
714   assert(!CurFn &&
715          "Do not use a CodeGenFunction object for more than one function");
716 
717   const Decl *D = GD.getDecl();
718 
719   DidCallStackSave = false;
720   CurCodeDecl = D;
721   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
722   if (FD && FD->usesSEHTry())
723     CurSEHParent = GD;
724   CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
725   FnRetTy = RetTy;
726   CurFn = Fn;
727   CurFnInfo = &FnInfo;
728   assert(CurFn->isDeclaration() && "Function already has body?");
729 
730   // If this function is ignored for any of the enabled sanitizers,
731   // disable the sanitizer for the function.
732   do {
733 #define SANITIZER(NAME, ID)                                                    \
734   if (SanOpts.empty())                                                         \
735     break;                                                                     \
736   if (SanOpts.has(SanitizerKind::ID))                                          \
737     if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc))                    \
738       SanOpts.set(SanitizerKind::ID, false);
739 
740 #include "clang/Basic/Sanitizers.def"
741 #undef SANITIZER
742   } while (false);
743 
744   if (D) {
745     const bool SanitizeBounds = SanOpts.hasOneOf(SanitizerKind::Bounds);
746     SanitizerMask no_sanitize_mask;
747     bool NoSanitizeCoverage = false;
748 
749     for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
750       no_sanitize_mask |= Attr->getMask();
751       // SanitizeCoverage is not handled by SanOpts.
752       if (Attr->hasCoverage())
753         NoSanitizeCoverage = true;
754     }
755 
756     // Apply the no_sanitize* attributes to SanOpts.
757     SanOpts.Mask &= ~no_sanitize_mask;
758     if (no_sanitize_mask & SanitizerKind::Address)
759       SanOpts.set(SanitizerKind::KernelAddress, false);
760     if (no_sanitize_mask & SanitizerKind::KernelAddress)
761       SanOpts.set(SanitizerKind::Address, false);
762     if (no_sanitize_mask & SanitizerKind::HWAddress)
763       SanOpts.set(SanitizerKind::KernelHWAddress, false);
764     if (no_sanitize_mask & SanitizerKind::KernelHWAddress)
765       SanOpts.set(SanitizerKind::HWAddress, false);
766 
767     if (SanitizeBounds && !SanOpts.hasOneOf(SanitizerKind::Bounds))
768       Fn->addFnAttr(llvm::Attribute::NoSanitizeBounds);
769 
770     if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
771       Fn->addFnAttr(llvm::Attribute::NoSanitizeCoverage);
772 
773     // Some passes need the non-negated no_sanitize attribute. Pass them on.
774     if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {
775       if (no_sanitize_mask & SanitizerKind::Thread)
776         Fn->addFnAttr("no_sanitize_thread");
777     }
778   }
779 
780   if (ShouldSkipSanitizerInstrumentation()) {
781     CurFn->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);
782   } else {
783     // Apply sanitizer attributes to the function.
784     if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
785       Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
786     if (SanOpts.hasOneOf(SanitizerKind::HWAddress |
787                          SanitizerKind::KernelHWAddress))
788       Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
789     if (SanOpts.has(SanitizerKind::MemtagStack))
790       Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
791     if (SanOpts.has(SanitizerKind::Thread))
792       Fn->addFnAttr(llvm::Attribute::SanitizeThread);
793     if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
794       Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
795   }
796   if (SanOpts.has(SanitizerKind::SafeStack))
797     Fn->addFnAttr(llvm::Attribute::SafeStack);
798   if (SanOpts.has(SanitizerKind::ShadowCallStack))
799     Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
800 
801   // Apply fuzzing attribute to the function.
802   if (SanOpts.hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
803     Fn->addFnAttr(llvm::Attribute::OptForFuzzing);
804 
805   // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
806   // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
807   if (SanOpts.has(SanitizerKind::Thread)) {
808     if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {
809       IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);
810       if (OMD->getMethodFamily() == OMF_dealloc ||
811           OMD->getMethodFamily() == OMF_initialize ||
812           (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) {
813         markAsIgnoreThreadCheckingAtRuntime(Fn);
814       }
815     }
816   }
817 
818   // Ignore unrelated casts in STL allocate() since the allocator must cast
819   // from void* to T* before object initialization completes. Don't match on the
820   // namespace because not all allocators are in std::
821   if (D && SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
822     if (matchesStlAllocatorFn(D, getContext()))
823       SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
824   }
825 
826   // Ignore null checks in coroutine functions since the coroutines passes
827   // are not aware of how to move the extra UBSan instructions across the split
828   // coroutine boundaries.
829   if (D && SanOpts.has(SanitizerKind::Null))
830     if (FD && FD->getBody() &&
831         FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
832       SanOpts.Mask &= ~SanitizerKind::Null;
833 
834   // Apply xray attributes to the function (as a string, for now)
835   bool AlwaysXRayAttr = false;
836   if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
837     if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
838             XRayInstrKind::FunctionEntry) ||
839         CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
840             XRayInstrKind::FunctionExit)) {
841       if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
842         Fn->addFnAttr("function-instrument", "xray-always");
843         AlwaysXRayAttr = true;
844       }
845       if (XRayAttr->neverXRayInstrument())
846         Fn->addFnAttr("function-instrument", "xray-never");
847       if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
848         if (ShouldXRayInstrumentFunction())
849           Fn->addFnAttr("xray-log-args",
850                         llvm::utostr(LogArgs->getArgumentCount()));
851     }
852   } else {
853     if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
854       Fn->addFnAttr(
855           "xray-instruction-threshold",
856           llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
857   }
858 
859   if (ShouldXRayInstrumentFunction()) {
860     if (CGM.getCodeGenOpts().XRayIgnoreLoops)
861       Fn->addFnAttr("xray-ignore-loops");
862 
863     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
864             XRayInstrKind::FunctionExit))
865       Fn->addFnAttr("xray-skip-exit");
866 
867     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
868             XRayInstrKind::FunctionEntry))
869       Fn->addFnAttr("xray-skip-entry");
870 
871     auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
872     if (FuncGroups > 1) {
873       auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
874                                               CurFn->getName().bytes_end());
875       auto Group = crc32(FuncName) % FuncGroups;
876       if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
877           !AlwaysXRayAttr)
878         Fn->addFnAttr("function-instrument", "xray-never");
879     }
880   }
881 
882   if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone) {
883     switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
884     case ProfileList::Skip:
885       Fn->addFnAttr(llvm::Attribute::SkipProfile);
886       break;
887     case ProfileList::Forbid:
888       Fn->addFnAttr(llvm::Attribute::NoProfile);
889       break;
890     case ProfileList::Allow:
891       break;
892     }
893   }
894 
895   unsigned Count, Offset;
896   if (const auto *Attr =
897           D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
898     Count = Attr->getCount();
899     Offset = Attr->getOffset();
900   } else {
901     Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
902     Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
903   }
904   if (Count && Offset <= Count) {
905     Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
906     if (Offset)
907       Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
908   }
909   // Instruct that functions for COFF/CodeView targets should start with a
910   // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
911   // backends as they don't need it -- instructions on these architectures are
912   // always atomically patchable at runtime.
913   if (CGM.getCodeGenOpts().HotPatch &&
914       getContext().getTargetInfo().getTriple().isX86() &&
915       getContext().getTargetInfo().getTriple().getEnvironment() !=
916           llvm::Triple::CODE16)
917     Fn->addFnAttr("patchable-function", "prologue-short-redirect");
918 
919   // Add no-jump-tables value.
920   if (CGM.getCodeGenOpts().NoUseJumpTables)
921     Fn->addFnAttr("no-jump-tables", "true");
922 
923   // Add no-inline-line-tables value.
924   if (CGM.getCodeGenOpts().NoInlineLineTables)
925     Fn->addFnAttr("no-inline-line-tables");
926 
927   // Add profile-sample-accurate value.
928   if (CGM.getCodeGenOpts().ProfileSampleAccurate)
929     Fn->addFnAttr("profile-sample-accurate");
930 
931   if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
932     Fn->addFnAttr("use-sample-profile");
933 
934   if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
935     Fn->addFnAttr("cfi-canonical-jump-table");
936 
937   if (D && D->hasAttr<NoProfileFunctionAttr>())
938     Fn->addFnAttr(llvm::Attribute::NoProfile);
939 
940   if (D) {
941     // Function attributes take precedence over command line flags.
942     if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
943       switch (A->getThunkType()) {
944       case FunctionReturnThunksAttr::Kind::Keep:
945         break;
946       case FunctionReturnThunksAttr::Kind::Extern:
947         Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
948         break;
949       }
950     } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
951       Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
952   }
953 
954   if (FD && (getLangOpts().OpenCL ||
955              (getLangOpts().HIP && getLangOpts().CUDAIsDevice))) {
956     // Add metadata for a kernel function.
957     EmitKernelMetadata(FD, Fn);
958   }
959 
960   // If we are checking function types, emit a function type signature as
961   // prologue data.
962   if (FD && SanOpts.has(SanitizerKind::Function)) {
963     if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
964       llvm::LLVMContext &Ctx = Fn->getContext();
965       llvm::MDBuilder MDB(Ctx);
966       Fn->setMetadata(
967           llvm::LLVMContext::MD_func_sanitize,
968           MDB.createRTTIPointerPrologue(
969               PrologueSig, getUBSanFunctionTypeHash(FD->getType())));
970     }
971   }
972 
973   // If we're checking nullability, we need to know whether we can check the
974   // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
975   if (SanOpts.has(SanitizerKind::NullabilityReturn)) {
976     auto Nullability = FnRetTy->getNullability();
977     if (Nullability && *Nullability == NullabilityKind::NonNull) {
978       if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) &&
979             CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
980         RetValNullabilityPrecondition =
981             llvm::ConstantInt::getTrue(getLLVMContext());
982     }
983   }
984 
985   // If we're in C++ mode and the function name is "main", it is guaranteed
986   // to be norecurse by the standard (3.6.1.3 "The function main shall not be
987   // used within a program").
988   //
989   // OpenCL C 2.0 v2.2-11 s6.9.i:
990   //     Recursion is not supported.
991   //
992   // SYCL v1.2.1 s3.10:
993   //     kernels cannot include RTTI information, exception classes,
994   //     recursive code, virtual functions or make use of C++ libraries that
995   //     are not compiled for the device.
996   if (FD && ((getLangOpts().CPlusPlus && FD->isMain()) ||
997              getLangOpts().OpenCL || getLangOpts().SYCLIsDevice ||
998              (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
999     Fn->addFnAttr(llvm::Attribute::NoRecurse);
1000 
1001   llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
1002   llvm::fp::ExceptionBehavior FPExceptionBehavior =
1003       ToConstrainedExceptMD(getLangOpts().getDefaultExceptionMode());
1004   Builder.setDefaultConstrainedRounding(RM);
1005   Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
1006   if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
1007       (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
1008                RM != llvm::RoundingMode::NearestTiesToEven))) {
1009     Builder.setIsFPConstrained(true);
1010     Fn->addFnAttr(llvm::Attribute::StrictFP);
1011   }
1012 
1013   // If a custom alignment is used, force realigning to this alignment on
1014   // any main function which certainly will need it.
1015   if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1016              CGM.getCodeGenOpts().StackAlignment))
1017     Fn->addFnAttr("stackrealign");
1018 
1019   // "main" doesn't need to zero out call-used registers.
1020   if (FD && FD->isMain())
1021     Fn->removeFnAttr("zero-call-used-regs");
1022 
1023   llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
1024 
1025   // Create a marker to make it easy to insert allocas into the entryblock
1026   // later.  Don't create this with the builder, because we don't want it
1027   // folded.
1028   llvm::Value *Undef = llvm::UndefValue::get(Int32Ty);
1029   AllocaInsertPt = new llvm::BitCastInst(Undef, Int32Ty, "allocapt", EntryBB);
1030 
1031   ReturnBlock = getJumpDestInCurrentScope("return");
1032 
1033   Builder.SetInsertPoint(EntryBB);
1034 
1035   // If we're checking the return value, allocate space for a pointer to a
1036   // precise source location of the checked return statement.
1037   if (requiresReturnValueCheck()) {
1038     ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr");
1039     Builder.CreateStore(llvm::ConstantPointerNull::get(Int8PtrTy),
1040                         ReturnLocation);
1041   }
1042 
1043   // Emit subprogram debug descriptor.
1044   if (CGDebugInfo *DI = getDebugInfo()) {
1045     // Reconstruct the type from the argument list so that implicit parameters,
1046     // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1047     // convention.
1048     DI->emitFunctionStart(GD, Loc, StartLoc,
1049                           DI->getFunctionType(FD, RetTy, Args), CurFn,
1050                           CurFuncIsThunk);
1051   }
1052 
1053   if (ShouldInstrumentFunction()) {
1054     if (CGM.getCodeGenOpts().InstrumentFunctions)
1055       CurFn->addFnAttr("instrument-function-entry", "__cyg_profile_func_enter");
1056     if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1057       CurFn->addFnAttr("instrument-function-entry-inlined",
1058                        "__cyg_profile_func_enter");
1059     if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1060       CurFn->addFnAttr("instrument-function-entry-inlined",
1061                        "__cyg_profile_func_enter_bare");
1062   }
1063 
1064   // Since emitting the mcount call here impacts optimizations such as function
1065   // inlining, we just add an attribute to insert a mcount call in backend.
1066   // The attribute "counting-function" is set to mcount function name which is
1067   // architecture dependent.
1068   if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1069     // Calls to fentry/mcount should not be generated if function has
1070     // the no_instrument_function attribute.
1071     if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1072       if (CGM.getCodeGenOpts().CallFEntry)
1073         Fn->addFnAttr("fentry-call", "true");
1074       else {
1075         Fn->addFnAttr("instrument-function-entry-inlined",
1076                       getTarget().getMCountName());
1077       }
1078       if (CGM.getCodeGenOpts().MNopMCount) {
1079         if (!CGM.getCodeGenOpts().CallFEntry)
1080           CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1081             << "-mnop-mcount" << "-mfentry";
1082         Fn->addFnAttr("mnop-mcount");
1083       }
1084 
1085       if (CGM.getCodeGenOpts().RecordMCount) {
1086         if (!CGM.getCodeGenOpts().CallFEntry)
1087           CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1088             << "-mrecord-mcount" << "-mfentry";
1089         Fn->addFnAttr("mrecord-mcount");
1090       }
1091     }
1092   }
1093 
1094   if (CGM.getCodeGenOpts().PackedStack) {
1095     if (getContext().getTargetInfo().getTriple().getArch() !=
1096         llvm::Triple::systemz)
1097       CGM.getDiags().Report(diag::err_opt_not_valid_on_target)
1098         << "-mpacked-stack";
1099     Fn->addFnAttr("packed-stack");
1100   }
1101 
1102   if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1103       !CGM.getDiags().isIgnored(diag::warn_fe_backend_frame_larger_than, Loc))
1104     Fn->addFnAttr("warn-stack-size",
1105                   std::to_string(CGM.getCodeGenOpts().WarnStackSize));
1106 
1107   if (RetTy->isVoidType()) {
1108     // Void type; nothing to return.
1109     ReturnValue = Address::invalid();
1110 
1111     // Count the implicit return.
1112     if (!endsWithReturn(D))
1113       ++NumReturnExprs;
1114   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1115     // Indirect return; emit returned value directly into sret slot.
1116     // This reduces code size, and affects correctness in C++.
1117     auto AI = CurFn->arg_begin();
1118     if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1119       ++AI;
1120     ReturnValue =
1121         Address(&*AI, ConvertType(RetTy),
1122                 CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull);
1123     if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1124       ReturnValuePointer = CreateDefaultAlignTempAlloca(
1125           ReturnValue.getPointer()->getType(), "result.ptr");
1126       Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer);
1127     }
1128   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1129              !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
1130     // Load the sret pointer from the argument struct and return into that.
1131     unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1132     llvm::Function::arg_iterator EI = CurFn->arg_end();
1133     --EI;
1134     llvm::Value *Addr = Builder.CreateStructGEP(
1135         CurFnInfo->getArgStruct(), &*EI, Idx);
1136     llvm::Type *Ty =
1137         cast<llvm::GetElementPtrInst>(Addr)->getResultElementType();
1138     ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1139     Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result");
1140     ReturnValue = Address(Addr, ConvertType(RetTy),
1141                           CGM.getNaturalTypeAlignment(RetTy), KnownNonNull);
1142   } else {
1143     ReturnValue = CreateIRTemp(RetTy, "retval");
1144 
1145     // Tell the epilog emitter to autorelease the result.  We do this
1146     // now so that various specialized functions can suppress it
1147     // during their IR-generation.
1148     if (getLangOpts().ObjCAutoRefCount &&
1149         !CurFnInfo->isReturnsRetained() &&
1150         RetTy->isObjCRetainableType())
1151       AutoreleaseResult = true;
1152   }
1153 
1154   EmitStartEHSpec(CurCodeDecl);
1155 
1156   PrologueCleanupDepth = EHStack.stable_begin();
1157 
1158   // Emit OpenMP specific initialization of the device functions.
1159   if (getLangOpts().OpenMP && CurCodeDecl)
1160     CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl);
1161 
1162   // Handle emitting HLSL entry functions.
1163   if (D && D->hasAttr<HLSLShaderAttr>())
1164     CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1165 
1166   EmitFunctionProlog(*CurFnInfo, CurFn, Args);
1167 
1168   if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(D);
1169       MD && !MD->isStatic()) {
1170     bool IsInLambda =
1171         MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;
1172     if (MD->isImplicitObjectMemberFunction())
1173       CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
1174     if (IsInLambda) {
1175       // We're in a lambda; figure out the captures.
1176       MD->getParent()->getCaptureFields(LambdaCaptureFields,
1177                                         LambdaThisCaptureField);
1178       if (LambdaThisCaptureField) {
1179         // If the lambda captures the object referred to by '*this' - either by
1180         // value or by reference, make sure CXXThisValue points to the correct
1181         // object.
1182 
1183         // Get the lvalue for the field (which is a copy of the enclosing object
1184         // or contains the address of the enclosing object).
1185         LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
1186         if (!LambdaThisCaptureField->getType()->isPointerType()) {
1187           // If the enclosing object was captured by value, just use its address.
1188           CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
1189         } else {
1190           // Load the lvalue pointed to by the field, since '*this' was captured
1191           // by reference.
1192           CXXThisValue =
1193               EmitLoadOfLValue(ThisFieldLValue, SourceLocation()).getScalarVal();
1194         }
1195       }
1196       for (auto *FD : MD->getParent()->fields()) {
1197         if (FD->hasCapturedVLAType()) {
1198           auto *ExprArg = EmitLoadOfLValue(EmitLValueForLambdaField(FD),
1199                                            SourceLocation()).getScalarVal();
1200           auto VAT = FD->getCapturedVLAType();
1201           VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1202         }
1203       }
1204     } else if (MD->isImplicitObjectMemberFunction()) {
1205       // Not in a lambda; just use 'this' from the method.
1206       // FIXME: Should we generate a new load for each use of 'this'?  The
1207       // fast register allocator would be happier...
1208       CXXThisValue = CXXABIThisValue;
1209     }
1210 
1211     // Check the 'this' pointer once per function, if it's available.
1212     if (CXXABIThisValue) {
1213       SanitizerSet SkippedChecks;
1214       SkippedChecks.set(SanitizerKind::ObjectSize, true);
1215       QualType ThisTy = MD->getThisType();
1216 
1217       // If this is the call operator of a lambda with no captures, it
1218       // may have a static invoker function, which may call this operator with
1219       // a null 'this' pointer.
1220       if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
1221         SkippedChecks.set(SanitizerKind::Null, true);
1222 
1223       EmitTypeCheck(
1224           isa<CXXConstructorDecl>(MD) ? TCK_ConstructorCall : TCK_MemberCall,
1225           Loc, CXXABIThisValue, ThisTy, CXXABIThisAlignment, SkippedChecks);
1226     }
1227   }
1228 
1229   // If any of the arguments have a variably modified type, make sure to
1230   // emit the type size, but only if the function is not naked. Naked functions
1231   // have no prolog to run this evaluation.
1232   if (!FD || !FD->hasAttr<NakedAttr>()) {
1233     for (const VarDecl *VD : Args) {
1234       // Dig out the type as written from ParmVarDecls; it's unclear whether
1235       // the standard (C99 6.9.1p10) requires this, but we're following the
1236       // precedent set by gcc.
1237       QualType Ty;
1238       if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
1239         Ty = PVD->getOriginalType();
1240       else
1241         Ty = VD->getType();
1242 
1243       if (Ty->isVariablyModifiedType())
1244         EmitVariablyModifiedType(Ty);
1245     }
1246   }
1247   // Emit a location at the end of the prologue.
1248   if (CGDebugInfo *DI = getDebugInfo())
1249     DI->EmitLocation(Builder, StartLoc);
1250   // TODO: Do we need to handle this in two places like we do with
1251   // target-features/target-cpu?
1252   if (CurFuncDecl)
1253     if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1254       LargestVectorWidth = VecWidth->getVectorWidth();
1255 }
1256 
1257 void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1258   incrementProfileCounter(Body);
1259   maybeCreateMCDCCondBitmap();
1260   if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
1261     EmitCompoundStmtWithoutScope(*S);
1262   else
1263     EmitStmt(Body);
1264 }
1265 
1266 /// When instrumenting to collect profile data, the counts for some blocks
1267 /// such as switch cases need to not include the fall-through counts, so
1268 /// emit a branch around the instrumentation code. When not instrumenting,
1269 /// this just calls EmitBlock().
1270 void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1271                                                const Stmt *S) {
1272   llvm::BasicBlock *SkipCountBB = nullptr;
1273   if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1274     // When instrumenting for profiling, the fallthrough to certain
1275     // statements needs to skip over the instrumentation code so that we
1276     // get an accurate count.
1277     SkipCountBB = createBasicBlock("skipcount");
1278     EmitBranch(SkipCountBB);
1279   }
1280   EmitBlock(BB);
1281   uint64_t CurrentCount = getCurrentProfileCount();
1282   incrementProfileCounter(S);
1283   setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1284   if (SkipCountBB)
1285     EmitBlock(SkipCountBB);
1286 }
1287 
1288 /// Tries to mark the given function nounwind based on the
1289 /// non-existence of any throwing calls within it.  We believe this is
1290 /// lightweight enough to do at -O0.
1291 static void TryMarkNoThrow(llvm::Function *F) {
1292   // LLVM treats 'nounwind' on a function as part of the type, so we
1293   // can't do this on functions that can be overwritten.
1294   if (F->isInterposable()) return;
1295 
1296   for (llvm::BasicBlock &BB : *F)
1297     for (llvm::Instruction &I : BB)
1298       if (I.mayThrow())
1299         return;
1300 
1301   F->setDoesNotThrow();
1302 }
1303 
1304 QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1305                                                FunctionArgList &Args) {
1306   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1307   QualType ResTy = FD->getReturnType();
1308 
1309   const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
1310   if (MD && MD->isImplicitObjectMemberFunction()) {
1311     if (CGM.getCXXABI().HasThisReturn(GD))
1312       ResTy = MD->getThisType();
1313     else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1314       ResTy = CGM.getContext().VoidPtrTy;
1315     CGM.getCXXABI().buildThisParam(*this, Args);
1316   }
1317 
1318   // The base version of an inheriting constructor whose constructed base is a
1319   // virtual base is not passed any arguments (because it doesn't actually call
1320   // the inherited constructor).
1321   bool PassedParams = true;
1322   if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
1323     if (auto Inherited = CD->getInheritedConstructor())
1324       PassedParams =
1325           getTypes().inheritingCtorHasParams(Inherited, GD.getCtorType());
1326 
1327   if (PassedParams) {
1328     for (auto *Param : FD->parameters()) {
1329       Args.push_back(Param);
1330       if (!Param->hasAttr<PassObjectSizeAttr>())
1331         continue;
1332 
1333       auto *Implicit = ImplicitParamDecl::Create(
1334           getContext(), Param->getDeclContext(), Param->getLocation(),
1335           /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamKind::Other);
1336       SizeArguments[Param] = Implicit;
1337       Args.push_back(Implicit);
1338     }
1339   }
1340 
1341   if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
1342     CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
1343 
1344   return ResTy;
1345 }
1346 
1347 void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1348                                    const CGFunctionInfo &FnInfo) {
1349   assert(Fn && "generating code for null Function");
1350   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1351   CurGD = GD;
1352 
1353   FunctionArgList Args;
1354   QualType ResTy = BuildFunctionArgList(GD, Args);
1355 
1356   if (FD->isInlineBuiltinDeclaration()) {
1357     // When generating code for a builtin with an inline declaration, use a
1358     // mangled name to hold the actual body, while keeping an external
1359     // definition in case the function pointer is referenced somewhere.
1360     std::string FDInlineName = (Fn->getName() + ".inline").str();
1361     llvm::Module *M = Fn->getParent();
1362     llvm::Function *Clone = M->getFunction(FDInlineName);
1363     if (!Clone) {
1364       Clone = llvm::Function::Create(Fn->getFunctionType(),
1365                                      llvm::GlobalValue::InternalLinkage,
1366                                      Fn->getAddressSpace(), FDInlineName, M);
1367       Clone->addFnAttr(llvm::Attribute::AlwaysInline);
1368     }
1369     Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1370     Fn = Clone;
1371   } else {
1372     // Detect the unusual situation where an inline version is shadowed by a
1373     // non-inline version. In that case we should pick the external one
1374     // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1375     // to detect that situation before we reach codegen, so do some late
1376     // replacement.
1377     for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1378          PD = PD->getPreviousDecl()) {
1379       if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1380         std::string FDInlineName = (Fn->getName() + ".inline").str();
1381         llvm::Module *M = Fn->getParent();
1382         if (llvm::Function *Clone = M->getFunction(FDInlineName)) {
1383           Clone->replaceAllUsesWith(Fn);
1384           Clone->eraseFromParent();
1385         }
1386         break;
1387       }
1388     }
1389   }
1390 
1391   // Check if we should generate debug info for this function.
1392   if (FD->hasAttr<NoDebugAttr>()) {
1393     // Clear non-distinct debug info that was possibly attached to the function
1394     // due to an earlier declaration without the nodebug attribute
1395     Fn->setSubprogram(nullptr);
1396     // Disable debug info indefinitely for this function
1397     DebugInfo = nullptr;
1398   }
1399 
1400   // The function might not have a body if we're generating thunks for a
1401   // function declaration.
1402   SourceRange BodyRange;
1403   if (Stmt *Body = FD->getBody())
1404     BodyRange = Body->getSourceRange();
1405   else
1406     BodyRange = FD->getLocation();
1407   CurEHLocation = BodyRange.getEnd();
1408 
1409   // Use the location of the start of the function to determine where
1410   // the function definition is located. By default use the location
1411   // of the declaration as the location for the subprogram. A function
1412   // may lack a declaration in the source code if it is created by code
1413   // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1414   SourceLocation Loc = FD->getLocation();
1415 
1416   // If this is a function specialization then use the pattern body
1417   // as the location for the function.
1418   if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1419     if (SpecDecl->hasBody(SpecDecl))
1420       Loc = SpecDecl->getLocation();
1421 
1422   Stmt *Body = FD->getBody();
1423 
1424   if (Body) {
1425     // Coroutines always emit lifetime markers.
1426     if (isa<CoroutineBodyStmt>(Body))
1427       ShouldEmitLifetimeMarkers = true;
1428 
1429     // Initialize helper which will detect jumps which can cause invalid
1430     // lifetime markers.
1431     if (ShouldEmitLifetimeMarkers)
1432       Bypasses.Init(Body);
1433   }
1434 
1435   // Emit the standard function prologue.
1436   StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
1437 
1438   // Save parameters for coroutine function.
1439   if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body))
1440     llvm::append_range(FnArgs, FD->parameters());
1441 
1442   // Ensure that the function adheres to the forward progress guarantee, which
1443   // is required by certain optimizations.
1444   if (checkIfFunctionMustProgress())
1445     CurFn->addFnAttr(llvm::Attribute::MustProgress);
1446 
1447   // Generate the body of the function.
1448   PGO.assignRegionCounters(GD, CurFn);
1449   if (isa<CXXDestructorDecl>(FD))
1450     EmitDestructorBody(Args);
1451   else if (isa<CXXConstructorDecl>(FD))
1452     EmitConstructorBody(Args);
1453   else if (getLangOpts().CUDA &&
1454            !getLangOpts().CUDAIsDevice &&
1455            FD->hasAttr<CUDAGlobalAttr>())
1456     CGM.getCUDARuntime().emitDeviceStub(*this, Args);
1457   else if (isa<CXXMethodDecl>(FD) &&
1458            cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()) {
1459     // The lambda static invoker function is special, because it forwards or
1460     // clones the body of the function call operator (but is actually static).
1461     EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD));
1462   } else if (isa<CXXMethodDecl>(FD) &&
1463              isLambdaCallOperator(cast<CXXMethodDecl>(FD)) &&
1464              !FnInfo.isDelegateCall() &&
1465              cast<CXXMethodDecl>(FD)->getParent()->getLambdaStaticInvoker() &&
1466              hasInAllocaArg(cast<CXXMethodDecl>(FD))) {
1467     // If emitting a lambda with static invoker on X86 Windows, change
1468     // the call operator body.
1469     // Make sure that this is a call operator with an inalloca arg and check
1470     // for delegate call to make sure this is the original call op and not the
1471     // new forwarding function for the static invoker.
1472     EmitLambdaInAllocaCallOpBody(cast<CXXMethodDecl>(FD));
1473   } else if (FD->isDefaulted() && isa<CXXMethodDecl>(FD) &&
1474              (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() ||
1475               cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator())) {
1476     // Implicit copy-assignment gets the same special treatment as implicit
1477     // copy-constructors.
1478     emitImplicitAssignmentOperatorBody(Args);
1479   } else if (Body) {
1480     EmitFunctionBody(Body);
1481   } else
1482     llvm_unreachable("no definition for emitted function");
1483 
1484   // C++11 [stmt.return]p2:
1485   //   Flowing off the end of a function [...] results in undefined behavior in
1486   //   a value-returning function.
1487   // C11 6.9.1p12:
1488   //   If the '}' that terminates a function is reached, and the value of the
1489   //   function call is used by the caller, the behavior is undefined.
1490   if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1491       !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1492     bool ShouldEmitUnreachable =
1493         CGM.getCodeGenOpts().StrictReturn ||
1494         !CGM.MayDropFunctionReturn(FD->getASTContext(), FD->getReturnType());
1495     if (SanOpts.has(SanitizerKind::Return)) {
1496       SanitizerScope SanScope(this);
1497       llvm::Value *IsFalse = Builder.getFalse();
1498       EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
1499                 SanitizerHandler::MissingReturn,
1500                 EmitCheckSourceLocation(FD->getLocation()), std::nullopt);
1501     } else if (ShouldEmitUnreachable) {
1502       if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1503         EmitTrapCall(llvm::Intrinsic::trap);
1504     }
1505     if (SanOpts.has(SanitizerKind::Return) || ShouldEmitUnreachable) {
1506       Builder.CreateUnreachable();
1507       Builder.ClearInsertionPoint();
1508     }
1509   }
1510 
1511   // Emit the standard function epilogue.
1512   FinishFunction(BodyRange.getEnd());
1513 
1514   // If we haven't marked the function nothrow through other means, do
1515   // a quick pass now to see if we can.
1516   if (!CurFn->doesNotThrow())
1517     TryMarkNoThrow(CurFn);
1518 }
1519 
1520 /// ContainsLabel - Return true if the statement contains a label in it.  If
1521 /// this statement is not executed normally, it not containing a label means
1522 /// that we can just remove the code.
1523 bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1524   // Null statement, not a label!
1525   if (!S) return false;
1526 
1527   // If this is a label, we have to emit the code, consider something like:
1528   // if (0) {  ...  foo:  bar(); }  goto foo;
1529   //
1530   // TODO: If anyone cared, we could track __label__'s, since we know that you
1531   // can't jump to one from outside their declared region.
1532   if (isa<LabelStmt>(S))
1533     return true;
1534 
1535   // If this is a case/default statement, and we haven't seen a switch, we have
1536   // to emit the code.
1537   if (isa<SwitchCase>(S) && !IgnoreCaseStmts)
1538     return true;
1539 
1540   // If this is a switch statement, we want to ignore cases below it.
1541   if (isa<SwitchStmt>(S))
1542     IgnoreCaseStmts = true;
1543 
1544   // Scan subexpressions for verboten labels.
1545   for (const Stmt *SubStmt : S->children())
1546     if (ContainsLabel(SubStmt, IgnoreCaseStmts))
1547       return true;
1548 
1549   return false;
1550 }
1551 
1552 /// containsBreak - Return true if the statement contains a break out of it.
1553 /// If the statement (recursively) contains a switch or loop with a break
1554 /// inside of it, this is fine.
1555 bool CodeGenFunction::containsBreak(const Stmt *S) {
1556   // Null statement, not a label!
1557   if (!S) return false;
1558 
1559   // If this is a switch or loop that defines its own break scope, then we can
1560   // include it and anything inside of it.
1561   if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||
1562       isa<ForStmt>(S))
1563     return false;
1564 
1565   if (isa<BreakStmt>(S))
1566     return true;
1567 
1568   // Scan subexpressions for verboten breaks.
1569   for (const Stmt *SubStmt : S->children())
1570     if (containsBreak(SubStmt))
1571       return true;
1572 
1573   return false;
1574 }
1575 
1576 bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1577   if (!S) return false;
1578 
1579   // Some statement kinds add a scope and thus never add a decl to the current
1580   // scope. Note, this list is longer than the list of statements that might
1581   // have an unscoped decl nested within them, but this way is conservatively
1582   // correct even if more statement kinds are added.
1583   if (isa<IfStmt>(S) || isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
1584       isa<DoStmt>(S) || isa<ForStmt>(S) || isa<CompoundStmt>(S) ||
1585       isa<CXXForRangeStmt>(S) || isa<CXXTryStmt>(S) ||
1586       isa<ObjCForCollectionStmt>(S) || isa<ObjCAtTryStmt>(S))
1587     return false;
1588 
1589   if (isa<DeclStmt>(S))
1590     return true;
1591 
1592   for (const Stmt *SubStmt : S->children())
1593     if (mightAddDeclToScope(SubStmt))
1594       return true;
1595 
1596   return false;
1597 }
1598 
1599 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1600 /// to a constant, or if it does but contains a label, return false.  If it
1601 /// constant folds return true and set the boolean result in Result.
1602 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1603                                                    bool &ResultBool,
1604                                                    bool AllowLabels) {
1605   // If MC/DC is enabled, disable folding so that we can instrument all
1606   // conditions to yield complete test vectors. We still keep track of
1607   // folded conditions during region mapping and visualization.
1608   if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
1609       CGM.getCodeGenOpts().MCDCCoverage)
1610     return false;
1611 
1612   llvm::APSInt ResultInt;
1613   if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))
1614     return false;
1615 
1616   ResultBool = ResultInt.getBoolValue();
1617   return true;
1618 }
1619 
1620 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1621 /// to a constant, or if it does but contains a label, return false.  If it
1622 /// constant folds return true and set the folded value.
1623 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1624                                                    llvm::APSInt &ResultInt,
1625                                                    bool AllowLabels) {
1626   // FIXME: Rename and handle conversion of other evaluatable things
1627   // to bool.
1628   Expr::EvalResult Result;
1629   if (!Cond->EvaluateAsInt(Result, getContext()))
1630     return false;  // Not foldable, not integer or not fully evaluatable.
1631 
1632   llvm::APSInt Int = Result.Val.getInt();
1633   if (!AllowLabels && CodeGenFunction::ContainsLabel(Cond))
1634     return false;  // Contains a label.
1635 
1636   ResultInt = Int;
1637   return true;
1638 }
1639 
1640 /// Strip parentheses and simplistic logical-NOT operators.
1641 const Expr *CodeGenFunction::stripCond(const Expr *C) {
1642   while (const UnaryOperator *Op = dyn_cast<UnaryOperator>(C->IgnoreParens())) {
1643     if (Op->getOpcode() != UO_LNot)
1644       break;
1645     C = Op->getSubExpr();
1646   }
1647   return C->IgnoreParens();
1648 }
1649 
1650 /// Determine whether the given condition is an instrumentable condition
1651 /// (i.e. no "&&" or "||").
1652 bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1653   const BinaryOperator *BOp = dyn_cast<BinaryOperator>(stripCond(C));
1654   return (!BOp || !BOp->isLogicalOp());
1655 }
1656 
1657 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1658 /// increments a profile counter based on the semantics of the given logical
1659 /// operator opcode.  This is used to instrument branch condition coverage for
1660 /// logical operators.
1661 void CodeGenFunction::EmitBranchToCounterBlock(
1662     const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1663     llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1664     Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1665   // If not instrumenting, just emit a branch.
1666   bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1667   if (!InstrumentRegions || !isInstrumentedCondition(Cond))
1668     return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1669 
1670   llvm::BasicBlock *ThenBlock = nullptr;
1671   llvm::BasicBlock *ElseBlock = nullptr;
1672   llvm::BasicBlock *NextBlock = nullptr;
1673 
1674   // Create the block we'll use to increment the appropriate counter.
1675   llvm::BasicBlock *CounterIncrBlock = createBasicBlock("lop.rhscnt");
1676 
1677   // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1678   // means we need to evaluate the condition and increment the counter on TRUE:
1679   //
1680   // if (Cond)
1681   //   goto CounterIncrBlock;
1682   // else
1683   //   goto FalseBlock;
1684   //
1685   // CounterIncrBlock:
1686   //   Counter++;
1687   //   goto TrueBlock;
1688 
1689   if (LOp == BO_LAnd) {
1690     ThenBlock = CounterIncrBlock;
1691     ElseBlock = FalseBlock;
1692     NextBlock = TrueBlock;
1693   }
1694 
1695   // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1696   // we need to evaluate the condition and increment the counter on FALSE:
1697   //
1698   // if (Cond)
1699   //   goto TrueBlock;
1700   // else
1701   //   goto CounterIncrBlock;
1702   //
1703   // CounterIncrBlock:
1704   //   Counter++;
1705   //   goto FalseBlock;
1706 
1707   else if (LOp == BO_LOr) {
1708     ThenBlock = TrueBlock;
1709     ElseBlock = CounterIncrBlock;
1710     NextBlock = FalseBlock;
1711   } else {
1712     llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1713   }
1714 
1715   // Emit Branch based on condition.
1716   EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, TrueCount, LH);
1717 
1718   // Emit the block containing the counter increment(s).
1719   EmitBlock(CounterIncrBlock);
1720 
1721   // Increment corresponding counter; if index not provided, use Cond as index.
1722   incrementProfileCounter(CntrIdx ? CntrIdx : Cond);
1723 
1724   // Go to the next block.
1725   EmitBranch(NextBlock);
1726 }
1727 
1728 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1729 /// statement) to the specified blocks.  Based on the condition, this might try
1730 /// to simplify the codegen of the conditional based on the branch.
1731 /// \param LH The value of the likelihood attribute on the True branch.
1732 /// \param ConditionalOp Used by MC/DC code coverage to track the result of the
1733 /// ConditionalOperator (ternary) through a recursive call for the operator's
1734 /// LHS and RHS nodes.
1735 void CodeGenFunction::EmitBranchOnBoolExpr(
1736     const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
1737     uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp) {
1738   Cond = Cond->IgnoreParens();
1739 
1740   if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
1741     // Handle X && Y in a condition.
1742     if (CondBOp->getOpcode() == BO_LAnd) {
1743       MCDCLogOpStack.push_back(CondBOp);
1744 
1745       // If we have "1 && X", simplify the code.  "0 && X" would have constant
1746       // folded if the case was simple enough.
1747       bool ConstantBool = false;
1748       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1749           ConstantBool) {
1750         // br(1 && X) -> br(X).
1751         incrementProfileCounter(CondBOp);
1752         EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1753                                  FalseBlock, TrueCount, LH);
1754         MCDCLogOpStack.pop_back();
1755         return;
1756       }
1757 
1758       // If we have "X && 1", simplify the code to use an uncond branch.
1759       // "X && 0" would have been constant folded to 0.
1760       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1761           ConstantBool) {
1762         // br(X && 1) -> br(X).
1763         EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock,
1764                                  FalseBlock, TrueCount, LH, CondBOp);
1765         MCDCLogOpStack.pop_back();
1766         return;
1767       }
1768 
1769       // Emit the LHS as a conditional.  If the LHS conditional is false, we
1770       // want to jump to the FalseBlock.
1771       llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
1772       // The counter tells us how often we evaluate RHS, and all of TrueCount
1773       // can be propagated to that branch.
1774       uint64_t RHSCount = getProfileCount(CondBOp->getRHS());
1775 
1776       ConditionalEvaluation eval(*this);
1777       {
1778         ApplyDebugLocation DL(*this, Cond);
1779         // Propagate the likelihood attribute like __builtin_expect
1780         // __builtin_expect(X && Y, 1) -> X and Y are likely
1781         // __builtin_expect(X && Y, 0) -> only Y is unlikely
1782         EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount,
1783                              LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1784         EmitBlock(LHSTrue);
1785       }
1786 
1787       incrementProfileCounter(CondBOp);
1788       setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1789 
1790       // Any temporaries created here are conditional.
1791       eval.begin(*this);
1792       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1793                                FalseBlock, TrueCount, LH);
1794       eval.end(*this);
1795       MCDCLogOpStack.pop_back();
1796       return;
1797     }
1798 
1799     if (CondBOp->getOpcode() == BO_LOr) {
1800       MCDCLogOpStack.push_back(CondBOp);
1801 
1802       // If we have "0 || X", simplify the code.  "1 || X" would have constant
1803       // folded if the case was simple enough.
1804       bool ConstantBool = false;
1805       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1806           !ConstantBool) {
1807         // br(0 || X) -> br(X).
1808         incrementProfileCounter(CondBOp);
1809         EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock,
1810                                  FalseBlock, TrueCount, LH);
1811         MCDCLogOpStack.pop_back();
1812         return;
1813       }
1814 
1815       // If we have "X || 0", simplify the code to use an uncond branch.
1816       // "X || 1" would have been constant folded to 1.
1817       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1818           !ConstantBool) {
1819         // br(X || 0) -> br(X).
1820         EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock,
1821                                  FalseBlock, TrueCount, LH, CondBOp);
1822         MCDCLogOpStack.pop_back();
1823         return;
1824       }
1825       // Emit the LHS as a conditional.  If the LHS conditional is true, we
1826       // want to jump to the TrueBlock.
1827       llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
1828       // We have the count for entry to the RHS and for the whole expression
1829       // being true, so we can divy up True count between the short circuit and
1830       // the RHS.
1831       uint64_t LHSCount =
1832           getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());
1833       uint64_t RHSCount = TrueCount - LHSCount;
1834 
1835       ConditionalEvaluation eval(*this);
1836       {
1837         // Propagate the likelihood attribute like __builtin_expect
1838         // __builtin_expect(X || Y, 1) -> only Y is likely
1839         // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
1840         ApplyDebugLocation DL(*this, Cond);
1841         EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount,
1842                              LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
1843         EmitBlock(LHSFalse);
1844       }
1845 
1846       incrementProfileCounter(CondBOp);
1847       setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1848 
1849       // Any temporaries created here are conditional.
1850       eval.begin(*this);
1851       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, FalseBlock,
1852                                RHSCount, LH);
1853 
1854       eval.end(*this);
1855       MCDCLogOpStack.pop_back();
1856       return;
1857     }
1858   }
1859 
1860   if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
1861     // br(!x, t, f) -> br(x, f, t)
1862     // Avoid doing this optimization when instrumenting a condition for MC/DC.
1863     // LNot is taken as part of the condition for simplicity, and changing its
1864     // sense negatively impacts test vector tracking.
1865     bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
1866                          CGM.getCodeGenOpts().MCDCCoverage &&
1867                          isInstrumentedCondition(Cond);
1868     if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
1869       // Negate the count.
1870       uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
1871       // The values of the enum are chosen to make this negation possible.
1872       LH = static_cast<Stmt::Likelihood>(-LH);
1873       // Negate the condition and swap the destination blocks.
1874       return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
1875                                   FalseCount, LH);
1876     }
1877   }
1878 
1879   if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {
1880     // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
1881     llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
1882     llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
1883 
1884     // The ConditionalOperator itself has no likelihood information for its
1885     // true and false branches. This matches the behavior of __builtin_expect.
1886     ConditionalEvaluation cond(*this);
1887     EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,
1888                          getProfileCount(CondOp), Stmt::LH_None);
1889 
1890     // When computing PGO branch weights, we only know the overall count for
1891     // the true block. This code is essentially doing tail duplication of the
1892     // naive code-gen, introducing new edges for which counts are not
1893     // available. Divide the counts proportionally between the LHS and RHS of
1894     // the conditional operator.
1895     uint64_t LHSScaledTrueCount = 0;
1896     if (TrueCount) {
1897       double LHSRatio =
1898           getProfileCount(CondOp) / (double)getCurrentProfileCount();
1899       LHSScaledTrueCount = TrueCount * LHSRatio;
1900     }
1901 
1902     cond.begin(*this);
1903     EmitBlock(LHSBlock);
1904     incrementProfileCounter(CondOp);
1905     {
1906       ApplyDebugLocation DL(*this, Cond);
1907       EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
1908                            LHSScaledTrueCount, LH, CondOp);
1909     }
1910     cond.end(*this);
1911 
1912     cond.begin(*this);
1913     EmitBlock(RHSBlock);
1914     EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
1915                          TrueCount - LHSScaledTrueCount, LH, CondOp);
1916     cond.end(*this);
1917 
1918     return;
1919   }
1920 
1921   if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {
1922     // Conditional operator handling can give us a throw expression as a
1923     // condition for a case like:
1924     //   br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
1925     // Fold this to:
1926     //   br(c, throw x, br(y, t, f))
1927     EmitCXXThrowExpr(Throw, /*KeepInsertionPoint*/false);
1928     return;
1929   }
1930 
1931   // Emit the code with the fully general case.
1932   llvm::Value *CondV;
1933   {
1934     ApplyDebugLocation DL(*this, Cond);
1935     CondV = EvaluateExprAsBool(Cond);
1936   }
1937 
1938   // If not at the top of the logical operator nest, update MCDC temp with the
1939   // boolean result of the evaluated condition.
1940   if (!MCDCLogOpStack.empty()) {
1941     const Expr *MCDCBaseExpr = Cond;
1942     // When a nested ConditionalOperator (ternary) is encountered in a boolean
1943     // expression, MC/DC tracks the result of the ternary, and this is tied to
1944     // the ConditionalOperator expression and not the ternary's LHS or RHS. If
1945     // this is the case, the ConditionalOperator expression is passed through
1946     // the ConditionalOp parameter and then used as the MCDC base expression.
1947     if (ConditionalOp)
1948       MCDCBaseExpr = ConditionalOp;
1949 
1950     maybeUpdateMCDCCondBitmap(MCDCBaseExpr, CondV);
1951   }
1952 
1953   llvm::MDNode *Weights = nullptr;
1954   llvm::MDNode *Unpredictable = nullptr;
1955 
1956   // If the branch has a condition wrapped by __builtin_unpredictable,
1957   // create metadata that specifies that the branch is unpredictable.
1958   // Don't bother if not optimizing because that metadata would not be used.
1959   auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
1960   if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
1961     auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1962     if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1963       llvm::MDBuilder MDHelper(getLLVMContext());
1964       Unpredictable = MDHelper.createUnpredictable();
1965     }
1966   }
1967 
1968   // If there is a Likelihood knowledge for the cond, lower it.
1969   // Note that if not optimizing this won't emit anything.
1970   llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
1971   if (CondV != NewCondV)
1972     CondV = NewCondV;
1973   else {
1974     // Otherwise, lower profile counts. Note that we do this even at -O0.
1975     uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
1976     Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
1977   }
1978 
1979   Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
1980 }
1981 
1982 /// ErrorUnsupported - Print out an error that codegen doesn't support the
1983 /// specified stmt yet.
1984 void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
1985   CGM.ErrorUnsupported(S, Type);
1986 }
1987 
1988 /// emitNonZeroVLAInit - Emit the "zero" initialization of a
1989 /// variable-length array whose elements have a non-zero bit-pattern.
1990 ///
1991 /// \param baseType the inner-most element type of the array
1992 /// \param src - a char* pointing to the bit-pattern for a single
1993 /// base element of the array
1994 /// \param sizeInChars - the total size of the VLA, in chars
1995 static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
1996                                Address dest, Address src,
1997                                llvm::Value *sizeInChars) {
1998   CGBuilderTy &Builder = CGF.Builder;
1999 
2000   CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);
2001   llvm::Value *baseSizeInChars
2002     = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
2003 
2004   Address begin = dest.withElementType(CGF.Int8Ty);
2005   llvm::Value *end = Builder.CreateInBoundsGEP(
2006       begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end");
2007 
2008   llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
2009   llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
2010   llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");
2011 
2012   // Make a loop over the VLA.  C99 guarantees that the VLA element
2013   // count must be nonzero.
2014   CGF.EmitBlock(loopBB);
2015 
2016   llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
2017   cur->addIncoming(begin.getPointer(), originBB);
2018 
2019   CharUnits curAlign =
2020     dest.getAlignment().alignmentOfArrayElement(baseSize);
2021 
2022   // memcpy the individual element bit-pattern.
2023   Builder.CreateMemCpy(Address(cur, CGF.Int8Ty, curAlign), src, baseSizeInChars,
2024                        /*volatile*/ false);
2025 
2026   // Go to the next element.
2027   llvm::Value *next =
2028     Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");
2029 
2030   // Leave if that's the end of the VLA.
2031   llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
2032   Builder.CreateCondBr(done, contBB, loopBB);
2033   cur->addIncoming(next, loopBB);
2034 
2035   CGF.EmitBlock(contBB);
2036 }
2037 
2038 void
2039 CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
2040   // Ignore empty classes in C++.
2041   if (getLangOpts().CPlusPlus) {
2042     if (const RecordType *RT = Ty->getAs<RecordType>()) {
2043       if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
2044         return;
2045     }
2046   }
2047 
2048   if (DestPtr.getElementType() != Int8Ty)
2049     DestPtr = DestPtr.withElementType(Int8Ty);
2050 
2051   // Get size and alignment info for this aggregate.
2052   CharUnits size = getContext().getTypeSizeInChars(Ty);
2053 
2054   llvm::Value *SizeVal;
2055   const VariableArrayType *vla;
2056 
2057   // Don't bother emitting a zero-byte memset.
2058   if (size.isZero()) {
2059     // But note that getTypeInfo returns 0 for a VLA.
2060     if (const VariableArrayType *vlaType =
2061           dyn_cast_or_null<VariableArrayType>(
2062                                           getContext().getAsArrayType(Ty))) {
2063       auto VlaSize = getVLASize(vlaType);
2064       SizeVal = VlaSize.NumElts;
2065       CharUnits eltSize = getContext().getTypeSizeInChars(VlaSize.Type);
2066       if (!eltSize.isOne())
2067         SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(eltSize));
2068       vla = vlaType;
2069     } else {
2070       return;
2071     }
2072   } else {
2073     SizeVal = CGM.getSize(size);
2074     vla = nullptr;
2075   }
2076 
2077   // If the type contains a pointer to data member we can't memset it to zero.
2078   // Instead, create a null constant and copy it to the destination.
2079   // TODO: there are other patterns besides zero that we can usefully memset,
2080   // like -1, which happens to be the pattern used by member-pointers.
2081   if (!CGM.getTypes().isZeroInitializable(Ty)) {
2082     // For a VLA, emit a single element, then splat that over the VLA.
2083     if (vla) Ty = getContext().getBaseElementType(vla);
2084 
2085     llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
2086 
2087     llvm::GlobalVariable *NullVariable =
2088       new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2089                                /*isConstant=*/true,
2090                                llvm::GlobalVariable::PrivateLinkage,
2091                                NullConstant, Twine());
2092     CharUnits NullAlign = DestPtr.getAlignment();
2093     NullVariable->setAlignment(NullAlign.getAsAlign());
2094     Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);
2095 
2096     if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
2097 
2098     // Get and call the appropriate llvm.memcpy overload.
2099     Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
2100     return;
2101   }
2102 
2103   // Otherwise, just memset the whole thing to zero.  This is legal
2104   // because in LLVM, all default initializers (other than the ones we just
2105   // handled above) are guaranteed to have a bit pattern of all zeros.
2106   Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
2107 }
2108 
2109 llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2110   // Make sure that there is a block for the indirect goto.
2111   if (!IndirectBranch)
2112     GetIndirectGotoBlock();
2113 
2114   llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
2115 
2116   // Make sure the indirect branch includes all of the address-taken blocks.
2117   IndirectBranch->addDestination(BB);
2118   return llvm::BlockAddress::get(CurFn, BB);
2119 }
2120 
2121 llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2122   // If we already made the indirect branch for indirect goto, return its block.
2123   if (IndirectBranch) return IndirectBranch->getParent();
2124 
2125   CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));
2126 
2127   // Create the PHI node that indirect gotos will add entries to.
2128   llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,
2129                                               "indirect.goto.dest");
2130 
2131   // Create the indirect branch instruction.
2132   IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
2133   return IndirectBranch->getParent();
2134 }
2135 
2136 /// Computes the length of an array in elements, as well as the base
2137 /// element type and a properly-typed first element pointer.
2138 llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2139                                               QualType &baseType,
2140                                               Address &addr) {
2141   const ArrayType *arrayType = origArrayType;
2142 
2143   // If it's a VLA, we have to load the stored size.  Note that
2144   // this is the size of the VLA in bytes, not its size in elements.
2145   llvm::Value *numVLAElements = nullptr;
2146   if (isa<VariableArrayType>(arrayType)) {
2147     numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).NumElts;
2148 
2149     // Walk into all VLAs.  This doesn't require changes to addr,
2150     // which has type T* where T is the first non-VLA element type.
2151     do {
2152       QualType elementType = arrayType->getElementType();
2153       arrayType = getContext().getAsArrayType(elementType);
2154 
2155       // If we only have VLA components, 'addr' requires no adjustment.
2156       if (!arrayType) {
2157         baseType = elementType;
2158         return numVLAElements;
2159       }
2160     } while (isa<VariableArrayType>(arrayType));
2161 
2162     // We get out here only if we find a constant array type
2163     // inside the VLA.
2164   }
2165 
2166   // We have some number of constant-length arrays, so addr should
2167   // have LLVM type [M x [N x [...]]]*.  Build a GEP that walks
2168   // down to the first element of addr.
2169   SmallVector<llvm::Value*, 8> gepIndices;
2170 
2171   // GEP down to the array type.
2172   llvm::ConstantInt *zero = Builder.getInt32(0);
2173   gepIndices.push_back(zero);
2174 
2175   uint64_t countFromCLAs = 1;
2176   QualType eltType;
2177 
2178   llvm::ArrayType *llvmArrayType =
2179     dyn_cast<llvm::ArrayType>(addr.getElementType());
2180   while (llvmArrayType) {
2181     assert(isa<ConstantArrayType>(arrayType));
2182     assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
2183              == llvmArrayType->getNumElements());
2184 
2185     gepIndices.push_back(zero);
2186     countFromCLAs *= llvmArrayType->getNumElements();
2187     eltType = arrayType->getElementType();
2188 
2189     llvmArrayType =
2190       dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());
2191     arrayType = getContext().getAsArrayType(arrayType->getElementType());
2192     assert((!llvmArrayType || arrayType) &&
2193            "LLVM and Clang types are out-of-synch");
2194   }
2195 
2196   if (arrayType) {
2197     // From this point onwards, the Clang array type has been emitted
2198     // as some other type (probably a packed struct). Compute the array
2199     // size, and just emit the 'begin' expression as a bitcast.
2200     while (arrayType) {
2201       countFromCLAs *=
2202           cast<ConstantArrayType>(arrayType)->getSize().getZExtValue();
2203       eltType = arrayType->getElementType();
2204       arrayType = getContext().getAsArrayType(eltType);
2205     }
2206 
2207     llvm::Type *baseType = ConvertType(eltType);
2208     addr = addr.withElementType(baseType);
2209   } else {
2210     // Create the actual GEP.
2211     addr = Address(Builder.CreateInBoundsGEP(
2212         addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"),
2213         ConvertTypeForMem(eltType),
2214         addr.getAlignment());
2215   }
2216 
2217   baseType = eltType;
2218 
2219   llvm::Value *numElements
2220     = llvm::ConstantInt::get(SizeTy, countFromCLAs);
2221 
2222   // If we had any VLA dimensions, factor them in.
2223   if (numVLAElements)
2224     numElements = Builder.CreateNUWMul(numVLAElements, numElements);
2225 
2226   return numElements;
2227 }
2228 
2229 CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2230   const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2231   assert(vla && "type was not a variable array type!");
2232   return getVLASize(vla);
2233 }
2234 
2235 CodeGenFunction::VlaSizePair
2236 CodeGenFunction::getVLASize(const VariableArrayType *type) {
2237   // The number of elements so far; always size_t.
2238   llvm::Value *numElements = nullptr;
2239 
2240   QualType elementType;
2241   do {
2242     elementType = type->getElementType();
2243     llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2244     assert(vlaSize && "no size for VLA!");
2245     assert(vlaSize->getType() == SizeTy);
2246 
2247     if (!numElements) {
2248       numElements = vlaSize;
2249     } else {
2250       // It's undefined behavior if this wraps around, so mark it that way.
2251       // FIXME: Teach -fsanitize=undefined to trap this.
2252       numElements = Builder.CreateNUWMul(numElements, vlaSize);
2253     }
2254   } while ((type = getContext().getAsVariableArrayType(elementType)));
2255 
2256   return { numElements, elementType };
2257 }
2258 
2259 CodeGenFunction::VlaSizePair
2260 CodeGenFunction::getVLAElements1D(QualType type) {
2261   const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2262   assert(vla && "type was not a variable array type!");
2263   return getVLAElements1D(vla);
2264 }
2265 
2266 CodeGenFunction::VlaSizePair
2267 CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2268   llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2269   assert(VlaSize && "no size for VLA!");
2270   assert(VlaSize->getType() == SizeTy);
2271   return { VlaSize, Vla->getElementType() };
2272 }
2273 
2274 void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2275   assert(type->isVariablyModifiedType() &&
2276          "Must pass variably modified type to EmitVLASizes!");
2277 
2278   EnsureInsertPoint();
2279 
2280   // We're going to walk down into the type and look for VLA
2281   // expressions.
2282   do {
2283     assert(type->isVariablyModifiedType());
2284 
2285     const Type *ty = type.getTypePtr();
2286     switch (ty->getTypeClass()) {
2287 
2288 #define TYPE(Class, Base)
2289 #define ABSTRACT_TYPE(Class, Base)
2290 #define NON_CANONICAL_TYPE(Class, Base)
2291 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
2292 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2293 #include "clang/AST/TypeNodes.inc"
2294       llvm_unreachable("unexpected dependent type!");
2295 
2296     // These types are never variably-modified.
2297     case Type::Builtin:
2298     case Type::Complex:
2299     case Type::Vector:
2300     case Type::ExtVector:
2301     case Type::ConstantMatrix:
2302     case Type::Record:
2303     case Type::Enum:
2304     case Type::Using:
2305     case Type::TemplateSpecialization:
2306     case Type::ObjCTypeParam:
2307     case Type::ObjCObject:
2308     case Type::ObjCInterface:
2309     case Type::ObjCObjectPointer:
2310     case Type::BitInt:
2311       llvm_unreachable("type class is never variably-modified!");
2312 
2313     case Type::Elaborated:
2314       type = cast<ElaboratedType>(ty)->getNamedType();
2315       break;
2316 
2317     case Type::Adjusted:
2318       type = cast<AdjustedType>(ty)->getAdjustedType();
2319       break;
2320 
2321     case Type::Decayed:
2322       type = cast<DecayedType>(ty)->getPointeeType();
2323       break;
2324 
2325     case Type::Pointer:
2326       type = cast<PointerType>(ty)->getPointeeType();
2327       break;
2328 
2329     case Type::BlockPointer:
2330       type = cast<BlockPointerType>(ty)->getPointeeType();
2331       break;
2332 
2333     case Type::LValueReference:
2334     case Type::RValueReference:
2335       type = cast<ReferenceType>(ty)->getPointeeType();
2336       break;
2337 
2338     case Type::MemberPointer:
2339       type = cast<MemberPointerType>(ty)->getPointeeType();
2340       break;
2341 
2342     case Type::ConstantArray:
2343     case Type::IncompleteArray:
2344       // Losing element qualification here is fine.
2345       type = cast<ArrayType>(ty)->getElementType();
2346       break;
2347 
2348     case Type::VariableArray: {
2349       // Losing element qualification here is fine.
2350       const VariableArrayType *vat = cast<VariableArrayType>(ty);
2351 
2352       // Unknown size indication requires no size computation.
2353       // Otherwise, evaluate and record it.
2354       if (const Expr *sizeExpr = vat->getSizeExpr()) {
2355         // It's possible that we might have emitted this already,
2356         // e.g. with a typedef and a pointer to it.
2357         llvm::Value *&entry = VLASizeMap[sizeExpr];
2358         if (!entry) {
2359           llvm::Value *size = EmitScalarExpr(sizeExpr);
2360 
2361           // C11 6.7.6.2p5:
2362           //   If the size is an expression that is not an integer constant
2363           //   expression [...] each time it is evaluated it shall have a value
2364           //   greater than zero.
2365           if (SanOpts.has(SanitizerKind::VLABound)) {
2366             SanitizerScope SanScope(this);
2367             llvm::Value *Zero = llvm::Constant::getNullValue(size->getType());
2368             clang::QualType SEType = sizeExpr->getType();
2369             llvm::Value *CheckCondition =
2370                 SEType->isSignedIntegerType()
2371                     ? Builder.CreateICmpSGT(size, Zero)
2372                     : Builder.CreateICmpUGT(size, Zero);
2373             llvm::Constant *StaticArgs[] = {
2374                 EmitCheckSourceLocation(sizeExpr->getBeginLoc()),
2375                 EmitCheckTypeDescriptor(SEType)};
2376             EmitCheck(std::make_pair(CheckCondition, SanitizerKind::VLABound),
2377                       SanitizerHandler::VLABoundNotPositive, StaticArgs, size);
2378           }
2379 
2380           // Always zexting here would be wrong if it weren't
2381           // undefined behavior to have a negative bound.
2382           // FIXME: What about when size's type is larger than size_t?
2383           entry = Builder.CreateIntCast(size, SizeTy, /*signed*/ false);
2384         }
2385       }
2386       type = vat->getElementType();
2387       break;
2388     }
2389 
2390     case Type::FunctionProto:
2391     case Type::FunctionNoProto:
2392       type = cast<FunctionType>(ty)->getReturnType();
2393       break;
2394 
2395     case Type::Paren:
2396     case Type::TypeOf:
2397     case Type::UnaryTransform:
2398     case Type::Attributed:
2399     case Type::BTFTagAttributed:
2400     case Type::SubstTemplateTypeParm:
2401     case Type::MacroQualified:
2402       // Keep walking after single level desugaring.
2403       type = type.getSingleStepDesugaredType(getContext());
2404       break;
2405 
2406     case Type::Typedef:
2407     case Type::Decltype:
2408     case Type::Auto:
2409     case Type::DeducedTemplateSpecialization:
2410       // Stop walking: nothing to do.
2411       return;
2412 
2413     case Type::TypeOfExpr:
2414       // Stop walking: emit typeof expression.
2415       EmitIgnoredExpr(cast<TypeOfExprType>(ty)->getUnderlyingExpr());
2416       return;
2417 
2418     case Type::Atomic:
2419       type = cast<AtomicType>(ty)->getValueType();
2420       break;
2421 
2422     case Type::Pipe:
2423       type = cast<PipeType>(ty)->getElementType();
2424       break;
2425     }
2426   } while (type->isVariablyModifiedType());
2427 }
2428 
2429 Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2430   if (getContext().getBuiltinVaListType()->isArrayType())
2431     return EmitPointerWithAlignment(E);
2432   return EmitLValue(E).getAddress(*this);
2433 }
2434 
2435 Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2436   return EmitLValue(E).getAddress(*this);
2437 }
2438 
2439 void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2440                                               const APValue &Init) {
2441   assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2442   if (CGDebugInfo *Dbg = getDebugInfo())
2443     if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2444       Dbg->EmitGlobalVariable(E->getDecl(), Init);
2445 }
2446 
2447 CodeGenFunction::PeepholeProtection
2448 CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2449   // At the moment, the only aggressive peephole we do in IR gen
2450   // is trunc(zext) folding, but if we add more, we can easily
2451   // extend this protection.
2452 
2453   if (!rvalue.isScalar()) return PeepholeProtection();
2454   llvm::Value *value = rvalue.getScalarVal();
2455   if (!isa<llvm::ZExtInst>(value)) return PeepholeProtection();
2456 
2457   // Just make an extra bitcast.
2458   assert(HaveInsertPoint());
2459   llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2460                                                   Builder.GetInsertBlock());
2461 
2462   PeepholeProtection protection;
2463   protection.Inst = inst;
2464   return protection;
2465 }
2466 
2467 void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2468   if (!protection.Inst) return;
2469 
2470   // In theory, we could try to duplicate the peepholes now, but whatever.
2471   protection.Inst->eraseFromParent();
2472 }
2473 
2474 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2475                                               QualType Ty, SourceLocation Loc,
2476                                               SourceLocation AssumptionLoc,
2477                                               llvm::Value *Alignment,
2478                                               llvm::Value *OffsetValue) {
2479   if (Alignment->getType() != IntPtrTy)
2480     Alignment =
2481         Builder.CreateIntCast(Alignment, IntPtrTy, false, "casted.align");
2482   if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2483     OffsetValue =
2484         Builder.CreateIntCast(OffsetValue, IntPtrTy, true, "casted.offset");
2485   llvm::Value *TheCheck = nullptr;
2486   if (SanOpts.has(SanitizerKind::Alignment)) {
2487     llvm::Value *PtrIntValue =
2488         Builder.CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");
2489 
2490     if (OffsetValue) {
2491       bool IsOffsetZero = false;
2492       if (const auto *CI = dyn_cast<llvm::ConstantInt>(OffsetValue))
2493         IsOffsetZero = CI->isZero();
2494 
2495       if (!IsOffsetZero)
2496         PtrIntValue = Builder.CreateSub(PtrIntValue, OffsetValue, "offsetptr");
2497     }
2498 
2499     llvm::Value *Zero = llvm::ConstantInt::get(IntPtrTy, 0);
2500     llvm::Value *Mask =
2501         Builder.CreateSub(Alignment, llvm::ConstantInt::get(IntPtrTy, 1));
2502     llvm::Value *MaskedPtr = Builder.CreateAnd(PtrIntValue, Mask, "maskedptr");
2503     TheCheck = Builder.CreateICmpEQ(MaskedPtr, Zero, "maskcond");
2504   }
2505   llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2506       CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2507 
2508   if (!SanOpts.has(SanitizerKind::Alignment))
2509     return;
2510   emitAlignmentAssumptionCheck(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2511                                OffsetValue, TheCheck, Assumption);
2512 }
2513 
2514 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2515                                               const Expr *E,
2516                                               SourceLocation AssumptionLoc,
2517                                               llvm::Value *Alignment,
2518                                               llvm::Value *OffsetValue) {
2519   QualType Ty = E->getType();
2520   SourceLocation Loc = E->getExprLoc();
2521 
2522   emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2523                           OffsetValue);
2524 }
2525 
2526 llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2527                                                  llvm::Value *AnnotatedVal,
2528                                                  StringRef AnnotationStr,
2529                                                  SourceLocation Location,
2530                                                  const AnnotateAttr *Attr) {
2531   SmallVector<llvm::Value *, 5> Args = {
2532       AnnotatedVal,
2533       CGM.EmitAnnotationString(AnnotationStr),
2534       CGM.EmitAnnotationUnit(Location),
2535       CGM.EmitAnnotationLineNo(Location),
2536   };
2537   if (Attr)
2538     Args.push_back(CGM.EmitAnnotationArgs(Attr));
2539   return Builder.CreateCall(AnnotationFn, Args);
2540 }
2541 
2542 void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2543   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2544   for (const auto *I : D->specific_attrs<AnnotateAttr>())
2545     EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation,
2546                                         {V->getType(), CGM.ConstGlobalsPtrTy}),
2547                        V, I->getAnnotation(), D->getLocation(), I);
2548 }
2549 
2550 Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2551                                               Address Addr) {
2552   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2553   llvm::Value *V = Addr.getPointer();
2554   llvm::Type *VTy = V->getType();
2555   auto *PTy = dyn_cast<llvm::PointerType>(VTy);
2556   unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2557   llvm::PointerType *IntrinTy =
2558       llvm::PointerType::get(CGM.getLLVMContext(), AS);
2559   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
2560                                        {IntrinTy, CGM.ConstGlobalsPtrTy});
2561 
2562   for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2563     // FIXME Always emit the cast inst so we can differentiate between
2564     // annotation on the first field of a struct and annotation on the struct
2565     // itself.
2566     if (VTy != IntrinTy)
2567       V = Builder.CreateBitCast(V, IntrinTy);
2568     V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation(), I);
2569     V = Builder.CreateBitCast(V, VTy);
2570   }
2571 
2572   return Address(V, Addr.getElementType(), Addr.getAlignment());
2573 }
2574 
2575 CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2576 
2577 CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2578     : CGF(CGF) {
2579   assert(!CGF->IsSanitizerScope);
2580   CGF->IsSanitizerScope = true;
2581 }
2582 
2583 CodeGenFunction::SanitizerScope::~SanitizerScope() {
2584   CGF->IsSanitizerScope = false;
2585 }
2586 
2587 void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2588                                    const llvm::Twine &Name,
2589                                    llvm::BasicBlock *BB,
2590                                    llvm::BasicBlock::iterator InsertPt) const {
2591   LoopStack.InsertHelper(I);
2592   if (IsSanitizerScope)
2593     I->setNoSanitizeMetadata();
2594 }
2595 
2596 void CGBuilderInserter::InsertHelper(
2597     llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
2598     llvm::BasicBlock::iterator InsertPt) const {
2599   llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
2600   if (CGF)
2601     CGF->InsertHelper(I, Name, BB, InsertPt);
2602 }
2603 
2604 // Emits an error if we don't have a valid set of target features for the
2605 // called function.
2606 void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2607                                           const FunctionDecl *TargetDecl) {
2608   return checkTargetFeatures(E->getBeginLoc(), TargetDecl);
2609 }
2610 
2611 // Emits an error if we don't have a valid set of target features for the
2612 // called function.
2613 void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2614                                           const FunctionDecl *TargetDecl) {
2615   // Early exit if this is an indirect call.
2616   if (!TargetDecl)
2617     return;
2618 
2619   // Get the current enclosing function if it exists. If it doesn't
2620   // we can't check the target features anyhow.
2621   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);
2622   if (!FD)
2623     return;
2624 
2625   // Grab the required features for the call. For a builtin this is listed in
2626   // the td file with the default cpu, for an always_inline function this is any
2627   // listed cpu and any listed features.
2628   unsigned BuiltinID = TargetDecl->getBuiltinID();
2629   std::string MissingFeature;
2630   llvm::StringMap<bool> CallerFeatureMap;
2631   CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
2632   // When compiling in HipStdPar mode we have to be conservative in rejecting
2633   // target specific features in the FE, and defer the possible error to the
2634   // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
2635   // referenced by an accelerator executable function, we emit an error.
2636   bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2637   if (BuiltinID) {
2638     StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
2639     if (!Builtin::evaluateRequiredTargetFeatures(
2640         FeatureList, CallerFeatureMap) && !IsHipStdPar) {
2641       CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
2642           << TargetDecl->getDeclName()
2643           << FeatureList;
2644     }
2645   } else if (!TargetDecl->isMultiVersion() &&
2646              TargetDecl->hasAttr<TargetAttr>()) {
2647     // Get the required features for the callee.
2648 
2649     const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2650     ParsedTargetAttr ParsedAttr =
2651         CGM.getContext().filterFunctionTargetAttrs(TD);
2652 
2653     SmallVector<StringRef, 1> ReqFeatures;
2654     llvm::StringMap<bool> CalleeFeatureMap;
2655     CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
2656 
2657     for (const auto &F : ParsedAttr.Features) {
2658       if (F[0] == '+' && CalleeFeatureMap.lookup(F.substr(1)))
2659         ReqFeatures.push_back(StringRef(F).substr(1));
2660     }
2661 
2662     for (const auto &F : CalleeFeatureMap) {
2663       // Only positive features are "required".
2664       if (F.getValue())
2665         ReqFeatures.push_back(F.getKey());
2666     }
2667     if (!llvm::all_of(ReqFeatures, [&](StringRef Feature) {
2668       if (!CallerFeatureMap.lookup(Feature)) {
2669         MissingFeature = Feature.str();
2670         return false;
2671       }
2672       return true;
2673     }) && !IsHipStdPar)
2674       CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2675           << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2676   } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
2677     llvm::StringMap<bool> CalleeFeatureMap;
2678     CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
2679 
2680     for (const auto &F : CalleeFeatureMap) {
2681       if (F.getValue() && (!CallerFeatureMap.lookup(F.getKey()) ||
2682                            !CallerFeatureMap.find(F.getKey())->getValue()) &&
2683           !IsHipStdPar)
2684         CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2685             << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
2686     }
2687   }
2688 }
2689 
2690 void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2691   if (!CGM.getCodeGenOpts().SanitizeStats)
2692     return;
2693 
2694   llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2695   IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2696   CGM.getSanStats().create(IRB, SSK);
2697 }
2698 
2699 void CodeGenFunction::EmitKCFIOperandBundle(
2700     const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2701   const FunctionProtoType *FP =
2702       Callee.getAbstractInfo().getCalleeFunctionProtoType();
2703   if (FP)
2704     Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar()));
2705 }
2706 
2707 llvm::Value *CodeGenFunction::FormAArch64ResolverCondition(
2708     const MultiVersionResolverOption &RO) {
2709   llvm::SmallVector<StringRef, 8> CondFeatures;
2710   for (const StringRef &Feature : RO.Conditions.Features) {
2711     // Form condition for features which are not yet enabled in target
2712     if (!getContext().getTargetInfo().hasFeature(Feature))
2713       CondFeatures.push_back(Feature);
2714   }
2715   if (!CondFeatures.empty()) {
2716     return EmitAArch64CpuSupports(CondFeatures);
2717   }
2718   return nullptr;
2719 }
2720 
2721 llvm::Value *CodeGenFunction::FormX86ResolverCondition(
2722     const MultiVersionResolverOption &RO) {
2723   llvm::Value *Condition = nullptr;
2724 
2725   if (!RO.Conditions.Architecture.empty()) {
2726     StringRef Arch = RO.Conditions.Architecture;
2727     // If arch= specifies an x86-64 micro-architecture level, test the feature
2728     // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
2729     if (Arch.starts_with("x86-64"))
2730       Condition = EmitX86CpuSupports({Arch});
2731     else
2732       Condition = EmitX86CpuIs(Arch);
2733   }
2734 
2735   if (!RO.Conditions.Features.empty()) {
2736     llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
2737     Condition =
2738         Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
2739   }
2740   return Condition;
2741 }
2742 
2743 static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2744                                              llvm::Function *Resolver,
2745                                              CGBuilderTy &Builder,
2746                                              llvm::Function *FuncToReturn,
2747                                              bool SupportsIFunc) {
2748   if (SupportsIFunc) {
2749     Builder.CreateRet(FuncToReturn);
2750     return;
2751   }
2752 
2753   llvm::SmallVector<llvm::Value *, 10> Args(
2754       llvm::make_pointer_range(Resolver->args()));
2755 
2756   llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
2757   Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2758 
2759   if (Resolver->getReturnType()->isVoidTy())
2760     Builder.CreateRetVoid();
2761   else
2762     Builder.CreateRet(Result);
2763 }
2764 
2765 void CodeGenFunction::EmitMultiVersionResolver(
2766     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2767 
2768   llvm::Triple::ArchType ArchType =
2769       getContext().getTargetInfo().getTriple().getArch();
2770 
2771   switch (ArchType) {
2772   case llvm::Triple::x86:
2773   case llvm::Triple::x86_64:
2774     EmitX86MultiVersionResolver(Resolver, Options);
2775     return;
2776   case llvm::Triple::aarch64:
2777     EmitAArch64MultiVersionResolver(Resolver, Options);
2778     return;
2779 
2780   default:
2781     assert(false && "Only implemented for x86 and AArch64 targets");
2782   }
2783 }
2784 
2785 void CodeGenFunction::EmitAArch64MultiVersionResolver(
2786     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2787   assert(!Options.empty() && "No multiversion resolver options found");
2788   assert(Options.back().Conditions.Features.size() == 0 &&
2789          "Default case must be last");
2790   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2791   assert(SupportsIFunc &&
2792          "Multiversion resolver requires target IFUNC support");
2793   bool AArch64CpuInitialized = false;
2794   llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2795 
2796   for (const MultiVersionResolverOption &RO : Options) {
2797     Builder.SetInsertPoint(CurBlock);
2798     llvm::Value *Condition = FormAArch64ResolverCondition(RO);
2799 
2800     // The 'default' or 'all features enabled' case.
2801     if (!Condition) {
2802       CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2803                                        SupportsIFunc);
2804       return;
2805     }
2806 
2807     if (!AArch64CpuInitialized) {
2808       Builder.SetInsertPoint(CurBlock, CurBlock->begin());
2809       EmitAArch64CpuInit();
2810       AArch64CpuInitialized = true;
2811       Builder.SetInsertPoint(CurBlock);
2812     }
2813 
2814     llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2815     CGBuilderTy RetBuilder(*this, RetBlock);
2816     CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2817                                      SupportsIFunc);
2818     CurBlock = createBasicBlock("resolver_else", Resolver);
2819     Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2820   }
2821 
2822   // If no default, emit an unreachable.
2823   Builder.SetInsertPoint(CurBlock);
2824   llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2825   TrapCall->setDoesNotReturn();
2826   TrapCall->setDoesNotThrow();
2827   Builder.CreateUnreachable();
2828   Builder.ClearInsertionPoint();
2829 }
2830 
2831 void CodeGenFunction::EmitX86MultiVersionResolver(
2832     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2833 
2834   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2835 
2836   // Main function's basic block.
2837   llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2838   Builder.SetInsertPoint(CurBlock);
2839   EmitX86CpuInit();
2840 
2841   for (const MultiVersionResolverOption &RO : Options) {
2842     Builder.SetInsertPoint(CurBlock);
2843     llvm::Value *Condition = FormX86ResolverCondition(RO);
2844 
2845     // The 'default' or 'generic' case.
2846     if (!Condition) {
2847       assert(&RO == Options.end() - 1 &&
2848              "Default or Generic case must be last");
2849       CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2850                                        SupportsIFunc);
2851       return;
2852     }
2853 
2854     llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2855     CGBuilderTy RetBuilder(*this, RetBlock);
2856     CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2857                                      SupportsIFunc);
2858     CurBlock = createBasicBlock("resolver_else", Resolver);
2859     Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2860   }
2861 
2862   // If no generic/default, emit an unreachable.
2863   Builder.SetInsertPoint(CurBlock);
2864   llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2865   TrapCall->setDoesNotReturn();
2866   TrapCall->setDoesNotThrow();
2867   Builder.CreateUnreachable();
2868   Builder.ClearInsertionPoint();
2869 }
2870 
2871 // Loc - where the diagnostic will point, where in the source code this
2872 //  alignment has failed.
2873 // SecondaryLoc - if present (will be present if sufficiently different from
2874 //  Loc), the diagnostic will additionally point a "Note:" to this location.
2875 //  It should be the location where the __attribute__((assume_aligned))
2876 //  was written e.g.
2877 void CodeGenFunction::emitAlignmentAssumptionCheck(
2878     llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
2879     SourceLocation SecondaryLoc, llvm::Value *Alignment,
2880     llvm::Value *OffsetValue, llvm::Value *TheCheck,
2881     llvm::Instruction *Assumption) {
2882   assert(Assumption && isa<llvm::CallInst>(Assumption) &&
2883          cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
2884              llvm::Intrinsic::getDeclaration(
2885                  Builder.GetInsertBlock()->getParent()->getParent(),
2886                  llvm::Intrinsic::assume) &&
2887          "Assumption should be a call to llvm.assume().");
2888   assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
2889          "Assumption should be the last instruction of the basic block, "
2890          "since the basic block is still being generated.");
2891 
2892   if (!SanOpts.has(SanitizerKind::Alignment))
2893     return;
2894 
2895   // Don't check pointers to volatile data. The behavior here is implementation-
2896   // defined.
2897   if (Ty->getPointeeType().isVolatileQualified())
2898     return;
2899 
2900   // We need to temorairly remove the assumption so we can insert the
2901   // sanitizer check before it, else the check will be dropped by optimizations.
2902   Assumption->removeFromParent();
2903 
2904   {
2905     SanitizerScope SanScope(this);
2906 
2907     if (!OffsetValue)
2908       OffsetValue = Builder.getInt1(false); // no offset.
2909 
2910     llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
2911                                     EmitCheckSourceLocation(SecondaryLoc),
2912                                     EmitCheckTypeDescriptor(Ty)};
2913     llvm::Value *DynamicData[] = {EmitCheckValue(Ptr),
2914                                   EmitCheckValue(Alignment),
2915                                   EmitCheckValue(OffsetValue)};
2916     EmitCheck({std::make_pair(TheCheck, SanitizerKind::Alignment)},
2917               SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
2918   }
2919 
2920   // We are now in the (new, empty) "cont" basic block.
2921   // Reintroduce the assumption.
2922   Builder.Insert(Assumption);
2923   // FIXME: Assumption still has it's original basic block as it's Parent.
2924 }
2925 
2926 llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
2927   if (CGDebugInfo *DI = getDebugInfo())
2928     return DI->SourceLocToDebugLoc(Location);
2929 
2930   return llvm::DebugLoc();
2931 }
2932 
2933 llvm::Value *
2934 CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
2935                                                       Stmt::Likelihood LH) {
2936   switch (LH) {
2937   case Stmt::LH_None:
2938     return Cond;
2939   case Stmt::LH_Likely:
2940   case Stmt::LH_Unlikely:
2941     // Don't generate llvm.expect on -O0 as the backend won't use it for
2942     // anything.
2943     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2944       return Cond;
2945     llvm::Type *CondTy = Cond->getType();
2946     assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
2947     llvm::Function *FnExpect =
2948         CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
2949     llvm::Value *ExpectedValueOfCond =
2950         llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
2951     return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
2952                               Cond->getName() + ".expval");
2953   }
2954   llvm_unreachable("Unknown Likelihood");
2955 }
2956 
2957 llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
2958                                                     unsigned NumElementsDst,
2959                                                     const llvm::Twine &Name) {
2960   auto *SrcTy = cast<llvm::FixedVectorType>(SrcVec->getType());
2961   unsigned NumElementsSrc = SrcTy->getNumElements();
2962   if (NumElementsSrc == NumElementsDst)
2963     return SrcVec;
2964 
2965   std::vector<int> ShuffleMask(NumElementsDst, -1);
2966   for (unsigned MaskIdx = 0;
2967        MaskIdx < std::min<>(NumElementsDst, NumElementsSrc); ++MaskIdx)
2968     ShuffleMask[MaskIdx] = MaskIdx;
2969 
2970   return Builder.CreateShuffleVector(SrcVec, ShuffleMask, Name);
2971 }
2972