xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp (revision da5432eda807c4b7232d030d5157d5b417ea4f52)
1 //===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This coordinates the per-function state used while generating code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CodeGenFunction.h"
14 #include "CGBlocks.h"
15 #include "CGCUDARuntime.h"
16 #include "CGCXXABI.h"
17 #include "CGCleanup.h"
18 #include "CGDebugInfo.h"
19 #include "CGHLSLRuntime.h"
20 #include "CGOpenMPRuntime.h"
21 #include "CodeGenModule.h"
22 #include "CodeGenPGO.h"
23 #include "TargetInfo.h"
24 #include "clang/AST/ASTContext.h"
25 #include "clang/AST/ASTLambda.h"
26 #include "clang/AST/Attr.h"
27 #include "clang/AST/Decl.h"
28 #include "clang/AST/DeclCXX.h"
29 #include "clang/AST/Expr.h"
30 #include "clang/AST/StmtCXX.h"
31 #include "clang/AST/StmtObjC.h"
32 #include "clang/Basic/Builtins.h"
33 #include "clang/Basic/CodeGenOptions.h"
34 #include "clang/Basic/TargetInfo.h"
35 #include "clang/CodeGen/CGFunctionInfo.h"
36 #include "clang/Frontend/FrontendDiagnostic.h"
37 #include "llvm/ADT/ArrayRef.h"
38 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
39 #include "llvm/IR/DataLayout.h"
40 #include "llvm/IR/Dominators.h"
41 #include "llvm/IR/FPEnv.h"
42 #include "llvm/IR/IntrinsicInst.h"
43 #include "llvm/IR/Intrinsics.h"
44 #include "llvm/IR/MDBuilder.h"
45 #include "llvm/IR/Operator.h"
46 #include "llvm/Support/CRC.h"
47 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
48 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
49 #include <optional>
50 
51 using namespace clang;
52 using namespace CodeGen;
53 
54 /// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
55 /// markers.
56 static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
57                                       const LangOptions &LangOpts) {
58   if (CGOpts.DisableLifetimeMarkers)
59     return false;
60 
61   // Sanitizers may use markers.
62   if (CGOpts.SanitizeAddressUseAfterScope ||
63       LangOpts.Sanitize.has(SanitizerKind::HWAddress) ||
64       LangOpts.Sanitize.has(SanitizerKind::Memory))
65     return true;
66 
67   // For now, only in optimized builds.
68   return CGOpts.OptimizationLevel != 0;
69 }
70 
71 CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
72     : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
73       Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
74               CGBuilderInserterTy(this)),
75       SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
76       DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
77       ShouldEmitLifetimeMarkers(
78           shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {
79   if (!suppressNewContext)
80     CGM.getCXXABI().getMangleContext().startNewFunction();
81   EHStack.setCGF(this);
82 
83   SetFastMathFlags(CurFPFeatures);
84 }
85 
86 CodeGenFunction::~CodeGenFunction() {
87   assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
88 
89   if (getLangOpts().OpenMP && CurFn)
90     CGM.getOpenMPRuntime().functionFinished(*this);
91 
92   // If we have an OpenMPIRBuilder we want to finalize functions (incl.
93   // outlining etc) at some point. Doing it once the function codegen is done
94   // seems to be a reasonable spot. We do it here, as opposed to the deletion
95   // time of the CodeGenModule, because we have to ensure the IR has not yet
96   // been "emitted" to the outside, thus, modifications are still sensible.
97   if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
98     CGM.getOpenMPRuntime().getOMPBuilder().finalize(CurFn);
99 }
100 
101 // Map the LangOption for exception behavior into
102 // the corresponding enum in the IR.
103 llvm::fp::ExceptionBehavior
104 clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
105 
106   switch (Kind) {
107   case LangOptions::FPE_Ignore:  return llvm::fp::ebIgnore;
108   case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
109   case LangOptions::FPE_Strict:  return llvm::fp::ebStrict;
110   default:
111     llvm_unreachable("Unsupported FP Exception Behavior");
112   }
113 }
114 
115 void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
116   llvm::FastMathFlags FMF;
117   FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
118   FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
119   FMF.setNoInfs(FPFeatures.getNoHonorInfs());
120   FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
121   FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
122   FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
123   FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
124   Builder.setFastMathFlags(FMF);
125 }
126 
127 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
128                                                   const Expr *E)
129     : CGF(CGF) {
130   ConstructorHelper(E->getFPFeaturesInEffect(CGF.getLangOpts()));
131 }
132 
133 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
134                                                   FPOptions FPFeatures)
135     : CGF(CGF) {
136   ConstructorHelper(FPFeatures);
137 }
138 
139 void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
140   OldFPFeatures = CGF.CurFPFeatures;
141   CGF.CurFPFeatures = FPFeatures;
142 
143   OldExcept = CGF.Builder.getDefaultConstrainedExcept();
144   OldRounding = CGF.Builder.getDefaultConstrainedRounding();
145 
146   if (OldFPFeatures == FPFeatures)
147     return;
148 
149   FMFGuard.emplace(CGF.Builder);
150 
151   llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();
152   CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
153   auto NewExceptionBehavior =
154       ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>(
155           FPFeatures.getExceptionMode()));
156   CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
157 
158   CGF.SetFastMathFlags(FPFeatures);
159 
160   assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
161           isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
162           isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
163           (NewExceptionBehavior == llvm::fp::ebIgnore &&
164            NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
165          "FPConstrained should be enabled on entire function");
166 
167   auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
168     auto OldValue =
169         CGF.CurFn->getFnAttribute(Name).getValueAsBool();
170     auto NewValue = OldValue & Value;
171     if (OldValue != NewValue)
172       CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));
173   };
174   mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
175   mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
176   mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
177   mergeFnAttrValue(
178       "unsafe-fp-math",
179       FPFeatures.getAllowFPReassociate() && FPFeatures.getAllowReciprocal() &&
180           FPFeatures.getAllowApproxFunc() && FPFeatures.getNoSignedZero() &&
181           FPFeatures.allowFPContractAcrossStatement());
182 }
183 
184 CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
185   CGF.CurFPFeatures = OldFPFeatures;
186   CGF.Builder.setDefaultConstrainedExcept(OldExcept);
187   CGF.Builder.setDefaultConstrainedRounding(OldRounding);
188 }
189 
190 LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
191   LValueBaseInfo BaseInfo;
192   TBAAAccessInfo TBAAInfo;
193   CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
194   Address Addr(V, ConvertTypeForMem(T), Alignment);
195   return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo);
196 }
197 
198 /// Given a value of type T* that may not be to a complete object,
199 /// construct an l-value with the natural pointee alignment of T.
200 LValue
201 CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
202   LValueBaseInfo BaseInfo;
203   TBAAAccessInfo TBAAInfo;
204   CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
205                                                 /* forPointeeType= */ true);
206   Address Addr(V, ConvertTypeForMem(T), Align);
207   return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
208 }
209 
210 
211 llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
212   return CGM.getTypes().ConvertTypeForMem(T);
213 }
214 
215 llvm::Type *CodeGenFunction::ConvertType(QualType T) {
216   return CGM.getTypes().ConvertType(T);
217 }
218 
219 TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
220   type = type.getCanonicalType();
221   while (true) {
222     switch (type->getTypeClass()) {
223 #define TYPE(name, parent)
224 #define ABSTRACT_TYPE(name, parent)
225 #define NON_CANONICAL_TYPE(name, parent) case Type::name:
226 #define DEPENDENT_TYPE(name, parent) case Type::name:
227 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
228 #include "clang/AST/TypeNodes.inc"
229       llvm_unreachable("non-canonical or dependent type in IR-generation");
230 
231     case Type::Auto:
232     case Type::DeducedTemplateSpecialization:
233       llvm_unreachable("undeduced type in IR-generation");
234 
235     // Various scalar types.
236     case Type::Builtin:
237     case Type::Pointer:
238     case Type::BlockPointer:
239     case Type::LValueReference:
240     case Type::RValueReference:
241     case Type::MemberPointer:
242     case Type::Vector:
243     case Type::ExtVector:
244     case Type::ConstantMatrix:
245     case Type::FunctionProto:
246     case Type::FunctionNoProto:
247     case Type::Enum:
248     case Type::ObjCObjectPointer:
249     case Type::Pipe:
250     case Type::BitInt:
251       return TEK_Scalar;
252 
253     // Complexes.
254     case Type::Complex:
255       return TEK_Complex;
256 
257     // Arrays, records, and Objective-C objects.
258     case Type::ConstantArray:
259     case Type::IncompleteArray:
260     case Type::VariableArray:
261     case Type::Record:
262     case Type::ObjCObject:
263     case Type::ObjCInterface:
264       return TEK_Aggregate;
265 
266     // We operate on atomic values according to their underlying type.
267     case Type::Atomic:
268       type = cast<AtomicType>(type)->getValueType();
269       continue;
270     }
271     llvm_unreachable("unknown type kind!");
272   }
273 }
274 
275 llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
276   // For cleanliness, we try to avoid emitting the return block for
277   // simple cases.
278   llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
279 
280   if (CurBB) {
281     assert(!CurBB->getTerminator() && "Unexpected terminated block.");
282 
283     // We have a valid insert point, reuse it if it is empty or there are no
284     // explicit jumps to the return block.
285     if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
286       ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
287       delete ReturnBlock.getBlock();
288       ReturnBlock = JumpDest();
289     } else
290       EmitBlock(ReturnBlock.getBlock());
291     return llvm::DebugLoc();
292   }
293 
294   // Otherwise, if the return block is the target of a single direct
295   // branch then we can just put the code in that block instead. This
296   // cleans up functions which started with a unified return block.
297   if (ReturnBlock.getBlock()->hasOneUse()) {
298     llvm::BranchInst *BI =
299       dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());
300     if (BI && BI->isUnconditional() &&
301         BI->getSuccessor(0) == ReturnBlock.getBlock()) {
302       // Record/return the DebugLoc of the simple 'return' expression to be used
303       // later by the actual 'ret' instruction.
304       llvm::DebugLoc Loc = BI->getDebugLoc();
305       Builder.SetInsertPoint(BI->getParent());
306       BI->eraseFromParent();
307       delete ReturnBlock.getBlock();
308       ReturnBlock = JumpDest();
309       return Loc;
310     }
311   }
312 
313   // FIXME: We are at an unreachable point, there is no reason to emit the block
314   // unless it has uses. However, we still need a place to put the debug
315   // region.end for now.
316 
317   EmitBlock(ReturnBlock.getBlock());
318   return llvm::DebugLoc();
319 }
320 
321 static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
322   if (!BB) return;
323   if (!BB->use_empty()) {
324     CGF.CurFn->insert(CGF.CurFn->end(), BB);
325     return;
326   }
327   delete BB;
328 }
329 
330 void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
331   assert(BreakContinueStack.empty() &&
332          "mismatched push/pop in break/continue stack!");
333 
334   bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
335     && NumSimpleReturnExprs == NumReturnExprs
336     && ReturnBlock.getBlock()->use_empty();
337   // Usually the return expression is evaluated before the cleanup
338   // code.  If the function contains only a simple return statement,
339   // such as a constant, the location before the cleanup code becomes
340   // the last useful breakpoint in the function, because the simple
341   // return expression will be evaluated after the cleanup code. To be
342   // safe, set the debug location for cleanup code to the location of
343   // the return statement.  Otherwise the cleanup code should be at the
344   // end of the function's lexical scope.
345   //
346   // If there are multiple branches to the return block, the branch
347   // instructions will get the location of the return statements and
348   // all will be fine.
349   if (CGDebugInfo *DI = getDebugInfo()) {
350     if (OnlySimpleReturnStmts)
351       DI->EmitLocation(Builder, LastStopPoint);
352     else
353       DI->EmitLocation(Builder, EndLoc);
354   }
355 
356   // Pop any cleanups that might have been associated with the
357   // parameters.  Do this in whatever block we're currently in; it's
358   // important to do this before we enter the return block or return
359   // edges will be *really* confused.
360   bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
361   bool HasOnlyLifetimeMarkers =
362       HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth);
363   bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers;
364 
365   std::optional<ApplyDebugLocation> OAL;
366   if (HasCleanups) {
367     // Make sure the line table doesn't jump back into the body for
368     // the ret after it's been at EndLoc.
369     if (CGDebugInfo *DI = getDebugInfo()) {
370       if (OnlySimpleReturnStmts)
371         DI->EmitLocation(Builder, EndLoc);
372       else
373         // We may not have a valid end location. Try to apply it anyway, and
374         // fall back to an artificial location if needed.
375         OAL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc);
376     }
377 
378     PopCleanupBlocks(PrologueCleanupDepth);
379   }
380 
381   // Emit function epilog (to return).
382   llvm::DebugLoc Loc = EmitReturnBlock();
383 
384   if (ShouldInstrumentFunction()) {
385     if (CGM.getCodeGenOpts().InstrumentFunctions)
386       CurFn->addFnAttr("instrument-function-exit", "__cyg_profile_func_exit");
387     if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
388       CurFn->addFnAttr("instrument-function-exit-inlined",
389                        "__cyg_profile_func_exit");
390   }
391 
392   // Emit debug descriptor for function end.
393   if (CGDebugInfo *DI = getDebugInfo())
394     DI->EmitFunctionEnd(Builder, CurFn);
395 
396   // Reset the debug location to that of the simple 'return' expression, if any
397   // rather than that of the end of the function's scope '}'.
398   ApplyDebugLocation AL(*this, Loc);
399   EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc);
400   EmitEndEHSpec(CurCodeDecl);
401 
402   assert(EHStack.empty() &&
403          "did not remove all scopes from cleanup stack!");
404 
405   // If someone did an indirect goto, emit the indirect goto block at the end of
406   // the function.
407   if (IndirectBranch) {
408     EmitBlock(IndirectBranch->getParent());
409     Builder.ClearInsertionPoint();
410   }
411 
412   // If some of our locals escaped, insert a call to llvm.localescape in the
413   // entry block.
414   if (!EscapedLocals.empty()) {
415     // Invert the map from local to index into a simple vector. There should be
416     // no holes.
417     SmallVector<llvm::Value *, 4> EscapeArgs;
418     EscapeArgs.resize(EscapedLocals.size());
419     for (auto &Pair : EscapedLocals)
420       EscapeArgs[Pair.second] = Pair.first;
421     llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
422         &CGM.getModule(), llvm::Intrinsic::localescape);
423     CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
424   }
425 
426   // Remove the AllocaInsertPt instruction, which is just a convenience for us.
427   llvm::Instruction *Ptr = AllocaInsertPt;
428   AllocaInsertPt = nullptr;
429   Ptr->eraseFromParent();
430 
431   // PostAllocaInsertPt, if created, was lazily created when it was required,
432   // remove it now since it was just created for our own convenience.
433   if (PostAllocaInsertPt) {
434     llvm::Instruction *PostPtr = PostAllocaInsertPt;
435     PostAllocaInsertPt = nullptr;
436     PostPtr->eraseFromParent();
437   }
438 
439   // If someone took the address of a label but never did an indirect goto, we
440   // made a zero entry PHI node, which is illegal, zap it now.
441   if (IndirectBranch) {
442     llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());
443     if (PN->getNumIncomingValues() == 0) {
444       PN->replaceAllUsesWith(llvm::UndefValue::get(PN->getType()));
445       PN->eraseFromParent();
446     }
447   }
448 
449   EmitIfUsed(*this, EHResumeBlock);
450   EmitIfUsed(*this, TerminateLandingPad);
451   EmitIfUsed(*this, TerminateHandler);
452   EmitIfUsed(*this, UnreachableBlock);
453 
454   for (const auto &FuncletAndParent : TerminateFunclets)
455     EmitIfUsed(*this, FuncletAndParent.second);
456 
457   if (CGM.getCodeGenOpts().EmitDeclMetadata)
458     EmitDeclMetadata();
459 
460   for (const auto &R : DeferredReplacements) {
461     if (llvm::Value *Old = R.first) {
462       Old->replaceAllUsesWith(R.second);
463       cast<llvm::Instruction>(Old)->eraseFromParent();
464     }
465   }
466   DeferredReplacements.clear();
467 
468   // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
469   // PHIs if the current function is a coroutine. We don't do it for all
470   // functions as it may result in slight increase in numbers of instructions
471   // if compiled with no optimizations. We do it for coroutine as the lifetime
472   // of CleanupDestSlot alloca make correct coroutine frame building very
473   // difficult.
474   if (NormalCleanupDest.isValid() && isCoroutine()) {
475     llvm::DominatorTree DT(*CurFn);
476     llvm::PromoteMemToReg(
477         cast<llvm::AllocaInst>(NormalCleanupDest.getPointer()), DT);
478     NormalCleanupDest = Address::invalid();
479   }
480 
481   // Scan function arguments for vector width.
482   for (llvm::Argument &A : CurFn->args())
483     if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
484       LargestVectorWidth =
485           std::max((uint64_t)LargestVectorWidth,
486                    VT->getPrimitiveSizeInBits().getKnownMinValue());
487 
488   // Update vector width based on return type.
489   if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))
490     LargestVectorWidth =
491         std::max((uint64_t)LargestVectorWidth,
492                  VT->getPrimitiveSizeInBits().getKnownMinValue());
493 
494   if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)
495     LargestVectorWidth = CurFnInfo->getMaxVectorWidth();
496 
497   // Add the required-vector-width attribute. This contains the max width from:
498   // 1. min-vector-width attribute used in the source program.
499   // 2. Any builtins used that have a vector width specified.
500   // 3. Values passed in and out of inline assembly.
501   // 4. Width of vector arguments and return types for this function.
502   // 5. Width of vector aguments and return types for functions called by this
503   //    function.
504   if (getContext().getTargetInfo().getTriple().isX86())
505     CurFn->addFnAttr("min-legal-vector-width",
506                      llvm::utostr(LargestVectorWidth));
507 
508   // Add vscale_range attribute if appropriate.
509   std::optional<std::pair<unsigned, unsigned>> VScaleRange =
510       getContext().getTargetInfo().getVScaleRange(getLangOpts());
511   if (VScaleRange) {
512     CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
513         getLLVMContext(), VScaleRange->first, VScaleRange->second));
514   }
515 
516   // If we generated an unreachable return block, delete it now.
517   if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
518     Builder.ClearInsertionPoint();
519     ReturnBlock.getBlock()->eraseFromParent();
520   }
521   if (ReturnValue.isValid()) {
522     auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
523     if (RetAlloca && RetAlloca->use_empty()) {
524       RetAlloca->eraseFromParent();
525       ReturnValue = Address::invalid();
526     }
527   }
528 }
529 
530 /// ShouldInstrumentFunction - Return true if the current function should be
531 /// instrumented with __cyg_profile_func_* calls
532 bool CodeGenFunction::ShouldInstrumentFunction() {
533   if (!CGM.getCodeGenOpts().InstrumentFunctions &&
534       !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
535       !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
536     return false;
537   if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
538     return false;
539   return true;
540 }
541 
542 bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {
543   if (!CurFuncDecl)
544     return false;
545   return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();
546 }
547 
548 /// ShouldXRayInstrument - Return true if the current function should be
549 /// instrumented with XRay nop sleds.
550 bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
551   return CGM.getCodeGenOpts().XRayInstrumentFunctions;
552 }
553 
554 /// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
555 /// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
556 bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
557   return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
558          (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
559           CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
560               XRayInstrKind::Custom);
561 }
562 
563 bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
564   return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
565          (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
566           CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
567               XRayInstrKind::Typed);
568 }
569 
570 llvm::Value *
571 CodeGenFunction::DecodeAddrUsedInPrologue(llvm::Value *F,
572                                           llvm::Value *EncodedAddr) {
573   // Reconstruct the address of the global.
574   auto *PCRelAsInt = Builder.CreateSExt(EncodedAddr, IntPtrTy);
575   auto *FuncAsInt = Builder.CreatePtrToInt(F, IntPtrTy, "func_addr.int");
576   auto *GOTAsInt = Builder.CreateAdd(PCRelAsInt, FuncAsInt, "global_addr.int");
577   auto *GOTAddr = Builder.CreateIntToPtr(GOTAsInt, Int8PtrPtrTy, "global_addr");
578 
579   // Load the original pointer through the global.
580   return Builder.CreateLoad(Address(GOTAddr, Int8PtrTy, getPointerAlign()),
581                             "decoded_addr");
582 }
583 
584 void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,
585                                          llvm::Function *Fn) {
586   if (!FD->hasAttr<OpenCLKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())
587     return;
588 
589   llvm::LLVMContext &Context = getLLVMContext();
590 
591   CGM.GenKernelArgMetadata(Fn, FD, this);
592 
593   if (!getLangOpts().OpenCL)
594     return;
595 
596   if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
597     QualType HintQTy = A->getTypeHint();
598     const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
599     bool IsSignedInteger =
600         HintQTy->isSignedIntegerType() ||
601         (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
602     llvm::Metadata *AttrMDArgs[] = {
603         llvm::ConstantAsMetadata::get(llvm::UndefValue::get(
604             CGM.getTypes().ConvertType(A->getTypeHint()))),
605         llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
606             llvm::IntegerType::get(Context, 32),
607             llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
608     Fn->setMetadata("vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));
609   }
610 
611   if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
612     llvm::Metadata *AttrMDArgs[] = {
613         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
614         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
615         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
616     Fn->setMetadata("work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));
617   }
618 
619   if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
620     llvm::Metadata *AttrMDArgs[] = {
621         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
622         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
623         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
624     Fn->setMetadata("reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));
625   }
626 
627   if (const OpenCLIntelReqdSubGroupSizeAttr *A =
628           FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
629     llvm::Metadata *AttrMDArgs[] = {
630         llvm::ConstantAsMetadata::get(Builder.getInt32(A->getSubGroupSize()))};
631     Fn->setMetadata("intel_reqd_sub_group_size",
632                     llvm::MDNode::get(Context, AttrMDArgs));
633   }
634 }
635 
636 /// Determine whether the function F ends with a return stmt.
637 static bool endsWithReturn(const Decl* F) {
638   const Stmt *Body = nullptr;
639   if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))
640     Body = FD->getBody();
641   else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
642     Body = OMD->getBody();
643 
644   if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
645     auto LastStmt = CS->body_rbegin();
646     if (LastStmt != CS->body_rend())
647       return isa<ReturnStmt>(*LastStmt);
648   }
649   return false;
650 }
651 
652 void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
653   if (SanOpts.has(SanitizerKind::Thread)) {
654     Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
655     Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
656   }
657 }
658 
659 /// Check if the return value of this function requires sanitization.
660 bool CodeGenFunction::requiresReturnValueCheck() const {
661   return requiresReturnValueNullabilityCheck() ||
662          (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
663           CurCodeDecl->getAttr<ReturnsNonNullAttr>());
664 }
665 
666 static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
667   auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);
668   if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
669       !MD->getDeclName().getAsIdentifierInfo()->isStr("allocate") ||
670       (MD->getNumParams() != 1 && MD->getNumParams() != 2))
671     return false;
672 
673   if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.getSizeType())
674     return false;
675 
676   if (MD->getNumParams() == 2) {
677     auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
678     if (!PT || !PT->isVoidPointerType() ||
679         !PT->getPointeeType().isConstQualified())
680       return false;
681   }
682 
683   return true;
684 }
685 
686 /// Return the UBSan prologue signature for \p FD if one is available.
687 static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
688                                             const FunctionDecl *FD) {
689   if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
690     if (!MD->isStatic())
691       return nullptr;
692   return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
693 }
694 
695 void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
696                                     llvm::Function *Fn,
697                                     const CGFunctionInfo &FnInfo,
698                                     const FunctionArgList &Args,
699                                     SourceLocation Loc,
700                                     SourceLocation StartLoc) {
701   assert(!CurFn &&
702          "Do not use a CodeGenFunction object for more than one function");
703 
704   const Decl *D = GD.getDecl();
705 
706   DidCallStackSave = false;
707   CurCodeDecl = D;
708   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
709   if (FD && FD->usesSEHTry())
710     CurSEHParent = GD;
711   CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
712   FnRetTy = RetTy;
713   CurFn = Fn;
714   CurFnInfo = &FnInfo;
715   assert(CurFn->isDeclaration() && "Function already has body?");
716 
717   // If this function is ignored for any of the enabled sanitizers,
718   // disable the sanitizer for the function.
719   do {
720 #define SANITIZER(NAME, ID)                                                    \
721   if (SanOpts.empty())                                                         \
722     break;                                                                     \
723   if (SanOpts.has(SanitizerKind::ID))                                          \
724     if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc))                    \
725       SanOpts.set(SanitizerKind::ID, false);
726 
727 #include "clang/Basic/Sanitizers.def"
728 #undef SANITIZER
729   } while (false);
730 
731   if (D) {
732     const bool SanitizeBounds = SanOpts.hasOneOf(SanitizerKind::Bounds);
733     bool NoSanitizeCoverage = false;
734 
735     for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {
736       // Apply the no_sanitize* attributes to SanOpts.
737       SanitizerMask mask = Attr->getMask();
738       SanOpts.Mask &= ~mask;
739       if (mask & SanitizerKind::Address)
740         SanOpts.set(SanitizerKind::KernelAddress, false);
741       if (mask & SanitizerKind::KernelAddress)
742         SanOpts.set(SanitizerKind::Address, false);
743       if (mask & SanitizerKind::HWAddress)
744         SanOpts.set(SanitizerKind::KernelHWAddress, false);
745       if (mask & SanitizerKind::KernelHWAddress)
746         SanOpts.set(SanitizerKind::HWAddress, false);
747 
748       // SanitizeCoverage is not handled by SanOpts.
749       if (Attr->hasCoverage())
750         NoSanitizeCoverage = true;
751     }
752 
753     if (SanitizeBounds && !SanOpts.hasOneOf(SanitizerKind::Bounds))
754       Fn->addFnAttr(llvm::Attribute::NoSanitizeBounds);
755 
756     if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())
757       Fn->addFnAttr(llvm::Attribute::NoSanitizeCoverage);
758   }
759 
760   if (ShouldSkipSanitizerInstrumentation()) {
761     CurFn->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);
762   } else {
763     // Apply sanitizer attributes to the function.
764     if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
765       Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
766     if (SanOpts.hasOneOf(SanitizerKind::HWAddress |
767                          SanitizerKind::KernelHWAddress))
768       Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
769     if (SanOpts.has(SanitizerKind::MemtagStack))
770       Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
771     if (SanOpts.has(SanitizerKind::Thread))
772       Fn->addFnAttr(llvm::Attribute::SanitizeThread);
773     if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
774       Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
775   }
776   if (SanOpts.has(SanitizerKind::SafeStack))
777     Fn->addFnAttr(llvm::Attribute::SafeStack);
778   if (SanOpts.has(SanitizerKind::ShadowCallStack))
779     Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
780 
781   // Apply fuzzing attribute to the function.
782   if (SanOpts.hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
783     Fn->addFnAttr(llvm::Attribute::OptForFuzzing);
784 
785   // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
786   // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
787   if (SanOpts.has(SanitizerKind::Thread)) {
788     if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {
789       IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);
790       if (OMD->getMethodFamily() == OMF_dealloc ||
791           OMD->getMethodFamily() == OMF_initialize ||
792           (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) {
793         markAsIgnoreThreadCheckingAtRuntime(Fn);
794       }
795     }
796   }
797 
798   // Ignore unrelated casts in STL allocate() since the allocator must cast
799   // from void* to T* before object initialization completes. Don't match on the
800   // namespace because not all allocators are in std::
801   if (D && SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
802     if (matchesStlAllocatorFn(D, getContext()))
803       SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
804   }
805 
806   // Ignore null checks in coroutine functions since the coroutines passes
807   // are not aware of how to move the extra UBSan instructions across the split
808   // coroutine boundaries.
809   if (D && SanOpts.has(SanitizerKind::Null))
810     if (FD && FD->getBody() &&
811         FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
812       SanOpts.Mask &= ~SanitizerKind::Null;
813 
814   // Apply xray attributes to the function (as a string, for now)
815   bool AlwaysXRayAttr = false;
816   if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
817     if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
818             XRayInstrKind::FunctionEntry) ||
819         CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
820             XRayInstrKind::FunctionExit)) {
821       if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
822         Fn->addFnAttr("function-instrument", "xray-always");
823         AlwaysXRayAttr = true;
824       }
825       if (XRayAttr->neverXRayInstrument())
826         Fn->addFnAttr("function-instrument", "xray-never");
827       if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
828         if (ShouldXRayInstrumentFunction())
829           Fn->addFnAttr("xray-log-args",
830                         llvm::utostr(LogArgs->getArgumentCount()));
831     }
832   } else {
833     if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
834       Fn->addFnAttr(
835           "xray-instruction-threshold",
836           llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
837   }
838 
839   if (ShouldXRayInstrumentFunction()) {
840     if (CGM.getCodeGenOpts().XRayIgnoreLoops)
841       Fn->addFnAttr("xray-ignore-loops");
842 
843     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
844             XRayInstrKind::FunctionExit))
845       Fn->addFnAttr("xray-skip-exit");
846 
847     if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
848             XRayInstrKind::FunctionEntry))
849       Fn->addFnAttr("xray-skip-entry");
850 
851     auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
852     if (FuncGroups > 1) {
853       auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),
854                                               CurFn->getName().bytes_end());
855       auto Group = crc32(FuncName) % FuncGroups;
856       if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
857           !AlwaysXRayAttr)
858         Fn->addFnAttr("function-instrument", "xray-never");
859     }
860   }
861 
862   if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone) {
863     switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {
864     case ProfileList::Skip:
865       Fn->addFnAttr(llvm::Attribute::SkipProfile);
866       break;
867     case ProfileList::Forbid:
868       Fn->addFnAttr(llvm::Attribute::NoProfile);
869       break;
870     case ProfileList::Allow:
871       break;
872     }
873   }
874 
875   unsigned Count, Offset;
876   if (const auto *Attr =
877           D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
878     Count = Attr->getCount();
879     Offset = Attr->getOffset();
880   } else {
881     Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
882     Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
883   }
884   if (Count && Offset <= Count) {
885     Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
886     if (Offset)
887       Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
888   }
889   // Instruct that functions for COFF/CodeView targets should start with a
890   // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64
891   // backends as they don't need it -- instructions on these architectures are
892   // always atomically patchable at runtime.
893   if (CGM.getCodeGenOpts().HotPatch &&
894       getContext().getTargetInfo().getTriple().isX86() &&
895       getContext().getTargetInfo().getTriple().getEnvironment() !=
896           llvm::Triple::CODE16)
897     Fn->addFnAttr("patchable-function", "prologue-short-redirect");
898 
899   // Add no-jump-tables value.
900   if (CGM.getCodeGenOpts().NoUseJumpTables)
901     Fn->addFnAttr("no-jump-tables", "true");
902 
903   // Add no-inline-line-tables value.
904   if (CGM.getCodeGenOpts().NoInlineLineTables)
905     Fn->addFnAttr("no-inline-line-tables");
906 
907   // Add profile-sample-accurate value.
908   if (CGM.getCodeGenOpts().ProfileSampleAccurate)
909     Fn->addFnAttr("profile-sample-accurate");
910 
911   if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
912     Fn->addFnAttr("use-sample-profile");
913 
914   if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
915     Fn->addFnAttr("cfi-canonical-jump-table");
916 
917   if (D && D->hasAttr<NoProfileFunctionAttr>())
918     Fn->addFnAttr(llvm::Attribute::NoProfile);
919 
920   if (D) {
921     // Function attributes take precedence over command line flags.
922     if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
923       switch (A->getThunkType()) {
924       case FunctionReturnThunksAttr::Kind::Keep:
925         break;
926       case FunctionReturnThunksAttr::Kind::Extern:
927         Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
928         break;
929       }
930     } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
931       Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
932   }
933 
934   if (FD && (getLangOpts().OpenCL ||
935              (getLangOpts().HIP && getLangOpts().CUDAIsDevice))) {
936     // Add metadata for a kernel function.
937     EmitKernelMetadata(FD, Fn);
938   }
939 
940   // If we are checking function types, emit a function type signature as
941   // prologue data.
942   if (FD && getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function)) {
943     if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
944       // Remove any (C++17) exception specifications, to allow calling e.g. a
945       // noexcept function through a non-noexcept pointer.
946       auto ProtoTy = getContext().getFunctionTypeWithExceptionSpec(
947           FD->getType(), EST_None);
948       llvm::Constant *FTRTTIConst =
949           CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true);
950       llvm::GlobalVariable *FTRTTIProxy =
951           CGM.GetOrCreateRTTIProxyGlobalVariable(FTRTTIConst);
952       llvm::LLVMContext &Ctx = Fn->getContext();
953       llvm::MDBuilder MDB(Ctx);
954       Fn->setMetadata(llvm::LLVMContext::MD_func_sanitize,
955                       MDB.createRTTIPointerPrologue(PrologueSig, FTRTTIProxy));
956       CGM.addCompilerUsedGlobal(FTRTTIProxy);
957     }
958   }
959 
960   // If we're checking nullability, we need to know whether we can check the
961   // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
962   if (SanOpts.has(SanitizerKind::NullabilityReturn)) {
963     auto Nullability = FnRetTy->getNullability();
964     if (Nullability && *Nullability == NullabilityKind::NonNull) {
965       if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) &&
966             CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
967         RetValNullabilityPrecondition =
968             llvm::ConstantInt::getTrue(getLLVMContext());
969     }
970   }
971 
972   // If we're in C++ mode and the function name is "main", it is guaranteed
973   // to be norecurse by the standard (3.6.1.3 "The function main shall not be
974   // used within a program").
975   //
976   // OpenCL C 2.0 v2.2-11 s6.9.i:
977   //     Recursion is not supported.
978   //
979   // SYCL v1.2.1 s3.10:
980   //     kernels cannot include RTTI information, exception classes,
981   //     recursive code, virtual functions or make use of C++ libraries that
982   //     are not compiled for the device.
983   if (FD && ((getLangOpts().CPlusPlus && FD->isMain()) ||
984              getLangOpts().OpenCL || getLangOpts().SYCLIsDevice ||
985              (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))
986     Fn->addFnAttr(llvm::Attribute::NoRecurse);
987 
988   llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();
989   llvm::fp::ExceptionBehavior FPExceptionBehavior =
990       ToConstrainedExceptMD(getLangOpts().getDefaultExceptionMode());
991   Builder.setDefaultConstrainedRounding(RM);
992   Builder.setDefaultConstrainedExcept(FPExceptionBehavior);
993   if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||
994       (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||
995                RM != llvm::RoundingMode::NearestTiesToEven))) {
996     Builder.setIsFPConstrained(true);
997     Fn->addFnAttr(llvm::Attribute::StrictFP);
998   }
999 
1000   // If a custom alignment is used, force realigning to this alignment on
1001   // any main function which certainly will need it.
1002   if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
1003              CGM.getCodeGenOpts().StackAlignment))
1004     Fn->addFnAttr("stackrealign");
1005 
1006   // "main" doesn't need to zero out call-used registers.
1007   if (FD && FD->isMain())
1008     Fn->removeFnAttr("zero-call-used-regs");
1009 
1010   llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
1011 
1012   // Create a marker to make it easy to insert allocas into the entryblock
1013   // later.  Don't create this with the builder, because we don't want it
1014   // folded.
1015   llvm::Value *Undef = llvm::UndefValue::get(Int32Ty);
1016   AllocaInsertPt = new llvm::BitCastInst(Undef, Int32Ty, "allocapt", EntryBB);
1017 
1018   ReturnBlock = getJumpDestInCurrentScope("return");
1019 
1020   Builder.SetInsertPoint(EntryBB);
1021 
1022   // If we're checking the return value, allocate space for a pointer to a
1023   // precise source location of the checked return statement.
1024   if (requiresReturnValueCheck()) {
1025     ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr");
1026     Builder.CreateStore(llvm::ConstantPointerNull::get(Int8PtrTy),
1027                         ReturnLocation);
1028   }
1029 
1030   // Emit subprogram debug descriptor.
1031   if (CGDebugInfo *DI = getDebugInfo()) {
1032     // Reconstruct the type from the argument list so that implicit parameters,
1033     // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
1034     // convention.
1035     DI->emitFunctionStart(GD, Loc, StartLoc,
1036                           DI->getFunctionType(FD, RetTy, Args), CurFn,
1037                           CurFuncIsThunk);
1038   }
1039 
1040   if (ShouldInstrumentFunction()) {
1041     if (CGM.getCodeGenOpts().InstrumentFunctions)
1042       CurFn->addFnAttr("instrument-function-entry", "__cyg_profile_func_enter");
1043     if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
1044       CurFn->addFnAttr("instrument-function-entry-inlined",
1045                        "__cyg_profile_func_enter");
1046     if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1047       CurFn->addFnAttr("instrument-function-entry-inlined",
1048                        "__cyg_profile_func_enter_bare");
1049   }
1050 
1051   // Since emitting the mcount call here impacts optimizations such as function
1052   // inlining, we just add an attribute to insert a mcount call in backend.
1053   // The attribute "counting-function" is set to mcount function name which is
1054   // architecture dependent.
1055   if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1056     // Calls to fentry/mcount should not be generated if function has
1057     // the no_instrument_function attribute.
1058     if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1059       if (CGM.getCodeGenOpts().CallFEntry)
1060         Fn->addFnAttr("fentry-call", "true");
1061       else {
1062         Fn->addFnAttr("instrument-function-entry-inlined",
1063                       getTarget().getMCountName());
1064       }
1065       if (CGM.getCodeGenOpts().MNopMCount) {
1066         if (!CGM.getCodeGenOpts().CallFEntry)
1067           CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1068             << "-mnop-mcount" << "-mfentry";
1069         Fn->addFnAttr("mnop-mcount");
1070       }
1071 
1072       if (CGM.getCodeGenOpts().RecordMCount) {
1073         if (!CGM.getCodeGenOpts().CallFEntry)
1074           CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1075             << "-mrecord-mcount" << "-mfentry";
1076         Fn->addFnAttr("mrecord-mcount");
1077       }
1078     }
1079   }
1080 
1081   if (CGM.getCodeGenOpts().PackedStack) {
1082     if (getContext().getTargetInfo().getTriple().getArch() !=
1083         llvm::Triple::systemz)
1084       CGM.getDiags().Report(diag::err_opt_not_valid_on_target)
1085         << "-mpacked-stack";
1086     Fn->addFnAttr("packed-stack");
1087   }
1088 
1089   if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&
1090       !CGM.getDiags().isIgnored(diag::warn_fe_backend_frame_larger_than, Loc))
1091     Fn->addFnAttr("warn-stack-size",
1092                   std::to_string(CGM.getCodeGenOpts().WarnStackSize));
1093 
1094   if (RetTy->isVoidType()) {
1095     // Void type; nothing to return.
1096     ReturnValue = Address::invalid();
1097 
1098     // Count the implicit return.
1099     if (!endsWithReturn(D))
1100       ++NumReturnExprs;
1101   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1102     // Indirect return; emit returned value directly into sret slot.
1103     // This reduces code size, and affects correctness in C++.
1104     auto AI = CurFn->arg_begin();
1105     if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1106       ++AI;
1107     ReturnValue = Address(&*AI, ConvertType(RetTy),
1108                           CurFnInfo->getReturnInfo().getIndirectAlign());
1109     if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1110       ReturnValuePointer =
1111           CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr");
1112       Builder.CreateStore(Builder.CreatePointerBitCastOrAddrSpaceCast(
1113                               ReturnValue.getPointer(), Int8PtrTy),
1114                           ReturnValuePointer);
1115     }
1116   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1117              !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
1118     // Load the sret pointer from the argument struct and return into that.
1119     unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1120     llvm::Function::arg_iterator EI = CurFn->arg_end();
1121     --EI;
1122     llvm::Value *Addr = Builder.CreateStructGEP(
1123         CurFnInfo->getArgStruct(), &*EI, Idx);
1124     llvm::Type *Ty =
1125         cast<llvm::GetElementPtrInst>(Addr)->getResultElementType();
1126     ReturnValuePointer = Address(Addr, Ty, getPointerAlign());
1127     Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result");
1128     ReturnValue =
1129         Address(Addr, ConvertType(RetTy), CGM.getNaturalTypeAlignment(RetTy));
1130   } else {
1131     ReturnValue = CreateIRTemp(RetTy, "retval");
1132 
1133     // Tell the epilog emitter to autorelease the result.  We do this
1134     // now so that various specialized functions can suppress it
1135     // during their IR-generation.
1136     if (getLangOpts().ObjCAutoRefCount &&
1137         !CurFnInfo->isReturnsRetained() &&
1138         RetTy->isObjCRetainableType())
1139       AutoreleaseResult = true;
1140   }
1141 
1142   EmitStartEHSpec(CurCodeDecl);
1143 
1144   PrologueCleanupDepth = EHStack.stable_begin();
1145 
1146   // Emit OpenMP specific initialization of the device functions.
1147   if (getLangOpts().OpenMP && CurCodeDecl)
1148     CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl);
1149 
1150   // Handle emitting HLSL entry functions.
1151   if (D && D->hasAttr<HLSLShaderAttr>())
1152     CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);
1153 
1154   EmitFunctionProlog(*CurFnInfo, CurFn, Args);
1155 
1156   if (isa_and_nonnull<CXXMethodDecl>(D) &&
1157       cast<CXXMethodDecl>(D)->isInstance()) {
1158     CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
1159     const CXXMethodDecl *MD = cast<CXXMethodDecl>(D);
1160     if (MD->getParent()->isLambda() &&
1161         MD->getOverloadedOperator() == OO_Call) {
1162       // We're in a lambda; figure out the captures.
1163       MD->getParent()->getCaptureFields(LambdaCaptureFields,
1164                                         LambdaThisCaptureField);
1165       if (LambdaThisCaptureField) {
1166         // If the lambda captures the object referred to by '*this' - either by
1167         // value or by reference, make sure CXXThisValue points to the correct
1168         // object.
1169 
1170         // Get the lvalue for the field (which is a copy of the enclosing object
1171         // or contains the address of the enclosing object).
1172         LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
1173         if (!LambdaThisCaptureField->getType()->isPointerType()) {
1174           // If the enclosing object was captured by value, just use its address.
1175           CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
1176         } else {
1177           // Load the lvalue pointed to by the field, since '*this' was captured
1178           // by reference.
1179           CXXThisValue =
1180               EmitLoadOfLValue(ThisFieldLValue, SourceLocation()).getScalarVal();
1181         }
1182       }
1183       for (auto *FD : MD->getParent()->fields()) {
1184         if (FD->hasCapturedVLAType()) {
1185           auto *ExprArg = EmitLoadOfLValue(EmitLValueForLambdaField(FD),
1186                                            SourceLocation()).getScalarVal();
1187           auto VAT = FD->getCapturedVLAType();
1188           VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1189         }
1190       }
1191     } else {
1192       // Not in a lambda; just use 'this' from the method.
1193       // FIXME: Should we generate a new load for each use of 'this'?  The
1194       // fast register allocator would be happier...
1195       CXXThisValue = CXXABIThisValue;
1196     }
1197 
1198     // Check the 'this' pointer once per function, if it's available.
1199     if (CXXABIThisValue) {
1200       SanitizerSet SkippedChecks;
1201       SkippedChecks.set(SanitizerKind::ObjectSize, true);
1202       QualType ThisTy = MD->getThisType();
1203 
1204       // If this is the call operator of a lambda with no capture-default, it
1205       // may have a static invoker function, which may call this operator with
1206       // a null 'this' pointer.
1207       if (isLambdaCallOperator(MD) &&
1208           MD->getParent()->getLambdaCaptureDefault() == LCD_None)
1209         SkippedChecks.set(SanitizerKind::Null, true);
1210 
1211       EmitTypeCheck(
1212           isa<CXXConstructorDecl>(MD) ? TCK_ConstructorCall : TCK_MemberCall,
1213           Loc, CXXABIThisValue, ThisTy, CXXABIThisAlignment, SkippedChecks);
1214     }
1215   }
1216 
1217   // If any of the arguments have a variably modified type, make sure to
1218   // emit the type size, but only if the function is not naked. Naked functions
1219   // have no prolog to run this evaluation.
1220   if (!FD || !FD->hasAttr<NakedAttr>()) {
1221     for (const VarDecl *VD : Args) {
1222       // Dig out the type as written from ParmVarDecls; it's unclear whether
1223       // the standard (C99 6.9.1p10) requires this, but we're following the
1224       // precedent set by gcc.
1225       QualType Ty;
1226       if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
1227         Ty = PVD->getOriginalType();
1228       else
1229         Ty = VD->getType();
1230 
1231       if (Ty->isVariablyModifiedType())
1232         EmitVariablyModifiedType(Ty);
1233     }
1234   }
1235   // Emit a location at the end of the prologue.
1236   if (CGDebugInfo *DI = getDebugInfo())
1237     DI->EmitLocation(Builder, StartLoc);
1238   // TODO: Do we need to handle this in two places like we do with
1239   // target-features/target-cpu?
1240   if (CurFuncDecl)
1241     if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1242       LargestVectorWidth = VecWidth->getVectorWidth();
1243 }
1244 
1245 void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1246   incrementProfileCounter(Body);
1247   if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
1248     EmitCompoundStmtWithoutScope(*S);
1249   else
1250     EmitStmt(Body);
1251 
1252   // This is checked after emitting the function body so we know if there
1253   // are any permitted infinite loops.
1254   if (checkIfFunctionMustProgress())
1255     CurFn->addFnAttr(llvm::Attribute::MustProgress);
1256 }
1257 
1258 /// When instrumenting to collect profile data, the counts for some blocks
1259 /// such as switch cases need to not include the fall-through counts, so
1260 /// emit a branch around the instrumentation code. When not instrumenting,
1261 /// this just calls EmitBlock().
1262 void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1263                                                const Stmt *S) {
1264   llvm::BasicBlock *SkipCountBB = nullptr;
1265   if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1266     // When instrumenting for profiling, the fallthrough to certain
1267     // statements needs to skip over the instrumentation code so that we
1268     // get an accurate count.
1269     SkipCountBB = createBasicBlock("skipcount");
1270     EmitBranch(SkipCountBB);
1271   }
1272   EmitBlock(BB);
1273   uint64_t CurrentCount = getCurrentProfileCount();
1274   incrementProfileCounter(S);
1275   setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1276   if (SkipCountBB)
1277     EmitBlock(SkipCountBB);
1278 }
1279 
1280 /// Tries to mark the given function nounwind based on the
1281 /// non-existence of any throwing calls within it.  We believe this is
1282 /// lightweight enough to do at -O0.
1283 static void TryMarkNoThrow(llvm::Function *F) {
1284   // LLVM treats 'nounwind' on a function as part of the type, so we
1285   // can't do this on functions that can be overwritten.
1286   if (F->isInterposable()) return;
1287 
1288   for (llvm::BasicBlock &BB : *F)
1289     for (llvm::Instruction &I : BB)
1290       if (I.mayThrow())
1291         return;
1292 
1293   F->setDoesNotThrow();
1294 }
1295 
1296 QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1297                                                FunctionArgList &Args) {
1298   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1299   QualType ResTy = FD->getReturnType();
1300 
1301   const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
1302   if (MD && MD->isInstance()) {
1303     if (CGM.getCXXABI().HasThisReturn(GD))
1304       ResTy = MD->getThisType();
1305     else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1306       ResTy = CGM.getContext().VoidPtrTy;
1307     CGM.getCXXABI().buildThisParam(*this, Args);
1308   }
1309 
1310   // The base version of an inheriting constructor whose constructed base is a
1311   // virtual base is not passed any arguments (because it doesn't actually call
1312   // the inherited constructor).
1313   bool PassedParams = true;
1314   if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
1315     if (auto Inherited = CD->getInheritedConstructor())
1316       PassedParams =
1317           getTypes().inheritingCtorHasParams(Inherited, GD.getCtorType());
1318 
1319   if (PassedParams) {
1320     for (auto *Param : FD->parameters()) {
1321       Args.push_back(Param);
1322       if (!Param->hasAttr<PassObjectSizeAttr>())
1323         continue;
1324 
1325       auto *Implicit = ImplicitParamDecl::Create(
1326           getContext(), Param->getDeclContext(), Param->getLocation(),
1327           /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamDecl::Other);
1328       SizeArguments[Param] = Implicit;
1329       Args.push_back(Implicit);
1330     }
1331   }
1332 
1333   if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
1334     CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
1335 
1336   return ResTy;
1337 }
1338 
1339 void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1340                                    const CGFunctionInfo &FnInfo) {
1341   assert(Fn && "generating code for null Function");
1342   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1343   CurGD = GD;
1344 
1345   FunctionArgList Args;
1346   QualType ResTy = BuildFunctionArgList(GD, Args);
1347 
1348   if (FD->isInlineBuiltinDeclaration()) {
1349     // When generating code for a builtin with an inline declaration, use a
1350     // mangled name to hold the actual body, while keeping an external
1351     // definition in case the function pointer is referenced somewhere.
1352     std::string FDInlineName = (Fn->getName() + ".inline").str();
1353     llvm::Module *M = Fn->getParent();
1354     llvm::Function *Clone = M->getFunction(FDInlineName);
1355     if (!Clone) {
1356       Clone = llvm::Function::Create(Fn->getFunctionType(),
1357                                      llvm::GlobalValue::InternalLinkage,
1358                                      Fn->getAddressSpace(), FDInlineName, M);
1359       Clone->addFnAttr(llvm::Attribute::AlwaysInline);
1360     }
1361     Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);
1362     Fn = Clone;
1363   } else {
1364     // Detect the unusual situation where an inline version is shadowed by a
1365     // non-inline version. In that case we should pick the external one
1366     // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way
1367     // to detect that situation before we reach codegen, so do some late
1368     // replacement.
1369     for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;
1370          PD = PD->getPreviousDecl()) {
1371       if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {
1372         std::string FDInlineName = (Fn->getName() + ".inline").str();
1373         llvm::Module *M = Fn->getParent();
1374         if (llvm::Function *Clone = M->getFunction(FDInlineName)) {
1375           Clone->replaceAllUsesWith(Fn);
1376           Clone->eraseFromParent();
1377         }
1378         break;
1379       }
1380     }
1381   }
1382 
1383   // Check if we should generate debug info for this function.
1384   if (FD->hasAttr<NoDebugAttr>()) {
1385     // Clear non-distinct debug info that was possibly attached to the function
1386     // due to an earlier declaration without the nodebug attribute
1387     Fn->setSubprogram(nullptr);
1388     // Disable debug info indefinitely for this function
1389     DebugInfo = nullptr;
1390   }
1391 
1392   // The function might not have a body if we're generating thunks for a
1393   // function declaration.
1394   SourceRange BodyRange;
1395   if (Stmt *Body = FD->getBody())
1396     BodyRange = Body->getSourceRange();
1397   else
1398     BodyRange = FD->getLocation();
1399   CurEHLocation = BodyRange.getEnd();
1400 
1401   // Use the location of the start of the function to determine where
1402   // the function definition is located. By default use the location
1403   // of the declaration as the location for the subprogram. A function
1404   // may lack a declaration in the source code if it is created by code
1405   // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1406   SourceLocation Loc = FD->getLocation();
1407 
1408   // If this is a function specialization then use the pattern body
1409   // as the location for the function.
1410   if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1411     if (SpecDecl->hasBody(SpecDecl))
1412       Loc = SpecDecl->getLocation();
1413 
1414   Stmt *Body = FD->getBody();
1415 
1416   if (Body) {
1417     // Coroutines always emit lifetime markers.
1418     if (isa<CoroutineBodyStmt>(Body))
1419       ShouldEmitLifetimeMarkers = true;
1420 
1421     // Initialize helper which will detect jumps which can cause invalid
1422     // lifetime markers.
1423     if (ShouldEmitLifetimeMarkers)
1424       Bypasses.Init(Body);
1425   }
1426 
1427   // Emit the standard function prologue.
1428   StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
1429 
1430   // Save parameters for coroutine function.
1431   if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body))
1432     llvm::append_range(FnArgs, FD->parameters());
1433 
1434   // Generate the body of the function.
1435   PGO.assignRegionCounters(GD, CurFn);
1436   if (isa<CXXDestructorDecl>(FD))
1437     EmitDestructorBody(Args);
1438   else if (isa<CXXConstructorDecl>(FD))
1439     EmitConstructorBody(Args);
1440   else if (getLangOpts().CUDA &&
1441            !getLangOpts().CUDAIsDevice &&
1442            FD->hasAttr<CUDAGlobalAttr>())
1443     CGM.getCUDARuntime().emitDeviceStub(*this, Args);
1444   else if (isa<CXXMethodDecl>(FD) &&
1445            cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()) {
1446     // The lambda static invoker function is special, because it forwards or
1447     // clones the body of the function call operator (but is actually static).
1448     EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD));
1449   } else if (FD->isDefaulted() && isa<CXXMethodDecl>(FD) &&
1450              (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() ||
1451               cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator())) {
1452     // Implicit copy-assignment gets the same special treatment as implicit
1453     // copy-constructors.
1454     emitImplicitAssignmentOperatorBody(Args);
1455   } else if (Body) {
1456     EmitFunctionBody(Body);
1457   } else
1458     llvm_unreachable("no definition for emitted function");
1459 
1460   // C++11 [stmt.return]p2:
1461   //   Flowing off the end of a function [...] results in undefined behavior in
1462   //   a value-returning function.
1463   // C11 6.9.1p12:
1464   //   If the '}' that terminates a function is reached, and the value of the
1465   //   function call is used by the caller, the behavior is undefined.
1466   if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1467       !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1468     bool ShouldEmitUnreachable =
1469         CGM.getCodeGenOpts().StrictReturn ||
1470         !CGM.MayDropFunctionReturn(FD->getASTContext(), FD->getReturnType());
1471     if (SanOpts.has(SanitizerKind::Return)) {
1472       SanitizerScope SanScope(this);
1473       llvm::Value *IsFalse = Builder.getFalse();
1474       EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
1475                 SanitizerHandler::MissingReturn,
1476                 EmitCheckSourceLocation(FD->getLocation()), std::nullopt);
1477     } else if (ShouldEmitUnreachable) {
1478       if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1479         EmitTrapCall(llvm::Intrinsic::trap);
1480     }
1481     if (SanOpts.has(SanitizerKind::Return) || ShouldEmitUnreachable) {
1482       Builder.CreateUnreachable();
1483       Builder.ClearInsertionPoint();
1484     }
1485   }
1486 
1487   // Emit the standard function epilogue.
1488   FinishFunction(BodyRange.getEnd());
1489 
1490   // If we haven't marked the function nothrow through other means, do
1491   // a quick pass now to see if we can.
1492   if (!CurFn->doesNotThrow())
1493     TryMarkNoThrow(CurFn);
1494 }
1495 
1496 /// ContainsLabel - Return true if the statement contains a label in it.  If
1497 /// this statement is not executed normally, it not containing a label means
1498 /// that we can just remove the code.
1499 bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1500   // Null statement, not a label!
1501   if (!S) return false;
1502 
1503   // If this is a label, we have to emit the code, consider something like:
1504   // if (0) {  ...  foo:  bar(); }  goto foo;
1505   //
1506   // TODO: If anyone cared, we could track __label__'s, since we know that you
1507   // can't jump to one from outside their declared region.
1508   if (isa<LabelStmt>(S))
1509     return true;
1510 
1511   // If this is a case/default statement, and we haven't seen a switch, we have
1512   // to emit the code.
1513   if (isa<SwitchCase>(S) && !IgnoreCaseStmts)
1514     return true;
1515 
1516   // If this is a switch statement, we want to ignore cases below it.
1517   if (isa<SwitchStmt>(S))
1518     IgnoreCaseStmts = true;
1519 
1520   // Scan subexpressions for verboten labels.
1521   for (const Stmt *SubStmt : S->children())
1522     if (ContainsLabel(SubStmt, IgnoreCaseStmts))
1523       return true;
1524 
1525   return false;
1526 }
1527 
1528 /// containsBreak - Return true if the statement contains a break out of it.
1529 /// If the statement (recursively) contains a switch or loop with a break
1530 /// inside of it, this is fine.
1531 bool CodeGenFunction::containsBreak(const Stmt *S) {
1532   // Null statement, not a label!
1533   if (!S) return false;
1534 
1535   // If this is a switch or loop that defines its own break scope, then we can
1536   // include it and anything inside of it.
1537   if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||
1538       isa<ForStmt>(S))
1539     return false;
1540 
1541   if (isa<BreakStmt>(S))
1542     return true;
1543 
1544   // Scan subexpressions for verboten breaks.
1545   for (const Stmt *SubStmt : S->children())
1546     if (containsBreak(SubStmt))
1547       return true;
1548 
1549   return false;
1550 }
1551 
1552 bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1553   if (!S) return false;
1554 
1555   // Some statement kinds add a scope and thus never add a decl to the current
1556   // scope. Note, this list is longer than the list of statements that might
1557   // have an unscoped decl nested within them, but this way is conservatively
1558   // correct even if more statement kinds are added.
1559   if (isa<IfStmt>(S) || isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
1560       isa<DoStmt>(S) || isa<ForStmt>(S) || isa<CompoundStmt>(S) ||
1561       isa<CXXForRangeStmt>(S) || isa<CXXTryStmt>(S) ||
1562       isa<ObjCForCollectionStmt>(S) || isa<ObjCAtTryStmt>(S))
1563     return false;
1564 
1565   if (isa<DeclStmt>(S))
1566     return true;
1567 
1568   for (const Stmt *SubStmt : S->children())
1569     if (mightAddDeclToScope(SubStmt))
1570       return true;
1571 
1572   return false;
1573 }
1574 
1575 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1576 /// to a constant, or if it does but contains a label, return false.  If it
1577 /// constant folds return true and set the boolean result in Result.
1578 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1579                                                    bool &ResultBool,
1580                                                    bool AllowLabels) {
1581   llvm::APSInt ResultInt;
1582   if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))
1583     return false;
1584 
1585   ResultBool = ResultInt.getBoolValue();
1586   return true;
1587 }
1588 
1589 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1590 /// to a constant, or if it does but contains a label, return false.  If it
1591 /// constant folds return true and set the folded value.
1592 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1593                                                    llvm::APSInt &ResultInt,
1594                                                    bool AllowLabels) {
1595   // FIXME: Rename and handle conversion of other evaluatable things
1596   // to bool.
1597   Expr::EvalResult Result;
1598   if (!Cond->EvaluateAsInt(Result, getContext()))
1599     return false;  // Not foldable, not integer or not fully evaluatable.
1600 
1601   llvm::APSInt Int = Result.Val.getInt();
1602   if (!AllowLabels && CodeGenFunction::ContainsLabel(Cond))
1603     return false;  // Contains a label.
1604 
1605   ResultInt = Int;
1606   return true;
1607 }
1608 
1609 /// Determine whether the given condition is an instrumentable condition
1610 /// (i.e. no "&&" or "||").
1611 bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1612   // Bypass simplistic logical-NOT operator before determining whether the
1613   // condition contains any other logical operator.
1614   if (const UnaryOperator *UnOp = dyn_cast<UnaryOperator>(C->IgnoreParens()))
1615     if (UnOp->getOpcode() == UO_LNot)
1616       C = UnOp->getSubExpr();
1617 
1618   const BinaryOperator *BOp = dyn_cast<BinaryOperator>(C->IgnoreParens());
1619   return (!BOp || !BOp->isLogicalOp());
1620 }
1621 
1622 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1623 /// increments a profile counter based on the semantics of the given logical
1624 /// operator opcode.  This is used to instrument branch condition coverage for
1625 /// logical operators.
1626 void CodeGenFunction::EmitBranchToCounterBlock(
1627     const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1628     llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1629     Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1630   // If not instrumenting, just emit a branch.
1631   bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1632   if (!InstrumentRegions || !isInstrumentedCondition(Cond))
1633     return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1634 
1635   llvm::BasicBlock *ThenBlock = nullptr;
1636   llvm::BasicBlock *ElseBlock = nullptr;
1637   llvm::BasicBlock *NextBlock = nullptr;
1638 
1639   // Create the block we'll use to increment the appropriate counter.
1640   llvm::BasicBlock *CounterIncrBlock = createBasicBlock("lop.rhscnt");
1641 
1642   // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1643   // means we need to evaluate the condition and increment the counter on TRUE:
1644   //
1645   // if (Cond)
1646   //   goto CounterIncrBlock;
1647   // else
1648   //   goto FalseBlock;
1649   //
1650   // CounterIncrBlock:
1651   //   Counter++;
1652   //   goto TrueBlock;
1653 
1654   if (LOp == BO_LAnd) {
1655     ThenBlock = CounterIncrBlock;
1656     ElseBlock = FalseBlock;
1657     NextBlock = TrueBlock;
1658   }
1659 
1660   // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1661   // we need to evaluate the condition and increment the counter on FALSE:
1662   //
1663   // if (Cond)
1664   //   goto TrueBlock;
1665   // else
1666   //   goto CounterIncrBlock;
1667   //
1668   // CounterIncrBlock:
1669   //   Counter++;
1670   //   goto FalseBlock;
1671 
1672   else if (LOp == BO_LOr) {
1673     ThenBlock = TrueBlock;
1674     ElseBlock = CounterIncrBlock;
1675     NextBlock = FalseBlock;
1676   } else {
1677     llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1678   }
1679 
1680   // Emit Branch based on condition.
1681   EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, TrueCount, LH);
1682 
1683   // Emit the block containing the counter increment(s).
1684   EmitBlock(CounterIncrBlock);
1685 
1686   // Increment corresponding counter; if index not provided, use Cond as index.
1687   incrementProfileCounter(CntrIdx ? CntrIdx : Cond);
1688 
1689   // Go to the next block.
1690   EmitBranch(NextBlock);
1691 }
1692 
1693 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1694 /// statement) to the specified blocks.  Based on the condition, this might try
1695 /// to simplify the codegen of the conditional based on the branch.
1696 /// \param LH The value of the likelihood attribute on the True branch.
1697 void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
1698                                            llvm::BasicBlock *TrueBlock,
1699                                            llvm::BasicBlock *FalseBlock,
1700                                            uint64_t TrueCount,
1701                                            Stmt::Likelihood LH) {
1702   Cond = Cond->IgnoreParens();
1703 
1704   if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
1705 
1706     // Handle X && Y in a condition.
1707     if (CondBOp->getOpcode() == BO_LAnd) {
1708       // If we have "1 && X", simplify the code.  "0 && X" would have constant
1709       // folded if the case was simple enough.
1710       bool ConstantBool = false;
1711       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1712           ConstantBool) {
1713         // br(1 && X) -> br(X).
1714         incrementProfileCounter(CondBOp);
1715         return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1716                                         FalseBlock, TrueCount, LH);
1717       }
1718 
1719       // If we have "X && 1", simplify the code to use an uncond branch.
1720       // "X && 0" would have been constant folded to 0.
1721       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1722           ConstantBool) {
1723         // br(X && 1) -> br(X).
1724         return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock,
1725                                         FalseBlock, TrueCount, LH, CondBOp);
1726       }
1727 
1728       // Emit the LHS as a conditional.  If the LHS conditional is false, we
1729       // want to jump to the FalseBlock.
1730       llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
1731       // The counter tells us how often we evaluate RHS, and all of TrueCount
1732       // can be propagated to that branch.
1733       uint64_t RHSCount = getProfileCount(CondBOp->getRHS());
1734 
1735       ConditionalEvaluation eval(*this);
1736       {
1737         ApplyDebugLocation DL(*this, Cond);
1738         // Propagate the likelihood attribute like __builtin_expect
1739         // __builtin_expect(X && Y, 1) -> X and Y are likely
1740         // __builtin_expect(X && Y, 0) -> only Y is unlikely
1741         EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount,
1742                              LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1743         EmitBlock(LHSTrue);
1744       }
1745 
1746       incrementProfileCounter(CondBOp);
1747       setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1748 
1749       // Any temporaries created here are conditional.
1750       eval.begin(*this);
1751       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1752                                FalseBlock, TrueCount, LH);
1753       eval.end(*this);
1754 
1755       return;
1756     }
1757 
1758     if (CondBOp->getOpcode() == BO_LOr) {
1759       // If we have "0 || X", simplify the code.  "1 || X" would have constant
1760       // folded if the case was simple enough.
1761       bool ConstantBool = false;
1762       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1763           !ConstantBool) {
1764         // br(0 || X) -> br(X).
1765         incrementProfileCounter(CondBOp);
1766         return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock,
1767                                         FalseBlock, TrueCount, LH);
1768       }
1769 
1770       // If we have "X || 0", simplify the code to use an uncond branch.
1771       // "X || 1" would have been constant folded to 1.
1772       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1773           !ConstantBool) {
1774         // br(X || 0) -> br(X).
1775         return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock,
1776                                         FalseBlock, TrueCount, LH, CondBOp);
1777       }
1778 
1779       // Emit the LHS as a conditional.  If the LHS conditional is true, we
1780       // want to jump to the TrueBlock.
1781       llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
1782       // We have the count for entry to the RHS and for the whole expression
1783       // being true, so we can divy up True count between the short circuit and
1784       // the RHS.
1785       uint64_t LHSCount =
1786           getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());
1787       uint64_t RHSCount = TrueCount - LHSCount;
1788 
1789       ConditionalEvaluation eval(*this);
1790       {
1791         // Propagate the likelihood attribute like __builtin_expect
1792         // __builtin_expect(X || Y, 1) -> only Y is likely
1793         // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
1794         ApplyDebugLocation DL(*this, Cond);
1795         EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount,
1796                              LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
1797         EmitBlock(LHSFalse);
1798       }
1799 
1800       incrementProfileCounter(CondBOp);
1801       setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1802 
1803       // Any temporaries created here are conditional.
1804       eval.begin(*this);
1805       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, FalseBlock,
1806                                RHSCount, LH);
1807 
1808       eval.end(*this);
1809 
1810       return;
1811     }
1812   }
1813 
1814   if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
1815     // br(!x, t, f) -> br(x, f, t)
1816     if (CondUOp->getOpcode() == UO_LNot) {
1817       // Negate the count.
1818       uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
1819       // The values of the enum are chosen to make this negation possible.
1820       LH = static_cast<Stmt::Likelihood>(-LH);
1821       // Negate the condition and swap the destination blocks.
1822       return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
1823                                   FalseCount, LH);
1824     }
1825   }
1826 
1827   if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {
1828     // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
1829     llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
1830     llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
1831 
1832     // The ConditionalOperator itself has no likelihood information for its
1833     // true and false branches. This matches the behavior of __builtin_expect.
1834     ConditionalEvaluation cond(*this);
1835     EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,
1836                          getProfileCount(CondOp), Stmt::LH_None);
1837 
1838     // When computing PGO branch weights, we only know the overall count for
1839     // the true block. This code is essentially doing tail duplication of the
1840     // naive code-gen, introducing new edges for which counts are not
1841     // available. Divide the counts proportionally between the LHS and RHS of
1842     // the conditional operator.
1843     uint64_t LHSScaledTrueCount = 0;
1844     if (TrueCount) {
1845       double LHSRatio =
1846           getProfileCount(CondOp) / (double)getCurrentProfileCount();
1847       LHSScaledTrueCount = TrueCount * LHSRatio;
1848     }
1849 
1850     cond.begin(*this);
1851     EmitBlock(LHSBlock);
1852     incrementProfileCounter(CondOp);
1853     {
1854       ApplyDebugLocation DL(*this, Cond);
1855       EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
1856                            LHSScaledTrueCount, LH);
1857     }
1858     cond.end(*this);
1859 
1860     cond.begin(*this);
1861     EmitBlock(RHSBlock);
1862     EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
1863                          TrueCount - LHSScaledTrueCount, LH);
1864     cond.end(*this);
1865 
1866     return;
1867   }
1868 
1869   if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {
1870     // Conditional operator handling can give us a throw expression as a
1871     // condition for a case like:
1872     //   br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
1873     // Fold this to:
1874     //   br(c, throw x, br(y, t, f))
1875     EmitCXXThrowExpr(Throw, /*KeepInsertionPoint*/false);
1876     return;
1877   }
1878 
1879   // Emit the code with the fully general case.
1880   llvm::Value *CondV;
1881   {
1882     ApplyDebugLocation DL(*this, Cond);
1883     CondV = EvaluateExprAsBool(Cond);
1884   }
1885 
1886   llvm::MDNode *Weights = nullptr;
1887   llvm::MDNode *Unpredictable = nullptr;
1888 
1889   // If the branch has a condition wrapped by __builtin_unpredictable,
1890   // create metadata that specifies that the branch is unpredictable.
1891   // Don't bother if not optimizing because that metadata would not be used.
1892   auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
1893   if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
1894     auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1895     if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1896       llvm::MDBuilder MDHelper(getLLVMContext());
1897       Unpredictable = MDHelper.createUnpredictable();
1898     }
1899   }
1900 
1901   // If there is a Likelihood knowledge for the cond, lower it.
1902   // Note that if not optimizing this won't emit anything.
1903   llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
1904   if (CondV != NewCondV)
1905     CondV = NewCondV;
1906   else {
1907     // Otherwise, lower profile counts. Note that we do this even at -O0.
1908     uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
1909     Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
1910   }
1911 
1912   Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
1913 }
1914 
1915 /// ErrorUnsupported - Print out an error that codegen doesn't support the
1916 /// specified stmt yet.
1917 void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
1918   CGM.ErrorUnsupported(S, Type);
1919 }
1920 
1921 /// emitNonZeroVLAInit - Emit the "zero" initialization of a
1922 /// variable-length array whose elements have a non-zero bit-pattern.
1923 ///
1924 /// \param baseType the inner-most element type of the array
1925 /// \param src - a char* pointing to the bit-pattern for a single
1926 /// base element of the array
1927 /// \param sizeInChars - the total size of the VLA, in chars
1928 static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
1929                                Address dest, Address src,
1930                                llvm::Value *sizeInChars) {
1931   CGBuilderTy &Builder = CGF.Builder;
1932 
1933   CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);
1934   llvm::Value *baseSizeInChars
1935     = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
1936 
1937   Address begin =
1938     Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin");
1939   llvm::Value *end = Builder.CreateInBoundsGEP(
1940       begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end");
1941 
1942   llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
1943   llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
1944   llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");
1945 
1946   // Make a loop over the VLA.  C99 guarantees that the VLA element
1947   // count must be nonzero.
1948   CGF.EmitBlock(loopBB);
1949 
1950   llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
1951   cur->addIncoming(begin.getPointer(), originBB);
1952 
1953   CharUnits curAlign =
1954     dest.getAlignment().alignmentOfArrayElement(baseSize);
1955 
1956   // memcpy the individual element bit-pattern.
1957   Builder.CreateMemCpy(Address(cur, CGF.Int8Ty, curAlign), src, baseSizeInChars,
1958                        /*volatile*/ false);
1959 
1960   // Go to the next element.
1961   llvm::Value *next =
1962     Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");
1963 
1964   // Leave if that's the end of the VLA.
1965   llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
1966   Builder.CreateCondBr(done, contBB, loopBB);
1967   cur->addIncoming(next, loopBB);
1968 
1969   CGF.EmitBlock(contBB);
1970 }
1971 
1972 void
1973 CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
1974   // Ignore empty classes in C++.
1975   if (getLangOpts().CPlusPlus) {
1976     if (const RecordType *RT = Ty->getAs<RecordType>()) {
1977       if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
1978         return;
1979     }
1980   }
1981 
1982   // Cast the dest ptr to the appropriate i8 pointer type.
1983   if (DestPtr.getElementType() != Int8Ty)
1984     DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
1985 
1986   // Get size and alignment info for this aggregate.
1987   CharUnits size = getContext().getTypeSizeInChars(Ty);
1988 
1989   llvm::Value *SizeVal;
1990   const VariableArrayType *vla;
1991 
1992   // Don't bother emitting a zero-byte memset.
1993   if (size.isZero()) {
1994     // But note that getTypeInfo returns 0 for a VLA.
1995     if (const VariableArrayType *vlaType =
1996           dyn_cast_or_null<VariableArrayType>(
1997                                           getContext().getAsArrayType(Ty))) {
1998       auto VlaSize = getVLASize(vlaType);
1999       SizeVal = VlaSize.NumElts;
2000       CharUnits eltSize = getContext().getTypeSizeInChars(VlaSize.Type);
2001       if (!eltSize.isOne())
2002         SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(eltSize));
2003       vla = vlaType;
2004     } else {
2005       return;
2006     }
2007   } else {
2008     SizeVal = CGM.getSize(size);
2009     vla = nullptr;
2010   }
2011 
2012   // If the type contains a pointer to data member we can't memset it to zero.
2013   // Instead, create a null constant and copy it to the destination.
2014   // TODO: there are other patterns besides zero that we can usefully memset,
2015   // like -1, which happens to be the pattern used by member-pointers.
2016   if (!CGM.getTypes().isZeroInitializable(Ty)) {
2017     // For a VLA, emit a single element, then splat that over the VLA.
2018     if (vla) Ty = getContext().getBaseElementType(vla);
2019 
2020     llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
2021 
2022     llvm::GlobalVariable *NullVariable =
2023       new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
2024                                /*isConstant=*/true,
2025                                llvm::GlobalVariable::PrivateLinkage,
2026                                NullConstant, Twine());
2027     CharUnits NullAlign = DestPtr.getAlignment();
2028     NullVariable->setAlignment(NullAlign.getAsAlign());
2029     Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()),
2030                    Builder.getInt8Ty(), NullAlign);
2031 
2032     if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
2033 
2034     // Get and call the appropriate llvm.memcpy overload.
2035     Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
2036     return;
2037   }
2038 
2039   // Otherwise, just memset the whole thing to zero.  This is legal
2040   // because in LLVM, all default initializers (other than the ones we just
2041   // handled above) are guaranteed to have a bit pattern of all zeros.
2042   Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
2043 }
2044 
2045 llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
2046   // Make sure that there is a block for the indirect goto.
2047   if (!IndirectBranch)
2048     GetIndirectGotoBlock();
2049 
2050   llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
2051 
2052   // Make sure the indirect branch includes all of the address-taken blocks.
2053   IndirectBranch->addDestination(BB);
2054   return llvm::BlockAddress::get(CurFn, BB);
2055 }
2056 
2057 llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
2058   // If we already made the indirect branch for indirect goto, return its block.
2059   if (IndirectBranch) return IndirectBranch->getParent();
2060 
2061   CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));
2062 
2063   // Create the PHI node that indirect gotos will add entries to.
2064   llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,
2065                                               "indirect.goto.dest");
2066 
2067   // Create the indirect branch instruction.
2068   IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
2069   return IndirectBranch->getParent();
2070 }
2071 
2072 /// Computes the length of an array in elements, as well as the base
2073 /// element type and a properly-typed first element pointer.
2074 llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
2075                                               QualType &baseType,
2076                                               Address &addr) {
2077   const ArrayType *arrayType = origArrayType;
2078 
2079   // If it's a VLA, we have to load the stored size.  Note that
2080   // this is the size of the VLA in bytes, not its size in elements.
2081   llvm::Value *numVLAElements = nullptr;
2082   if (isa<VariableArrayType>(arrayType)) {
2083     numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).NumElts;
2084 
2085     // Walk into all VLAs.  This doesn't require changes to addr,
2086     // which has type T* where T is the first non-VLA element type.
2087     do {
2088       QualType elementType = arrayType->getElementType();
2089       arrayType = getContext().getAsArrayType(elementType);
2090 
2091       // If we only have VLA components, 'addr' requires no adjustment.
2092       if (!arrayType) {
2093         baseType = elementType;
2094         return numVLAElements;
2095       }
2096     } while (isa<VariableArrayType>(arrayType));
2097 
2098     // We get out here only if we find a constant array type
2099     // inside the VLA.
2100   }
2101 
2102   // We have some number of constant-length arrays, so addr should
2103   // have LLVM type [M x [N x [...]]]*.  Build a GEP that walks
2104   // down to the first element of addr.
2105   SmallVector<llvm::Value*, 8> gepIndices;
2106 
2107   // GEP down to the array type.
2108   llvm::ConstantInt *zero = Builder.getInt32(0);
2109   gepIndices.push_back(zero);
2110 
2111   uint64_t countFromCLAs = 1;
2112   QualType eltType;
2113 
2114   llvm::ArrayType *llvmArrayType =
2115     dyn_cast<llvm::ArrayType>(addr.getElementType());
2116   while (llvmArrayType) {
2117     assert(isa<ConstantArrayType>(arrayType));
2118     assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
2119              == llvmArrayType->getNumElements());
2120 
2121     gepIndices.push_back(zero);
2122     countFromCLAs *= llvmArrayType->getNumElements();
2123     eltType = arrayType->getElementType();
2124 
2125     llvmArrayType =
2126       dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());
2127     arrayType = getContext().getAsArrayType(arrayType->getElementType());
2128     assert((!llvmArrayType || arrayType) &&
2129            "LLVM and Clang types are out-of-synch");
2130   }
2131 
2132   if (arrayType) {
2133     // From this point onwards, the Clang array type has been emitted
2134     // as some other type (probably a packed struct). Compute the array
2135     // size, and just emit the 'begin' expression as a bitcast.
2136     while (arrayType) {
2137       countFromCLAs *=
2138           cast<ConstantArrayType>(arrayType)->getSize().getZExtValue();
2139       eltType = arrayType->getElementType();
2140       arrayType = getContext().getAsArrayType(eltType);
2141     }
2142 
2143     llvm::Type *baseType = ConvertType(eltType);
2144     addr = Builder.CreateElementBitCast(addr, baseType, "array.begin");
2145   } else {
2146     // Create the actual GEP.
2147     addr = Address(Builder.CreateInBoundsGEP(
2148         addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"),
2149         ConvertTypeForMem(eltType),
2150         addr.getAlignment());
2151   }
2152 
2153   baseType = eltType;
2154 
2155   llvm::Value *numElements
2156     = llvm::ConstantInt::get(SizeTy, countFromCLAs);
2157 
2158   // If we had any VLA dimensions, factor them in.
2159   if (numVLAElements)
2160     numElements = Builder.CreateNUWMul(numVLAElements, numElements);
2161 
2162   return numElements;
2163 }
2164 
2165 CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2166   const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2167   assert(vla && "type was not a variable array type!");
2168   return getVLASize(vla);
2169 }
2170 
2171 CodeGenFunction::VlaSizePair
2172 CodeGenFunction::getVLASize(const VariableArrayType *type) {
2173   // The number of elements so far; always size_t.
2174   llvm::Value *numElements = nullptr;
2175 
2176   QualType elementType;
2177   do {
2178     elementType = type->getElementType();
2179     llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2180     assert(vlaSize && "no size for VLA!");
2181     assert(vlaSize->getType() == SizeTy);
2182 
2183     if (!numElements) {
2184       numElements = vlaSize;
2185     } else {
2186       // It's undefined behavior if this wraps around, so mark it that way.
2187       // FIXME: Teach -fsanitize=undefined to trap this.
2188       numElements = Builder.CreateNUWMul(numElements, vlaSize);
2189     }
2190   } while ((type = getContext().getAsVariableArrayType(elementType)));
2191 
2192   return { numElements, elementType };
2193 }
2194 
2195 CodeGenFunction::VlaSizePair
2196 CodeGenFunction::getVLAElements1D(QualType type) {
2197   const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2198   assert(vla && "type was not a variable array type!");
2199   return getVLAElements1D(vla);
2200 }
2201 
2202 CodeGenFunction::VlaSizePair
2203 CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2204   llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2205   assert(VlaSize && "no size for VLA!");
2206   assert(VlaSize->getType() == SizeTy);
2207   return { VlaSize, Vla->getElementType() };
2208 }
2209 
2210 void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2211   assert(type->isVariablyModifiedType() &&
2212          "Must pass variably modified type to EmitVLASizes!");
2213 
2214   EnsureInsertPoint();
2215 
2216   // We're going to walk down into the type and look for VLA
2217   // expressions.
2218   do {
2219     assert(type->isVariablyModifiedType());
2220 
2221     const Type *ty = type.getTypePtr();
2222     switch (ty->getTypeClass()) {
2223 
2224 #define TYPE(Class, Base)
2225 #define ABSTRACT_TYPE(Class, Base)
2226 #define NON_CANONICAL_TYPE(Class, Base)
2227 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
2228 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2229 #include "clang/AST/TypeNodes.inc"
2230       llvm_unreachable("unexpected dependent type!");
2231 
2232     // These types are never variably-modified.
2233     case Type::Builtin:
2234     case Type::Complex:
2235     case Type::Vector:
2236     case Type::ExtVector:
2237     case Type::ConstantMatrix:
2238     case Type::Record:
2239     case Type::Enum:
2240     case Type::Using:
2241     case Type::TemplateSpecialization:
2242     case Type::ObjCTypeParam:
2243     case Type::ObjCObject:
2244     case Type::ObjCInterface:
2245     case Type::ObjCObjectPointer:
2246     case Type::BitInt:
2247       llvm_unreachable("type class is never variably-modified!");
2248 
2249     case Type::Elaborated:
2250       type = cast<ElaboratedType>(ty)->getNamedType();
2251       break;
2252 
2253     case Type::Adjusted:
2254       type = cast<AdjustedType>(ty)->getAdjustedType();
2255       break;
2256 
2257     case Type::Decayed:
2258       type = cast<DecayedType>(ty)->getPointeeType();
2259       break;
2260 
2261     case Type::Pointer:
2262       type = cast<PointerType>(ty)->getPointeeType();
2263       break;
2264 
2265     case Type::BlockPointer:
2266       type = cast<BlockPointerType>(ty)->getPointeeType();
2267       break;
2268 
2269     case Type::LValueReference:
2270     case Type::RValueReference:
2271       type = cast<ReferenceType>(ty)->getPointeeType();
2272       break;
2273 
2274     case Type::MemberPointer:
2275       type = cast<MemberPointerType>(ty)->getPointeeType();
2276       break;
2277 
2278     case Type::ConstantArray:
2279     case Type::IncompleteArray:
2280       // Losing element qualification here is fine.
2281       type = cast<ArrayType>(ty)->getElementType();
2282       break;
2283 
2284     case Type::VariableArray: {
2285       // Losing element qualification here is fine.
2286       const VariableArrayType *vat = cast<VariableArrayType>(ty);
2287 
2288       // Unknown size indication requires no size computation.
2289       // Otherwise, evaluate and record it.
2290       if (const Expr *sizeExpr = vat->getSizeExpr()) {
2291         // It's possible that we might have emitted this already,
2292         // e.g. with a typedef and a pointer to it.
2293         llvm::Value *&entry = VLASizeMap[sizeExpr];
2294         if (!entry) {
2295           llvm::Value *size = EmitScalarExpr(sizeExpr);
2296 
2297           // C11 6.7.6.2p5:
2298           //   If the size is an expression that is not an integer constant
2299           //   expression [...] each time it is evaluated it shall have a value
2300           //   greater than zero.
2301           if (SanOpts.has(SanitizerKind::VLABound)) {
2302             SanitizerScope SanScope(this);
2303             llvm::Value *Zero = llvm::Constant::getNullValue(size->getType());
2304             clang::QualType SEType = sizeExpr->getType();
2305             llvm::Value *CheckCondition =
2306                 SEType->isSignedIntegerType()
2307                     ? Builder.CreateICmpSGT(size, Zero)
2308                     : Builder.CreateICmpUGT(size, Zero);
2309             llvm::Constant *StaticArgs[] = {
2310                 EmitCheckSourceLocation(sizeExpr->getBeginLoc()),
2311                 EmitCheckTypeDescriptor(SEType)};
2312             EmitCheck(std::make_pair(CheckCondition, SanitizerKind::VLABound),
2313                       SanitizerHandler::VLABoundNotPositive, StaticArgs, size);
2314           }
2315 
2316           // Always zexting here would be wrong if it weren't
2317           // undefined behavior to have a negative bound.
2318           // FIXME: What about when size's type is larger than size_t?
2319           entry = Builder.CreateIntCast(size, SizeTy, /*signed*/ false);
2320         }
2321       }
2322       type = vat->getElementType();
2323       break;
2324     }
2325 
2326     case Type::FunctionProto:
2327     case Type::FunctionNoProto:
2328       type = cast<FunctionType>(ty)->getReturnType();
2329       break;
2330 
2331     case Type::Paren:
2332     case Type::TypeOf:
2333     case Type::UnaryTransform:
2334     case Type::Attributed:
2335     case Type::BTFTagAttributed:
2336     case Type::SubstTemplateTypeParm:
2337     case Type::MacroQualified:
2338       // Keep walking after single level desugaring.
2339       type = type.getSingleStepDesugaredType(getContext());
2340       break;
2341 
2342     case Type::Typedef:
2343     case Type::Decltype:
2344     case Type::Auto:
2345     case Type::DeducedTemplateSpecialization:
2346       // Stop walking: nothing to do.
2347       return;
2348 
2349     case Type::TypeOfExpr:
2350       // Stop walking: emit typeof expression.
2351       EmitIgnoredExpr(cast<TypeOfExprType>(ty)->getUnderlyingExpr());
2352       return;
2353 
2354     case Type::Atomic:
2355       type = cast<AtomicType>(ty)->getValueType();
2356       break;
2357 
2358     case Type::Pipe:
2359       type = cast<PipeType>(ty)->getElementType();
2360       break;
2361     }
2362   } while (type->isVariablyModifiedType());
2363 }
2364 
2365 Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2366   if (getContext().getBuiltinVaListType()->isArrayType())
2367     return EmitPointerWithAlignment(E);
2368   return EmitLValue(E).getAddress(*this);
2369 }
2370 
2371 Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2372   return EmitLValue(E).getAddress(*this);
2373 }
2374 
2375 void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2376                                               const APValue &Init) {
2377   assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2378   if (CGDebugInfo *Dbg = getDebugInfo())
2379     if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2380       Dbg->EmitGlobalVariable(E->getDecl(), Init);
2381 }
2382 
2383 CodeGenFunction::PeepholeProtection
2384 CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2385   // At the moment, the only aggressive peephole we do in IR gen
2386   // is trunc(zext) folding, but if we add more, we can easily
2387   // extend this protection.
2388 
2389   if (!rvalue.isScalar()) return PeepholeProtection();
2390   llvm::Value *value = rvalue.getScalarVal();
2391   if (!isa<llvm::ZExtInst>(value)) return PeepholeProtection();
2392 
2393   // Just make an extra bitcast.
2394   assert(HaveInsertPoint());
2395   llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2396                                                   Builder.GetInsertBlock());
2397 
2398   PeepholeProtection protection;
2399   protection.Inst = inst;
2400   return protection;
2401 }
2402 
2403 void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2404   if (!protection.Inst) return;
2405 
2406   // In theory, we could try to duplicate the peepholes now, but whatever.
2407   protection.Inst->eraseFromParent();
2408 }
2409 
2410 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2411                                               QualType Ty, SourceLocation Loc,
2412                                               SourceLocation AssumptionLoc,
2413                                               llvm::Value *Alignment,
2414                                               llvm::Value *OffsetValue) {
2415   if (Alignment->getType() != IntPtrTy)
2416     Alignment =
2417         Builder.CreateIntCast(Alignment, IntPtrTy, false, "casted.align");
2418   if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2419     OffsetValue =
2420         Builder.CreateIntCast(OffsetValue, IntPtrTy, true, "casted.offset");
2421   llvm::Value *TheCheck = nullptr;
2422   if (SanOpts.has(SanitizerKind::Alignment)) {
2423     llvm::Value *PtrIntValue =
2424         Builder.CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");
2425 
2426     if (OffsetValue) {
2427       bool IsOffsetZero = false;
2428       if (const auto *CI = dyn_cast<llvm::ConstantInt>(OffsetValue))
2429         IsOffsetZero = CI->isZero();
2430 
2431       if (!IsOffsetZero)
2432         PtrIntValue = Builder.CreateSub(PtrIntValue, OffsetValue, "offsetptr");
2433     }
2434 
2435     llvm::Value *Zero = llvm::ConstantInt::get(IntPtrTy, 0);
2436     llvm::Value *Mask =
2437         Builder.CreateSub(Alignment, llvm::ConstantInt::get(IntPtrTy, 1));
2438     llvm::Value *MaskedPtr = Builder.CreateAnd(PtrIntValue, Mask, "maskedptr");
2439     TheCheck = Builder.CreateICmpEQ(MaskedPtr, Zero, "maskcond");
2440   }
2441   llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2442       CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2443 
2444   if (!SanOpts.has(SanitizerKind::Alignment))
2445     return;
2446   emitAlignmentAssumptionCheck(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2447                                OffsetValue, TheCheck, Assumption);
2448 }
2449 
2450 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2451                                               const Expr *E,
2452                                               SourceLocation AssumptionLoc,
2453                                               llvm::Value *Alignment,
2454                                               llvm::Value *OffsetValue) {
2455   QualType Ty = E->getType();
2456   SourceLocation Loc = E->getExprLoc();
2457 
2458   emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2459                           OffsetValue);
2460 }
2461 
2462 llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2463                                                  llvm::Value *AnnotatedVal,
2464                                                  StringRef AnnotationStr,
2465                                                  SourceLocation Location,
2466                                                  const AnnotateAttr *Attr) {
2467   SmallVector<llvm::Value *, 5> Args = {
2468       AnnotatedVal,
2469       Builder.CreateBitCast(CGM.EmitAnnotationString(AnnotationStr),
2470                             ConstGlobalsPtrTy),
2471       Builder.CreateBitCast(CGM.EmitAnnotationUnit(Location),
2472                             ConstGlobalsPtrTy),
2473       CGM.EmitAnnotationLineNo(Location),
2474   };
2475   if (Attr)
2476     Args.push_back(CGM.EmitAnnotationArgs(Attr));
2477   return Builder.CreateCall(AnnotationFn, Args);
2478 }
2479 
2480 void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2481   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2482   // FIXME We create a new bitcast for every annotation because that's what
2483   // llvm-gcc was doing.
2484   unsigned AS = V->getType()->getPointerAddressSpace();
2485   llvm::Type *I8PtrTy = Builder.getInt8PtrTy(AS);
2486   for (const auto *I : D->specific_attrs<AnnotateAttr>())
2487     EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation,
2488                                         {I8PtrTy, CGM.ConstGlobalsPtrTy}),
2489                        Builder.CreateBitCast(V, I8PtrTy, V->getName()),
2490                        I->getAnnotation(), D->getLocation(), I);
2491 }
2492 
2493 Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2494                                               Address Addr) {
2495   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2496   llvm::Value *V = Addr.getPointer();
2497   llvm::Type *VTy = V->getType();
2498   auto *PTy = dyn_cast<llvm::PointerType>(VTy);
2499   unsigned AS = PTy ? PTy->getAddressSpace() : 0;
2500   llvm::PointerType *IntrinTy =
2501       llvm::PointerType::getWithSamePointeeType(CGM.Int8PtrTy, AS);
2502   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
2503                                        {IntrinTy, CGM.ConstGlobalsPtrTy});
2504 
2505   for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2506     // FIXME Always emit the cast inst so we can differentiate between
2507     // annotation on the first field of a struct and annotation on the struct
2508     // itself.
2509     if (VTy != IntrinTy)
2510       V = Builder.CreateBitCast(V, IntrinTy);
2511     V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation(), I);
2512     V = Builder.CreateBitCast(V, VTy);
2513   }
2514 
2515   return Address(V, Addr.getElementType(), Addr.getAlignment());
2516 }
2517 
2518 CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2519 
2520 CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2521     : CGF(CGF) {
2522   assert(!CGF->IsSanitizerScope);
2523   CGF->IsSanitizerScope = true;
2524 }
2525 
2526 CodeGenFunction::SanitizerScope::~SanitizerScope() {
2527   CGF->IsSanitizerScope = false;
2528 }
2529 
2530 void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2531                                    const llvm::Twine &Name,
2532                                    llvm::BasicBlock *BB,
2533                                    llvm::BasicBlock::iterator InsertPt) const {
2534   LoopStack.InsertHelper(I);
2535   if (IsSanitizerScope)
2536     CGM.getSanitizerMetadata()->disableSanitizerForInstruction(I);
2537 }
2538 
2539 void CGBuilderInserter::InsertHelper(
2540     llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
2541     llvm::BasicBlock::iterator InsertPt) const {
2542   llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
2543   if (CGF)
2544     CGF->InsertHelper(I, Name, BB, InsertPt);
2545 }
2546 
2547 // Emits an error if we don't have a valid set of target features for the
2548 // called function.
2549 void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2550                                           const FunctionDecl *TargetDecl) {
2551   return checkTargetFeatures(E->getBeginLoc(), TargetDecl);
2552 }
2553 
2554 // Emits an error if we don't have a valid set of target features for the
2555 // called function.
2556 void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2557                                           const FunctionDecl *TargetDecl) {
2558   // Early exit if this is an indirect call.
2559   if (!TargetDecl)
2560     return;
2561 
2562   // Get the current enclosing function if it exists. If it doesn't
2563   // we can't check the target features anyhow.
2564   const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);
2565   if (!FD)
2566     return;
2567 
2568   // Grab the required features for the call. For a builtin this is listed in
2569   // the td file with the default cpu, for an always_inline function this is any
2570   // listed cpu and any listed features.
2571   unsigned BuiltinID = TargetDecl->getBuiltinID();
2572   std::string MissingFeature;
2573   llvm::StringMap<bool> CallerFeatureMap;
2574   CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
2575   if (BuiltinID) {
2576     StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
2577     if (!Builtin::evaluateRequiredTargetFeatures(
2578         FeatureList, CallerFeatureMap)) {
2579       CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
2580           << TargetDecl->getDeclName()
2581           << FeatureList;
2582     }
2583   } else if (!TargetDecl->isMultiVersion() &&
2584              TargetDecl->hasAttr<TargetAttr>()) {
2585     // Get the required features for the callee.
2586 
2587     const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2588     ParsedTargetAttr ParsedAttr =
2589         CGM.getContext().filterFunctionTargetAttrs(TD);
2590 
2591     SmallVector<StringRef, 1> ReqFeatures;
2592     llvm::StringMap<bool> CalleeFeatureMap;
2593     CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
2594 
2595     for (const auto &F : ParsedAttr.Features) {
2596       if (F[0] == '+' && CalleeFeatureMap.lookup(F.substr(1)))
2597         ReqFeatures.push_back(StringRef(F).substr(1));
2598     }
2599 
2600     for (const auto &F : CalleeFeatureMap) {
2601       // Only positive features are "required".
2602       if (F.getValue())
2603         ReqFeatures.push_back(F.getKey());
2604     }
2605     if (!llvm::all_of(ReqFeatures, [&](StringRef Feature) {
2606       if (!CallerFeatureMap.lookup(Feature)) {
2607         MissingFeature = Feature.str();
2608         return false;
2609       }
2610       return true;
2611     }))
2612       CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2613           << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2614   }
2615 }
2616 
2617 void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2618   if (!CGM.getCodeGenOpts().SanitizeStats)
2619     return;
2620 
2621   llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2622   IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2623   CGM.getSanStats().create(IRB, SSK);
2624 }
2625 
2626 void CodeGenFunction::EmitKCFIOperandBundle(
2627     const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
2628   const FunctionProtoType *FP =
2629       Callee.getAbstractInfo().getCalleeFunctionProtoType();
2630   if (FP)
2631     Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar()));
2632 }
2633 
2634 llvm::Value *CodeGenFunction::FormAArch64ResolverCondition(
2635     const MultiVersionResolverOption &RO) {
2636   llvm::SmallVector<StringRef, 8> CondFeatures;
2637   for (const StringRef &Feature : RO.Conditions.Features) {
2638     // Form condition for features which are not yet enabled in target
2639     if (!getContext().getTargetInfo().hasFeature(Feature))
2640       CondFeatures.push_back(Feature);
2641   }
2642   if (!CondFeatures.empty()) {
2643     return EmitAArch64CpuSupports(CondFeatures);
2644   }
2645   return nullptr;
2646 }
2647 
2648 llvm::Value *CodeGenFunction::FormX86ResolverCondition(
2649     const MultiVersionResolverOption &RO) {
2650   llvm::Value *Condition = nullptr;
2651 
2652   if (!RO.Conditions.Architecture.empty())
2653     Condition = EmitX86CpuIs(RO.Conditions.Architecture);
2654 
2655   if (!RO.Conditions.Features.empty()) {
2656     llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
2657     Condition =
2658         Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
2659   }
2660   return Condition;
2661 }
2662 
2663 static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2664                                              llvm::Function *Resolver,
2665                                              CGBuilderTy &Builder,
2666                                              llvm::Function *FuncToReturn,
2667                                              bool SupportsIFunc) {
2668   if (SupportsIFunc) {
2669     Builder.CreateRet(FuncToReturn);
2670     return;
2671   }
2672 
2673   llvm::SmallVector<llvm::Value *, 10> Args(
2674       llvm::make_pointer_range(Resolver->args()));
2675 
2676   llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
2677   Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2678 
2679   if (Resolver->getReturnType()->isVoidTy())
2680     Builder.CreateRetVoid();
2681   else
2682     Builder.CreateRet(Result);
2683 }
2684 
2685 void CodeGenFunction::EmitMultiVersionResolver(
2686     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2687 
2688   llvm::Triple::ArchType ArchType =
2689       getContext().getTargetInfo().getTriple().getArch();
2690 
2691   switch (ArchType) {
2692   case llvm::Triple::x86:
2693   case llvm::Triple::x86_64:
2694     EmitX86MultiVersionResolver(Resolver, Options);
2695     return;
2696   case llvm::Triple::aarch64:
2697     EmitAArch64MultiVersionResolver(Resolver, Options);
2698     return;
2699 
2700   default:
2701     assert(false && "Only implemented for x86 and AArch64 targets");
2702   }
2703 }
2704 
2705 void CodeGenFunction::EmitAArch64MultiVersionResolver(
2706     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2707   assert(!Options.empty() && "No multiversion resolver options found");
2708   assert(Options.back().Conditions.Features.size() == 0 &&
2709          "Default case must be last");
2710   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2711   assert(SupportsIFunc &&
2712          "Multiversion resolver requires target IFUNC support");
2713   bool AArch64CpuInitialized = false;
2714   llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2715 
2716   for (const MultiVersionResolverOption &RO : Options) {
2717     Builder.SetInsertPoint(CurBlock);
2718     llvm::Value *Condition = FormAArch64ResolverCondition(RO);
2719 
2720     // The 'default' or 'all features enabled' case.
2721     if (!Condition) {
2722       CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2723                                        SupportsIFunc);
2724       return;
2725     }
2726 
2727     if (!AArch64CpuInitialized) {
2728       Builder.SetInsertPoint(CurBlock, CurBlock->begin());
2729       EmitAArch64CpuInit();
2730       AArch64CpuInitialized = true;
2731       Builder.SetInsertPoint(CurBlock);
2732     }
2733 
2734     llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2735     CGBuilderTy RetBuilder(*this, RetBlock);
2736     CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2737                                      SupportsIFunc);
2738     CurBlock = createBasicBlock("resolver_else", Resolver);
2739     Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2740   }
2741 
2742   // If no default, emit an unreachable.
2743   Builder.SetInsertPoint(CurBlock);
2744   llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2745   TrapCall->setDoesNotReturn();
2746   TrapCall->setDoesNotThrow();
2747   Builder.CreateUnreachable();
2748   Builder.ClearInsertionPoint();
2749 }
2750 
2751 void CodeGenFunction::EmitX86MultiVersionResolver(
2752     llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2753 
2754   bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2755 
2756   // Main function's basic block.
2757   llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2758   Builder.SetInsertPoint(CurBlock);
2759   EmitX86CpuInit();
2760 
2761   for (const MultiVersionResolverOption &RO : Options) {
2762     Builder.SetInsertPoint(CurBlock);
2763     llvm::Value *Condition = FormX86ResolverCondition(RO);
2764 
2765     // The 'default' or 'generic' case.
2766     if (!Condition) {
2767       assert(&RO == Options.end() - 1 &&
2768              "Default or Generic case must be last");
2769       CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2770                                        SupportsIFunc);
2771       return;
2772     }
2773 
2774     llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2775     CGBuilderTy RetBuilder(*this, RetBlock);
2776     CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2777                                      SupportsIFunc);
2778     CurBlock = createBasicBlock("resolver_else", Resolver);
2779     Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2780   }
2781 
2782   // If no generic/default, emit an unreachable.
2783   Builder.SetInsertPoint(CurBlock);
2784   llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2785   TrapCall->setDoesNotReturn();
2786   TrapCall->setDoesNotThrow();
2787   Builder.CreateUnreachable();
2788   Builder.ClearInsertionPoint();
2789 }
2790 
2791 // Loc - where the diagnostic will point, where in the source code this
2792 //  alignment has failed.
2793 // SecondaryLoc - if present (will be present if sufficiently different from
2794 //  Loc), the diagnostic will additionally point a "Note:" to this location.
2795 //  It should be the location where the __attribute__((assume_aligned))
2796 //  was written e.g.
2797 void CodeGenFunction::emitAlignmentAssumptionCheck(
2798     llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
2799     SourceLocation SecondaryLoc, llvm::Value *Alignment,
2800     llvm::Value *OffsetValue, llvm::Value *TheCheck,
2801     llvm::Instruction *Assumption) {
2802   assert(Assumption && isa<llvm::CallInst>(Assumption) &&
2803          cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
2804              llvm::Intrinsic::getDeclaration(
2805                  Builder.GetInsertBlock()->getParent()->getParent(),
2806                  llvm::Intrinsic::assume) &&
2807          "Assumption should be a call to llvm.assume().");
2808   assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
2809          "Assumption should be the last instruction of the basic block, "
2810          "since the basic block is still being generated.");
2811 
2812   if (!SanOpts.has(SanitizerKind::Alignment))
2813     return;
2814 
2815   // Don't check pointers to volatile data. The behavior here is implementation-
2816   // defined.
2817   if (Ty->getPointeeType().isVolatileQualified())
2818     return;
2819 
2820   // We need to temorairly remove the assumption so we can insert the
2821   // sanitizer check before it, else the check will be dropped by optimizations.
2822   Assumption->removeFromParent();
2823 
2824   {
2825     SanitizerScope SanScope(this);
2826 
2827     if (!OffsetValue)
2828       OffsetValue = Builder.getInt1(false); // no offset.
2829 
2830     llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
2831                                     EmitCheckSourceLocation(SecondaryLoc),
2832                                     EmitCheckTypeDescriptor(Ty)};
2833     llvm::Value *DynamicData[] = {EmitCheckValue(Ptr),
2834                                   EmitCheckValue(Alignment),
2835                                   EmitCheckValue(OffsetValue)};
2836     EmitCheck({std::make_pair(TheCheck, SanitizerKind::Alignment)},
2837               SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
2838   }
2839 
2840   // We are now in the (new, empty) "cont" basic block.
2841   // Reintroduce the assumption.
2842   Builder.Insert(Assumption);
2843   // FIXME: Assumption still has it's original basic block as it's Parent.
2844 }
2845 
2846 llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
2847   if (CGDebugInfo *DI = getDebugInfo())
2848     return DI->SourceLocToDebugLoc(Location);
2849 
2850   return llvm::DebugLoc();
2851 }
2852 
2853 llvm::Value *
2854 CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
2855                                                       Stmt::Likelihood LH) {
2856   switch (LH) {
2857   case Stmt::LH_None:
2858     return Cond;
2859   case Stmt::LH_Likely:
2860   case Stmt::LH_Unlikely:
2861     // Don't generate llvm.expect on -O0 as the backend won't use it for
2862     // anything.
2863     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2864       return Cond;
2865     llvm::Type *CondTy = Cond->getType();
2866     assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
2867     llvm::Function *FnExpect =
2868         CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
2869     llvm::Value *ExpectedValueOfCond =
2870         llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
2871     return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
2872                               Cond->getName() + ".expval");
2873   }
2874   llvm_unreachable("Unknown Likelihood");
2875 }
2876 
2877 llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,
2878                                                     unsigned NumElementsDst,
2879                                                     const llvm::Twine &Name) {
2880   auto *SrcTy = cast<llvm::FixedVectorType>(SrcVec->getType());
2881   unsigned NumElementsSrc = SrcTy->getNumElements();
2882   if (NumElementsSrc == NumElementsDst)
2883     return SrcVec;
2884 
2885   std::vector<int> ShuffleMask(NumElementsDst, -1);
2886   for (unsigned MaskIdx = 0;
2887        MaskIdx < std::min<>(NumElementsDst, NumElementsSrc); ++MaskIdx)
2888     ShuffleMask[MaskIdx] = MaskIdx;
2889 
2890   return Builder.CreateShuffleVector(SrcVec, ShuffleMask, Name);
2891 }
2892