xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp (revision 725a9f47324d42037db93c27ceb40d4956872f3e)
1 //===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Expr nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCUDARuntime.h"
14 #include "CGCXXABI.h"
15 #include "CGCall.h"
16 #include "CGCleanup.h"
17 #include "CGDebugInfo.h"
18 #include "CGObjCRuntime.h"
19 #include "CGOpenMPRuntime.h"
20 #include "CGRecordLayout.h"
21 #include "CodeGenFunction.h"
22 #include "CodeGenModule.h"
23 #include "ConstantEmitter.h"
24 #include "TargetInfo.h"
25 #include "clang/AST/ASTContext.h"
26 #include "clang/AST/Attr.h"
27 #include "clang/AST/DeclObjC.h"
28 #include "clang/AST/NSAPI.h"
29 #include "clang/AST/StmtVisitor.h"
30 #include "clang/Basic/Builtins.h"
31 #include "clang/Basic/CodeGenOptions.h"
32 #include "clang/Basic/SourceManager.h"
33 #include "llvm/ADT/Hashing.h"
34 #include "llvm/ADT/STLExtras.h"
35 #include "llvm/ADT/StringExtras.h"
36 #include "llvm/IR/DataLayout.h"
37 #include "llvm/IR/Intrinsics.h"
38 #include "llvm/IR/IntrinsicsWebAssembly.h"
39 #include "llvm/IR/LLVMContext.h"
40 #include "llvm/IR/MDBuilder.h"
41 #include "llvm/IR/MatrixBuilder.h"
42 #include "llvm/Passes/OptimizationLevel.h"
43 #include "llvm/Support/ConvertUTF.h"
44 #include "llvm/Support/MathExtras.h"
45 #include "llvm/Support/Path.h"
46 #include "llvm/Support/SaveAndRestore.h"
47 #include "llvm/Support/xxhash.h"
48 #include "llvm/Transforms/Utils/SanitizerStats.h"
49 
50 #include <optional>
51 #include <string>
52 
53 using namespace clang;
54 using namespace CodeGen;
55 
56 // Experiment to make sanitizers easier to debug
57 static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization(
58     "ubsan-unique-traps", llvm::cl::Optional,
59     llvm::cl::desc("Deoptimize traps for UBSAN so there is 1 trap per check"),
60     llvm::cl::init(false));
61 
62 //===--------------------------------------------------------------------===//
63 //                        Miscellaneous Helper Methods
64 //===--------------------------------------------------------------------===//
65 
66 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
67 /// block.
68 Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
69                                                      CharUnits Align,
70                                                      const Twine &Name,
71                                                      llvm::Value *ArraySize) {
72   auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
73   Alloca->setAlignment(Align.getAsAlign());
74   return Address(Alloca, Ty, Align, KnownNonNull);
75 }
76 
77 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
78 /// block. The alloca is casted to default address space if necessary.
79 Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
80                                           const Twine &Name,
81                                           llvm::Value *ArraySize,
82                                           Address *AllocaAddr) {
83   auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
84   if (AllocaAddr)
85     *AllocaAddr = Alloca;
86   llvm::Value *V = Alloca.getPointer();
87   // Alloca always returns a pointer in alloca address space, which may
88   // be different from the type defined by the language. For example,
89   // in C++ the auto variables are in the default address space. Therefore
90   // cast alloca to the default address space when necessary.
91   if (getASTAllocaAddressSpace() != LangAS::Default) {
92     auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default);
93     llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
94     // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
95     // otherwise alloca is inserted at the current insertion point of the
96     // builder.
97     if (!ArraySize)
98       Builder.SetInsertPoint(getPostAllocaInsertPoint());
99     V = getTargetHooks().performAddrSpaceCast(
100         *this, V, getASTAllocaAddressSpace(), LangAS::Default,
101         Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
102   }
103 
104   return Address(V, Ty, Align, KnownNonNull);
105 }
106 
107 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
108 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
109 /// insertion point of the builder.
110 llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
111                                                     const Twine &Name,
112                                                     llvm::Value *ArraySize) {
113   if (ArraySize)
114     return Builder.CreateAlloca(Ty, ArraySize, Name);
115   return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
116                               ArraySize, Name, AllocaInsertPt);
117 }
118 
119 /// CreateDefaultAlignTempAlloca - This creates an alloca with the
120 /// default alignment of the corresponding LLVM type, which is *not*
121 /// guaranteed to be related in any way to the expected alignment of
122 /// an AST type that might have been lowered to Ty.
123 Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
124                                                       const Twine &Name) {
125   CharUnits Align =
126       CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty));
127   return CreateTempAlloca(Ty, Align, Name);
128 }
129 
130 Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
131   CharUnits Align = getContext().getTypeAlignInChars(Ty);
132   return CreateTempAlloca(ConvertType(Ty), Align, Name);
133 }
134 
135 Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
136                                        Address *Alloca) {
137   // FIXME: Should we prefer the preferred type alignment here?
138   return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);
139 }
140 
141 Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
142                                        const Twine &Name, Address *Alloca) {
143   Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,
144                                     /*ArraySize=*/nullptr, Alloca);
145 
146   if (Ty->isConstantMatrixType()) {
147     auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
148     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
149                                                 ArrayTy->getNumElements());
150 
151     Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),
152                      KnownNonNull);
153   }
154   return Result;
155 }
156 
157 Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align,
158                                                   const Twine &Name) {
159   return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);
160 }
161 
162 Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
163                                                   const Twine &Name) {
164   return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),
165                                   Name);
166 }
167 
168 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
169 /// expression and compare the result against zero, returning an Int1Ty value.
170 llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
171   PGO.setCurrentStmt(E);
172   if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
173     llvm::Value *MemPtr = EmitScalarExpr(E);
174     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
175   }
176 
177   QualType BoolTy = getContext().BoolTy;
178   SourceLocation Loc = E->getExprLoc();
179   CGFPOptionsRAII FPOptsRAII(*this, E);
180   if (!E->getType()->isAnyComplexType())
181     return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc);
182 
183   return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy,
184                                        Loc);
185 }
186 
187 /// EmitIgnoredExpr - Emit code to compute the specified expression,
188 /// ignoring the result.
189 void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
190   if (E->isPRValue())
191     return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true);
192 
193   // if this is a bitfield-resulting conditional operator, we can special case
194   // emit this. The normal 'EmitLValue' version of this is particularly
195   // difficult to codegen for, since creating a single "LValue" for two
196   // different sized arguments here is not particularly doable.
197   if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
198           E->IgnoreParenNoopCasts(getContext()))) {
199     if (CondOp->getObjectKind() == OK_BitField)
200       return EmitIgnoredConditionalOperator(CondOp);
201   }
202 
203   // Just emit it as an l-value and drop the result.
204   EmitLValue(E);
205 }
206 
207 /// EmitAnyExpr - Emit code to compute the specified expression which
208 /// can have any type.  The result is returned as an RValue struct.
209 /// If this is an aggregate expression, AggSlot indicates where the
210 /// result should be returned.
211 RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
212                                     AggValueSlot aggSlot,
213                                     bool ignoreResult) {
214   switch (getEvaluationKind(E->getType())) {
215   case TEK_Scalar:
216     return RValue::get(EmitScalarExpr(E, ignoreResult));
217   case TEK_Complex:
218     return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));
219   case TEK_Aggregate:
220     if (!ignoreResult && aggSlot.isIgnored())
221       aggSlot = CreateAggTemp(E->getType(), "agg-temp");
222     EmitAggExpr(E, aggSlot);
223     return aggSlot.asRValue();
224   }
225   llvm_unreachable("bad evaluation kind");
226 }
227 
228 /// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
229 /// always be accessible even if no aggregate location is provided.
230 RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
231   AggValueSlot AggSlot = AggValueSlot::ignored();
232 
233   if (hasAggregateEvaluationKind(E->getType()))
234     AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
235   return EmitAnyExpr(E, AggSlot);
236 }
237 
238 /// EmitAnyExprToMem - Evaluate an expression into a given memory
239 /// location.
240 void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
241                                        Address Location,
242                                        Qualifiers Quals,
243                                        bool IsInit) {
244   // FIXME: This function should take an LValue as an argument.
245   switch (getEvaluationKind(E->getType())) {
246   case TEK_Complex:
247     EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()),
248                               /*isInit*/ false);
249     return;
250 
251   case TEK_Aggregate: {
252     EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
253                                          AggValueSlot::IsDestructed_t(IsInit),
254                                          AggValueSlot::DoesNotNeedGCBarriers,
255                                          AggValueSlot::IsAliased_t(!IsInit),
256                                          AggValueSlot::MayOverlap));
257     return;
258   }
259 
260   case TEK_Scalar: {
261     RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
262     LValue LV = MakeAddrLValue(Location, E->getType());
263     EmitStoreThroughLValue(RV, LV);
264     return;
265   }
266   }
267   llvm_unreachable("bad evaluation kind");
268 }
269 
270 static void
271 pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
272                      const Expr *E, Address ReferenceTemporary) {
273   // Objective-C++ ARC:
274   //   If we are binding a reference to a temporary that has ownership, we
275   //   need to perform retain/release operations on the temporary.
276   //
277   // FIXME: This should be looking at E, not M.
278   if (auto Lifetime = M->getType().getObjCLifetime()) {
279     switch (Lifetime) {
280     case Qualifiers::OCL_None:
281     case Qualifiers::OCL_ExplicitNone:
282       // Carry on to normal cleanup handling.
283       break;
284 
285     case Qualifiers::OCL_Autoreleasing:
286       // Nothing to do; cleaned up by an autorelease pool.
287       return;
288 
289     case Qualifiers::OCL_Strong:
290     case Qualifiers::OCL_Weak:
291       switch (StorageDuration Duration = M->getStorageDuration()) {
292       case SD_Static:
293         // Note: we intentionally do not register a cleanup to release
294         // the object on program termination.
295         return;
296 
297       case SD_Thread:
298         // FIXME: We should probably register a cleanup in this case.
299         return;
300 
301       case SD_Automatic:
302       case SD_FullExpression:
303         CodeGenFunction::Destroyer *Destroy;
304         CleanupKind CleanupKind;
305         if (Lifetime == Qualifiers::OCL_Strong) {
306           const ValueDecl *VD = M->getExtendingDecl();
307           bool Precise =
308               VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>();
309           CleanupKind = CGF.getARCCleanupKind();
310           Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
311                             : &CodeGenFunction::destroyARCStrongImprecise;
312         } else {
313           // __weak objects always get EH cleanups; otherwise, exceptions
314           // could cause really nasty crashes instead of mere leaks.
315           CleanupKind = NormalAndEHCleanup;
316           Destroy = &CodeGenFunction::destroyARCWeak;
317         }
318         if (Duration == SD_FullExpression)
319           CGF.pushDestroy(CleanupKind, ReferenceTemporary,
320                           M->getType(), *Destroy,
321                           CleanupKind & EHCleanup);
322         else
323           CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,
324                                           M->getType(),
325                                           *Destroy, CleanupKind & EHCleanup);
326         return;
327 
328       case SD_Dynamic:
329         llvm_unreachable("temporary cannot have dynamic storage duration");
330       }
331       llvm_unreachable("unknown storage duration");
332     }
333   }
334 
335   CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
336   if (const RecordType *RT =
337           E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
338     // Get the destructor for the reference temporary.
339     auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
340     if (!ClassDecl->hasTrivialDestructor())
341       ReferenceTemporaryDtor = ClassDecl->getDestructor();
342   }
343 
344   if (!ReferenceTemporaryDtor)
345     return;
346 
347   // Call the destructor for the temporary.
348   switch (M->getStorageDuration()) {
349   case SD_Static:
350   case SD_Thread: {
351     llvm::FunctionCallee CleanupFn;
352     llvm::Constant *CleanupArg;
353     if (E->getType()->isArrayType()) {
354       CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
355           ReferenceTemporary, E->getType(),
356           CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions,
357           dyn_cast_or_null<VarDecl>(M->getExtendingDecl()));
358       CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);
359     } else {
360       CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
361           GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
362       CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
363     }
364     CGF.CGM.getCXXABI().registerGlobalDtor(
365         CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
366     break;
367   }
368 
369   case SD_FullExpression:
370     CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(),
371                     CodeGenFunction::destroyCXXObject,
372                     CGF.getLangOpts().Exceptions);
373     break;
374 
375   case SD_Automatic:
376     CGF.pushLifetimeExtendedDestroy(NormalAndEHCleanup,
377                                     ReferenceTemporary, E->getType(),
378                                     CodeGenFunction::destroyCXXObject,
379                                     CGF.getLangOpts().Exceptions);
380     break;
381 
382   case SD_Dynamic:
383     llvm_unreachable("temporary cannot have dynamic storage duration");
384   }
385 }
386 
387 static Address createReferenceTemporary(CodeGenFunction &CGF,
388                                         const MaterializeTemporaryExpr *M,
389                                         const Expr *Inner,
390                                         Address *Alloca = nullptr) {
391   auto &TCG = CGF.getTargetHooks();
392   switch (M->getStorageDuration()) {
393   case SD_FullExpression:
394   case SD_Automatic: {
395     // If we have a constant temporary array or record try to promote it into a
396     // constant global under the same rules a normal constant would've been
397     // promoted. This is easier on the optimizer and generally emits fewer
398     // instructions.
399     QualType Ty = Inner->getType();
400     if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
401         (Ty->isArrayType() || Ty->isRecordType()) &&
402         Ty.isConstantStorage(CGF.getContext(), true, false))
403       if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) {
404         auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
405         auto *GV = new llvm::GlobalVariable(
406             CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
407             llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
408             llvm::GlobalValue::NotThreadLocal,
409             CGF.getContext().getTargetAddressSpace(AS));
410         CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
411         GV->setAlignment(alignment.getAsAlign());
412         llvm::Constant *C = GV;
413         if (AS != LangAS::Default)
414           C = TCG.performAddrSpaceCast(
415               CGF.CGM, GV, AS, LangAS::Default,
416               GV->getValueType()->getPointerTo(
417                   CGF.getContext().getTargetAddressSpace(LangAS::Default)));
418         // FIXME: Should we put the new global into a COMDAT?
419         return Address(C, GV->getValueType(), alignment);
420       }
421     return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
422   }
423   case SD_Thread:
424   case SD_Static:
425     return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);
426 
427   case SD_Dynamic:
428     llvm_unreachable("temporary can't have dynamic storage duration");
429   }
430   llvm_unreachable("unknown storage duration");
431 }
432 
433 /// Helper method to check if the underlying ABI is AAPCS
434 static bool isAAPCS(const TargetInfo &TargetInfo) {
435   return TargetInfo.getABI().starts_with("aapcs");
436 }
437 
438 LValue CodeGenFunction::
439 EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
440   const Expr *E = M->getSubExpr();
441 
442   assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
443           !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
444          "Reference should never be pseudo-strong!");
445 
446   // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
447   // as that will cause the lifetime adjustment to be lost for ARC
448   auto ownership = M->getType().getObjCLifetime();
449   if (ownership != Qualifiers::OCL_None &&
450       ownership != Qualifiers::OCL_ExplicitNone) {
451     Address Object = createReferenceTemporary(*this, M, E);
452     if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
453       llvm::Type *Ty = ConvertTypeForMem(E->getType());
454       Object = Object.withElementType(Ty);
455 
456       // createReferenceTemporary will promote the temporary to a global with a
457       // constant initializer if it can.  It can only do this to a value of
458       // ARC-manageable type if the value is global and therefore "immune" to
459       // ref-counting operations.  Therefore we have no need to emit either a
460       // dynamic initialization or a cleanup and we can just return the address
461       // of the temporary.
462       if (Var->hasInitializer())
463         return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
464 
465       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
466     }
467     LValue RefTempDst = MakeAddrLValue(Object, M->getType(),
468                                        AlignmentSource::Decl);
469 
470     switch (getEvaluationKind(E->getType())) {
471     default: llvm_unreachable("expected scalar or aggregate expression");
472     case TEK_Scalar:
473       EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);
474       break;
475     case TEK_Aggregate: {
476       EmitAggExpr(E, AggValueSlot::forAddr(Object,
477                                            E->getType().getQualifiers(),
478                                            AggValueSlot::IsDestructed,
479                                            AggValueSlot::DoesNotNeedGCBarriers,
480                                            AggValueSlot::IsNotAliased,
481                                            AggValueSlot::DoesNotOverlap));
482       break;
483     }
484     }
485 
486     pushTemporaryCleanup(*this, M, E, Object);
487     return RefTempDst;
488   }
489 
490   SmallVector<const Expr *, 2> CommaLHSs;
491   SmallVector<SubobjectAdjustment, 2> Adjustments;
492   E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);
493 
494   for (const auto &Ignored : CommaLHSs)
495     EmitIgnoredExpr(Ignored);
496 
497   if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) {
498     if (opaque->getType()->isRecordType()) {
499       assert(Adjustments.empty());
500       return EmitOpaqueValueLValue(opaque);
501     }
502   }
503 
504   // Create and initialize the reference temporary.
505   Address Alloca = Address::invalid();
506   Address Object = createReferenceTemporary(*this, M, E, &Alloca);
507   if (auto *Var = dyn_cast<llvm::GlobalVariable>(
508           Object.getPointer()->stripPointerCasts())) {
509     llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
510     Object = Object.withElementType(TemporaryType);
511     // If the temporary is a global and has a constant initializer or is a
512     // constant temporary that we promoted to a global, we may have already
513     // initialized it.
514     if (!Var->hasInitializer()) {
515       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
516       EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
517     }
518   } else {
519     switch (M->getStorageDuration()) {
520     case SD_Automatic:
521       if (auto *Size = EmitLifetimeStart(
522               CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
523               Alloca.getPointer())) {
524         pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker,
525                                                   Alloca, Size);
526       }
527       break;
528 
529     case SD_FullExpression: {
530       if (!ShouldEmitLifetimeMarkers)
531         break;
532 
533       // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
534       // marker. Instead, start the lifetime of a conditional temporary earlier
535       // so that it's unconditional. Don't do this with sanitizers which need
536       // more precise lifetime marks. However when inside an "await.suspend"
537       // block, we should always avoid conditional cleanup because it creates
538       // boolean marker that lives across await_suspend, which can destroy coro
539       // frame.
540       ConditionalEvaluation *OldConditional = nullptr;
541       CGBuilderTy::InsertPoint OldIP;
542       if (isInConditionalBranch() && !E->getType().isDestructedType() &&
543           ((!SanOpts.has(SanitizerKind::HWAddress) &&
544             !SanOpts.has(SanitizerKind::Memory) &&
545             !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
546            inSuspendBlock())) {
547         OldConditional = OutermostConditional;
548         OutermostConditional = nullptr;
549 
550         OldIP = Builder.saveIP();
551         llvm::BasicBlock *Block = OldConditional->getStartingBlock();
552         Builder.restoreIP(CGBuilderTy::InsertPoint(
553             Block, llvm::BasicBlock::iterator(Block->back())));
554       }
555 
556       if (auto *Size = EmitLifetimeStart(
557               CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
558               Alloca.getPointer())) {
559         pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca,
560                                              Size);
561       }
562 
563       if (OldConditional) {
564         OutermostConditional = OldConditional;
565         Builder.restoreIP(OldIP);
566       }
567       break;
568     }
569 
570     default:
571       break;
572     }
573     EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
574   }
575   pushTemporaryCleanup(*this, M, E, Object);
576 
577   // Perform derived-to-base casts and/or field accesses, to get from the
578   // temporary object we created (and, potentially, for which we extended
579   // the lifetime) to the subobject we're binding the reference to.
580   for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) {
581     switch (Adjustment.Kind) {
582     case SubobjectAdjustment::DerivedToBaseAdjustment:
583       Object =
584           GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass,
585                                 Adjustment.DerivedToBase.BasePath->path_begin(),
586                                 Adjustment.DerivedToBase.BasePath->path_end(),
587                                 /*NullCheckValue=*/ false, E->getExprLoc());
588       break;
589 
590     case SubobjectAdjustment::FieldAdjustment: {
591       LValue LV = MakeAddrLValue(Object, E->getType(), AlignmentSource::Decl);
592       LV = EmitLValueForField(LV, Adjustment.Field);
593       assert(LV.isSimple() &&
594              "materialized temporary field is not a simple lvalue");
595       Object = LV.getAddress(*this);
596       break;
597     }
598 
599     case SubobjectAdjustment::MemberPointerAdjustment: {
600       llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS);
601       Object = EmitCXXMemberDataPointerAddress(E, Object, Ptr,
602                                                Adjustment.Ptr.MPT);
603       break;
604     }
605     }
606   }
607 
608   return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
609 }
610 
611 RValue
612 CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
613   // Emit the expression as an lvalue.
614   LValue LV = EmitLValue(E);
615   assert(LV.isSimple());
616   llvm::Value *Value = LV.getPointer(*this);
617 
618   if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
619     // C++11 [dcl.ref]p5 (as amended by core issue 453):
620     //   If a glvalue to which a reference is directly bound designates neither
621     //   an existing object or function of an appropriate type nor a region of
622     //   storage of suitable size and alignment to contain an object of the
623     //   reference's type, the behavior is undefined.
624     QualType Ty = E->getType();
625     EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty);
626   }
627 
628   return RValue::get(Value);
629 }
630 
631 
632 /// getAccessedFieldNo - Given an encoded value and a result number, return the
633 /// input field number being accessed.
634 unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
635                                              const llvm::Constant *Elts) {
636   return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
637       ->getZExtValue();
638 }
639 
640 /// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
641 static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
642                                     llvm::Value *High) {
643   llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
644   llvm::Value *K47 = Builder.getInt64(47);
645   llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
646   llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
647   llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
648   llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
649   return Builder.CreateMul(B1, KMul);
650 }
651 
652 bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
653   return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
654          TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
655 }
656 
657 bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
658   CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
659   return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
660          (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
661           TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
662           TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
663 }
664 
665 bool CodeGenFunction::sanitizePerformTypeCheck() const {
666   return SanOpts.has(SanitizerKind::Null) ||
667          SanOpts.has(SanitizerKind::Alignment) ||
668          SanOpts.has(SanitizerKind::ObjectSize) ||
669          SanOpts.has(SanitizerKind::Vptr);
670 }
671 
672 void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
673                                     llvm::Value *Ptr, QualType Ty,
674                                     CharUnits Alignment,
675                                     SanitizerSet SkippedChecks,
676                                     llvm::Value *ArraySize) {
677   if (!sanitizePerformTypeCheck())
678     return;
679 
680   // Don't check pointers outside the default address space. The null check
681   // isn't correct, the object-size check isn't supported by LLVM, and we can't
682   // communicate the addresses to the runtime handler for the vptr check.
683   if (Ptr->getType()->getPointerAddressSpace())
684     return;
685 
686   // Don't check pointers to volatile data. The behavior here is implementation-
687   // defined.
688   if (Ty.isVolatileQualified())
689     return;
690 
691   SanitizerScope SanScope(this);
692 
693   SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks;
694   llvm::BasicBlock *Done = nullptr;
695 
696   // Quickly determine whether we have a pointer to an alloca. It's possible
697   // to skip null checks, and some alignment checks, for these pointers. This
698   // can reduce compile-time significantly.
699   auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
700 
701   llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());
702   llvm::Value *IsNonNull = nullptr;
703   bool IsGuaranteedNonNull =
704       SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca;
705   bool AllowNullPointers = isNullPointerAllowed(TCK);
706   if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
707       !IsGuaranteedNonNull) {
708     // The glvalue must not be an empty glvalue.
709     IsNonNull = Builder.CreateIsNotNull(Ptr);
710 
711     // The IR builder can constant-fold the null check if the pointer points to
712     // a constant.
713     IsGuaranteedNonNull = IsNonNull == True;
714 
715     // Skip the null check if the pointer is known to be non-null.
716     if (!IsGuaranteedNonNull) {
717       if (AllowNullPointers) {
718         // When performing pointer casts, it's OK if the value is null.
719         // Skip the remaining checks in that case.
720         Done = createBasicBlock("null");
721         llvm::BasicBlock *Rest = createBasicBlock("not.null");
722         Builder.CreateCondBr(IsNonNull, Rest, Done);
723         EmitBlock(Rest);
724       } else {
725         Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null));
726       }
727     }
728   }
729 
730   if (SanOpts.has(SanitizerKind::ObjectSize) &&
731       !SkippedChecks.has(SanitizerKind::ObjectSize) &&
732       !Ty->isIncompleteType()) {
733     uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
734     llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);
735     if (ArraySize)
736       Size = Builder.CreateMul(Size, ArraySize);
737 
738     // Degenerate case: new X[0] does not need an objectsize check.
739     llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
740     if (!ConstantSize || !ConstantSize->isNullValue()) {
741       // The glvalue must refer to a large enough storage region.
742       // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
743       //        to check this.
744       // FIXME: Get object address space
745       llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
746       llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
747       llvm::Value *Min = Builder.getFalse();
748       llvm::Value *NullIsUnknown = Builder.getFalse();
749       llvm::Value *Dynamic = Builder.getFalse();
750       llvm::Value *LargeEnough = Builder.CreateICmpUGE(
751           Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic}), Size);
752       Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
753     }
754   }
755 
756   llvm::MaybeAlign AlignVal;
757   llvm::Value *PtrAsInt = nullptr;
758 
759   if (SanOpts.has(SanitizerKind::Alignment) &&
760       !SkippedChecks.has(SanitizerKind::Alignment)) {
761     AlignVal = Alignment.getAsMaybeAlign();
762     if (!Ty->isIncompleteType() && !AlignVal)
763       AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
764                                              /*ForPointeeType=*/true)
765                      .getAsMaybeAlign();
766 
767     // The glvalue must be suitably aligned.
768     if (AlignVal && *AlignVal > llvm::Align(1) &&
769         (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
770       PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
771       llvm::Value *Align = Builder.CreateAnd(
772           PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
773       llvm::Value *Aligned =
774           Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
775       if (Aligned != True)
776         Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment));
777     }
778   }
779 
780   if (Checks.size() > 0) {
781     llvm::Constant *StaticData[] = {
782         EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty),
783         llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
784         llvm::ConstantInt::get(Int8Ty, TCK)};
785     EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
786               PtrAsInt ? PtrAsInt : Ptr);
787   }
788 
789   // If possible, check that the vptr indicates that there is a subobject of
790   // type Ty at offset zero within this object.
791   //
792   // C++11 [basic.life]p5,6:
793   //   [For storage which does not refer to an object within its lifetime]
794   //   The program has undefined behavior if:
795   //    -- the [pointer or glvalue] is used to access a non-static data member
796   //       or call a non-static member function
797   if (SanOpts.has(SanitizerKind::Vptr) &&
798       !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
799     // Ensure that the pointer is non-null before loading it. If there is no
800     // compile-time guarantee, reuse the run-time null check or emit a new one.
801     if (!IsGuaranteedNonNull) {
802       if (!IsNonNull)
803         IsNonNull = Builder.CreateIsNotNull(Ptr);
804       if (!Done)
805         Done = createBasicBlock("vptr.null");
806       llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null");
807       Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);
808       EmitBlock(VptrNotNull);
809     }
810 
811     // Compute a hash of the mangled name of the type.
812     //
813     // FIXME: This is not guaranteed to be deterministic! Move to a
814     //        fingerprinting mechanism once LLVM provides one. For the time
815     //        being the implementation happens to be deterministic.
816     SmallString<64> MangledName;
817     llvm::raw_svector_ostream Out(MangledName);
818     CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(),
819                                                      Out);
820 
821     // Contained in NoSanitizeList based on the mangled type.
822     if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr,
823                                                            Out.str())) {
824       llvm::hash_code TypeHash = hash_value(Out.str());
825 
826       // Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
827       llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
828       Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());
829       llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
830       llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
831 
832       llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
833       Hash = Builder.CreateTrunc(Hash, IntPtrTy);
834 
835       // Look the hash up in our cache.
836       const int CacheSize = 128;
837       llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);
838       llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,
839                                                      "__ubsan_vptr_type_cache");
840       llvm::Value *Slot = Builder.CreateAnd(Hash,
841                                             llvm::ConstantInt::get(IntPtrTy,
842                                                                    CacheSize-1));
843       llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
844       llvm::Value *CacheVal = Builder.CreateAlignedLoad(
845           IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices),
846           getPointerAlign());
847 
848       // If the hash isn't in the cache, call a runtime handler to perform the
849       // hard work of checking whether the vptr is for an object of the right
850       // type. This will either fill in the cache and return, or produce a
851       // diagnostic.
852       llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash);
853       llvm::Constant *StaticData[] = {
854         EmitCheckSourceLocation(Loc),
855         EmitCheckTypeDescriptor(Ty),
856         CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()),
857         llvm::ConstantInt::get(Int8Ty, TCK)
858       };
859       llvm::Value *DynamicData[] = { Ptr, Hash };
860       EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
861                 SanitizerHandler::DynamicTypeCacheMiss, StaticData,
862                 DynamicData);
863     }
864   }
865 
866   if (Done) {
867     Builder.CreateBr(Done);
868     EmitBlock(Done);
869   }
870 }
871 
872 llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
873                                                    QualType EltTy) {
874   ASTContext &C = getContext();
875   uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity();
876   if (!EltSize)
877     return nullptr;
878 
879   auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts());
880   if (!ArrayDeclRef)
881     return nullptr;
882 
883   auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());
884   if (!ParamDecl)
885     return nullptr;
886 
887   auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
888   if (!POSAttr)
889     return nullptr;
890 
891   // Don't load the size if it's a lower bound.
892   int POSType = POSAttr->getType();
893   if (POSType != 0 && POSType != 1)
894     return nullptr;
895 
896   // Find the implicit size parameter.
897   auto PassedSizeIt = SizeArguments.find(ParamDecl);
898   if (PassedSizeIt == SizeArguments.end())
899     return nullptr;
900 
901   const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
902   assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
903   Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
904   llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false,
905                                               C.getSizeType(), E->getExprLoc());
906   llvm::Value *SizeOfElement =
907       llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);
908   return Builder.CreateUDiv(SizeInBytes, SizeOfElement);
909 }
910 
911 /// If Base is known to point to the start of an array, return the length of
912 /// that array. Return 0 if the length cannot be determined.
913 static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
914                                           const Expr *Base,
915                                           QualType &IndexedType,
916                                           LangOptions::StrictFlexArraysLevelKind
917                                           StrictFlexArraysLevel) {
918   // For the vector indexing extension, the bound is the number of elements.
919   if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
920     IndexedType = Base->getType();
921     return CGF.Builder.getInt32(VT->getNumElements());
922   }
923 
924   Base = Base->IgnoreParens();
925 
926   if (const auto *CE = dyn_cast<CastExpr>(Base)) {
927     if (CE->getCastKind() == CK_ArrayToPointerDecay &&
928         !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(),
929                                                      StrictFlexArraysLevel)) {
930       CodeGenFunction::SanitizerScope SanScope(&CGF);
931 
932       IndexedType = CE->getSubExpr()->getType();
933       const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
934       if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
935         return CGF.Builder.getInt(CAT->getSize());
936 
937       if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
938         return CGF.getVLASize(VAT).NumElts;
939       // Ignore pass_object_size here. It's not applicable on decayed pointers.
940     }
941   }
942 
943   CodeGenFunction::SanitizerScope SanScope(&CGF);
944 
945   QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
946   if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {
947     IndexedType = Base->getType();
948     return POS;
949   }
950 
951   return nullptr;
952 }
953 
954 namespace {
955 
956 /// \p StructAccessBase returns the base \p Expr of a field access. It returns
957 /// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:
958 ///
959 ///     p in p-> a.b.c
960 ///
961 /// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're
962 /// looking for:
963 ///
964 ///     struct s {
965 ///       struct s *ptr;
966 ///       int count;
967 ///       char array[] __attribute__((counted_by(count)));
968 ///     };
969 ///
970 /// If we have an expression like \p p->ptr->array[index], we want the
971 /// \p MemberExpr for \p p->ptr instead of \p p.
972 class StructAccessBase
973     : public ConstStmtVisitor<StructAccessBase, const Expr *> {
974   const RecordDecl *ExpectedRD;
975 
976   bool IsExpectedRecordDecl(const Expr *E) const {
977     QualType Ty = E->getType();
978     if (Ty->isPointerType())
979       Ty = Ty->getPointeeType();
980     return ExpectedRD == Ty->getAsRecordDecl();
981   }
982 
983 public:
984   StructAccessBase(const RecordDecl *ExpectedRD) : ExpectedRD(ExpectedRD) {}
985 
986   //===--------------------------------------------------------------------===//
987   //                            Visitor Methods
988   //===--------------------------------------------------------------------===//
989 
990   // NOTE: If we build C++ support for counted_by, then we'll have to handle
991   // horrors like this:
992   //
993   //     struct S {
994   //       int x, y;
995   //       int blah[] __attribute__((counted_by(x)));
996   //     } s;
997   //
998   //     int foo(int index, int val) {
999   //       int (S::*IHatePMDs)[] = &S::blah;
1000   //       (s.*IHatePMDs)[index] = val;
1001   //     }
1002 
1003   const Expr *Visit(const Expr *E) {
1004     return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(E);
1005   }
1006 
1007   const Expr *VisitStmt(const Stmt *S) { return nullptr; }
1008 
1009   // These are the types we expect to return (in order of most to least
1010   // likely):
1011   //
1012   //   1. DeclRefExpr - This is the expression for the base of the structure.
1013   //      It's exactly what we want to build an access to the \p counted_by
1014   //      field.
1015   //   2. MemberExpr - This is the expression that has the same \p RecordDecl
1016   //      as the flexble array member's lexical enclosing \p RecordDecl. This
1017   //      allows us to catch things like: "p->p->array"
1018   //   3. CompoundLiteralExpr - This is for people who create something
1019   //      heretical like (struct foo has a flexible array member):
1020   //
1021   //        (struct foo){ 1, 2 }.blah[idx];
1022   const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {
1023     return IsExpectedRecordDecl(E) ? E : nullptr;
1024   }
1025   const Expr *VisitMemberExpr(const MemberExpr *E) {
1026     if (IsExpectedRecordDecl(E) && E->isArrow())
1027       return E;
1028     const Expr *Res = Visit(E->getBase());
1029     return !Res && IsExpectedRecordDecl(E) ? E : Res;
1030   }
1031   const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
1032     return IsExpectedRecordDecl(E) ? E : nullptr;
1033   }
1034   const Expr *VisitCallExpr(const CallExpr *E) {
1035     return IsExpectedRecordDecl(E) ? E : nullptr;
1036   }
1037 
1038   const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1039     if (IsExpectedRecordDecl(E))
1040       return E;
1041     return Visit(E->getBase());
1042   }
1043   const Expr *VisitCastExpr(const CastExpr *E) {
1044     return Visit(E->getSubExpr());
1045   }
1046   const Expr *VisitParenExpr(const ParenExpr *E) {
1047     return Visit(E->getSubExpr());
1048   }
1049   const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {
1050     return Visit(E->getSubExpr());
1051   }
1052   const Expr *VisitUnaryDeref(const UnaryOperator *E) {
1053     return Visit(E->getSubExpr());
1054   }
1055 };
1056 
1057 } // end anonymous namespace
1058 
1059 using RecIndicesTy =
1060     SmallVector<std::pair<const RecordDecl *, llvm::Value *>, 8>;
1061 
1062 static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,
1063                                  const FieldDecl *FD, RecIndicesTy &Indices) {
1064   const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);
1065   int64_t FieldNo = -1;
1066   for (const Decl *D : RD->decls()) {
1067     if (const auto *Field = dyn_cast<FieldDecl>(D)) {
1068       FieldNo = Layout.getLLVMFieldNo(Field);
1069       if (FD == Field) {
1070         Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
1071         return true;
1072       }
1073     }
1074 
1075     if (const auto *Record = dyn_cast<RecordDecl>(D)) {
1076       ++FieldNo;
1077       if (getGEPIndicesToField(CGF, Record, FD, Indices)) {
1078         if (RD->isUnion())
1079           FieldNo = 0;
1080         Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
1081         return true;
1082       }
1083     }
1084   }
1085 
1086   return false;
1087 }
1088 
1089 /// This method is typically called in contexts where we can't generate
1090 /// side-effects, like in __builtin_dynamic_object_size. When finding
1091 /// expressions, only choose those that have either already been emitted or can
1092 /// be loaded without side-effects.
1093 ///
1094 /// - \p FAMDecl: the \p Decl for the flexible array member. It may not be
1095 ///   within the top-level struct.
1096 /// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.
1097 llvm::Value *CodeGenFunction::EmitCountedByFieldExpr(
1098     const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
1099   const RecordDecl *RD = CountDecl->getParent()->getOuterLexicalRecordContext();
1100 
1101   // Find the base struct expr (i.e. p in p->a.b.c.d).
1102   const Expr *StructBase = StructAccessBase(RD).Visit(Base);
1103   if (!StructBase || StructBase->HasSideEffects(getContext()))
1104     return nullptr;
1105 
1106   llvm::Value *Res = nullptr;
1107   if (const auto *DRE = dyn_cast<DeclRefExpr>(StructBase)) {
1108     Res = EmitDeclRefLValue(DRE).getPointer(*this);
1109     Res = Builder.CreateAlignedLoad(ConvertType(DRE->getType()), Res,
1110                                     getPointerAlign(), "dre.load");
1111   } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(StructBase)) {
1112     LValue LV = EmitMemberExpr(ME);
1113     Address Addr = LV.getAddress(*this);
1114     Res = Addr.getPointer();
1115   } else if (StructBase->getType()->isPointerType()) {
1116     LValueBaseInfo BaseInfo;
1117     TBAAAccessInfo TBAAInfo;
1118     Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo);
1119     Res = Addr.getPointer();
1120   } else {
1121     return nullptr;
1122   }
1123 
1124   llvm::Value *Zero = Builder.getInt32(0);
1125   RecIndicesTy Indices;
1126 
1127   getGEPIndicesToField(*this, RD, CountDecl, Indices);
1128 
1129   for (auto I = Indices.rbegin(), E = Indices.rend(); I != E; ++I)
1130     Res = Builder.CreateInBoundsGEP(
1131         ConvertType(QualType(I->first->getTypeForDecl(), 0)), Res,
1132         {Zero, I->second}, "..counted_by.gep");
1133 
1134   return Builder.CreateAlignedLoad(ConvertType(CountDecl->getType()), Res,
1135                                    getIntAlign(), "..counted_by.load");
1136 }
1137 
1138 const FieldDecl *CodeGenFunction::FindCountedByField(const FieldDecl *FD) {
1139   if (!FD || !FD->hasAttr<CountedByAttr>())
1140     return nullptr;
1141 
1142   const auto *CBA = FD->getAttr<CountedByAttr>();
1143   if (!CBA)
1144     return nullptr;
1145 
1146   auto GetNonAnonStructOrUnion =
1147       [](const RecordDecl *RD) -> const RecordDecl * {
1148     while (RD && RD->isAnonymousStructOrUnion()) {
1149       const auto *R = dyn_cast<RecordDecl>(RD->getDeclContext());
1150       if (!R)
1151         return nullptr;
1152       RD = R;
1153     }
1154     return RD;
1155   };
1156   const RecordDecl *EnclosingRD = GetNonAnonStructOrUnion(FD->getParent());
1157   if (!EnclosingRD)
1158     return nullptr;
1159 
1160   DeclarationName DName(CBA->getCountedByField());
1161   DeclContext::lookup_result Lookup = EnclosingRD->lookup(DName);
1162 
1163   if (Lookup.empty())
1164     return nullptr;
1165 
1166   const NamedDecl *ND = Lookup.front();
1167   if (const auto *IFD = dyn_cast<IndirectFieldDecl>(ND))
1168     ND = IFD->getAnonField();
1169 
1170   return dyn_cast<FieldDecl>(ND);
1171 }
1172 
1173 void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
1174                                       llvm::Value *Index, QualType IndexType,
1175                                       bool Accessed) {
1176   assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
1177          "should not be called unless adding bounds checks");
1178   const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
1179       getLangOpts().getStrictFlexArraysLevel();
1180   QualType IndexedType;
1181   llvm::Value *Bound =
1182       getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel);
1183 
1184   EmitBoundsCheckImpl(E, Bound, Index, IndexType, IndexedType, Accessed);
1185 }
1186 
1187 void CodeGenFunction::EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
1188                                           llvm::Value *Index,
1189                                           QualType IndexType,
1190                                           QualType IndexedType, bool Accessed) {
1191   if (!Bound)
1192     return;
1193 
1194   SanitizerScope SanScope(this);
1195 
1196   bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
1197   llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
1198   llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);
1199 
1200   llvm::Constant *StaticData[] = {
1201     EmitCheckSourceLocation(E->getExprLoc()),
1202     EmitCheckTypeDescriptor(IndexedType),
1203     EmitCheckTypeDescriptor(IndexType)
1204   };
1205   llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal)
1206                                 : Builder.CreateICmpULE(IndexVal, BoundVal);
1207   EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds),
1208             SanitizerHandler::OutOfBounds, StaticData, Index);
1209 }
1210 
1211 CodeGenFunction::ComplexPairTy CodeGenFunction::
1212 EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
1213                          bool isInc, bool isPre) {
1214   ComplexPairTy InVal = EmitLoadOfComplex(LV, E->getExprLoc());
1215 
1216   llvm::Value *NextVal;
1217   if (isa<llvm::IntegerType>(InVal.first->getType())) {
1218     uint64_t AmountVal = isInc ? 1 : -1;
1219     NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
1220 
1221     // Add the inc/dec to the real part.
1222     NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1223   } else {
1224     QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1225     llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
1226     if (!isInc)
1227       FVal.changeSign();
1228     NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
1229 
1230     // Add the inc/dec to the real part.
1231     NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1232   }
1233 
1234   ComplexPairTy IncVal(NextVal, InVal.second);
1235 
1236   // Store the updated result through the lvalue.
1237   EmitStoreOfComplex(IncVal, LV, /*init*/ false);
1238   if (getLangOpts().OpenMP)
1239     CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
1240                                                               E->getSubExpr());
1241 
1242   // If this is a postinc, return the value read from memory, otherwise use the
1243   // updated value.
1244   return isPre ? IncVal : InVal;
1245 }
1246 
1247 void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1248                                              CodeGenFunction *CGF) {
1249   // Bind VLAs in the cast type.
1250   if (CGF && E->getType()->isVariablyModifiedType())
1251     CGF->EmitVariablyModifiedType(E->getType());
1252 
1253   if (CGDebugInfo *DI = getModuleDebugInfo())
1254     DI->EmitExplicitCastType(E->getType());
1255 }
1256 
1257 //===----------------------------------------------------------------------===//
1258 //                         LValue Expression Emission
1259 //===----------------------------------------------------------------------===//
1260 
1261 static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
1262                                         TBAAAccessInfo *TBAAInfo,
1263                                         KnownNonNull_t IsKnownNonNull,
1264                                         CodeGenFunction &CGF) {
1265   // We allow this with ObjC object pointers because of fragile ABIs.
1266   assert(E->getType()->isPointerType() ||
1267          E->getType()->isObjCObjectPointerType());
1268   E = E->IgnoreParens();
1269 
1270   // Casts:
1271   if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
1272     if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
1273       CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
1274 
1275     switch (CE->getCastKind()) {
1276     // Non-converting casts (but not C's implicit conversion from void*).
1277     case CK_BitCast:
1278     case CK_NoOp:
1279     case CK_AddressSpaceConversion:
1280       if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1281         if (PtrTy->getPointeeType()->isVoidType())
1282           break;
1283 
1284         LValueBaseInfo InnerBaseInfo;
1285         TBAAAccessInfo InnerTBAAInfo;
1286         Address Addr = CGF.EmitPointerWithAlignment(
1287             CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull);
1288         if (BaseInfo) *BaseInfo = InnerBaseInfo;
1289         if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1290 
1291         if (isa<ExplicitCastExpr>(CE)) {
1292           LValueBaseInfo TargetTypeBaseInfo;
1293           TBAAAccessInfo TargetTypeTBAAInfo;
1294           CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(
1295               E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);
1296           if (TBAAInfo)
1297             *TBAAInfo =
1298                 CGF.CGM.mergeTBAAInfoForCast(*TBAAInfo, TargetTypeTBAAInfo);
1299           // If the source l-value is opaque, honor the alignment of the
1300           // casted-to type.
1301           if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1302             if (BaseInfo)
1303               BaseInfo->mergeForCast(TargetTypeBaseInfo);
1304             Addr = Address(Addr.getPointer(), Addr.getElementType(), Align,
1305                            IsKnownNonNull);
1306           }
1307         }
1308 
1309         if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast) &&
1310             CE->getCastKind() == CK_BitCast) {
1311           if (auto PT = E->getType()->getAs<PointerType>())
1312             CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr,
1313                                           /*MayBeNull=*/true,
1314                                           CodeGenFunction::CFITCK_UnrelatedCast,
1315                                           CE->getBeginLoc());
1316         }
1317 
1318         llvm::Type *ElemTy =
1319             CGF.ConvertTypeForMem(E->getType()->getPointeeType());
1320         Addr = Addr.withElementType(ElemTy);
1321         if (CE->getCastKind() == CK_AddressSpaceConversion)
1322           Addr = CGF.Builder.CreateAddrSpaceCast(Addr,
1323                                                  CGF.ConvertType(E->getType()));
1324         return Addr;
1325       }
1326       break;
1327 
1328     // Array-to-pointer decay.
1329     case CK_ArrayToPointerDecay:
1330       return CGF.EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo);
1331 
1332     // Derived-to-base conversions.
1333     case CK_UncheckedDerivedToBase:
1334     case CK_DerivedToBase: {
1335       // TODO: Support accesses to members of base classes in TBAA. For now, we
1336       // conservatively pretend that the complete object is of the base class
1337       // type.
1338       if (TBAAInfo)
1339         *TBAAInfo = CGF.CGM.getTBAAAccessInfo(E->getType());
1340       Address Addr = CGF.EmitPointerWithAlignment(
1341           CE->getSubExpr(), BaseInfo, nullptr,
1342           (KnownNonNull_t)(IsKnownNonNull ||
1343                            CE->getCastKind() == CK_UncheckedDerivedToBase));
1344       auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1345       return CGF.GetAddressOfBaseClass(
1346           Addr, Derived, CE->path_begin(), CE->path_end(),
1347           CGF.ShouldNullCheckClassCastValue(CE), CE->getExprLoc());
1348     }
1349 
1350     // TODO: Is there any reason to treat base-to-derived conversions
1351     // specially?
1352     default:
1353       break;
1354     }
1355   }
1356 
1357   // Unary &.
1358   if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
1359     if (UO->getOpcode() == UO_AddrOf) {
1360       LValue LV = CGF.EmitLValue(UO->getSubExpr(), IsKnownNonNull);
1361       if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1362       if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1363       return LV.getAddress(CGF);
1364     }
1365   }
1366 
1367   // std::addressof and variants.
1368   if (auto *Call = dyn_cast<CallExpr>(E)) {
1369     switch (Call->getBuiltinCallee()) {
1370     default:
1371       break;
1372     case Builtin::BIaddressof:
1373     case Builtin::BI__addressof:
1374     case Builtin::BI__builtin_addressof: {
1375       LValue LV = CGF.EmitLValue(Call->getArg(0), IsKnownNonNull);
1376       if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1377       if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1378       return LV.getAddress(CGF);
1379     }
1380     }
1381   }
1382 
1383   // TODO: conditional operators, comma.
1384 
1385   // Otherwise, use the alignment of the type.
1386   CharUnits Align =
1387       CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
1388   llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType());
1389   return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull);
1390 }
1391 
1392 /// EmitPointerWithAlignment - Given an expression of pointer type, try to
1393 /// derive a more accurate bound on the alignment of the pointer.
1394 Address CodeGenFunction::EmitPointerWithAlignment(
1395     const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1396     KnownNonNull_t IsKnownNonNull) {
1397   Address Addr =
1398       ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, *this);
1399   if (IsKnownNonNull && !Addr.isKnownNonNull())
1400     Addr.setKnownNonNull();
1401   return Addr;
1402 }
1403 
1404 llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1405   llvm::Value *V = RV.getScalarVal();
1406   if (auto MPT = T->getAs<MemberPointerType>())
1407     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, V, MPT);
1408   return Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
1409 }
1410 
1411 RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1412   if (Ty->isVoidType())
1413     return RValue::get(nullptr);
1414 
1415   switch (getEvaluationKind(Ty)) {
1416   case TEK_Complex: {
1417     llvm::Type *EltTy =
1418       ConvertType(Ty->castAs<ComplexType>()->getElementType());
1419     llvm::Value *U = llvm::UndefValue::get(EltTy);
1420     return RValue::getComplex(std::make_pair(U, U));
1421   }
1422 
1423   // If this is a use of an undefined aggregate type, the aggregate must have an
1424   // identifiable address.  Just because the contents of the value are undefined
1425   // doesn't mean that the address can't be taken and compared.
1426   case TEK_Aggregate: {
1427     Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
1428     return RValue::getAggregate(DestPtr);
1429   }
1430 
1431   case TEK_Scalar:
1432     return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
1433   }
1434   llvm_unreachable("bad evaluation kind");
1435 }
1436 
1437 RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1438                                               const char *Name) {
1439   ErrorUnsupported(E, Name);
1440   return GetUndefRValue(E->getType());
1441 }
1442 
1443 LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1444                                               const char *Name) {
1445   ErrorUnsupported(E, Name);
1446   llvm::Type *ElTy = ConvertType(E->getType());
1447   llvm::Type *Ty = UnqualPtrTy;
1448   return MakeAddrLValue(
1449       Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType());
1450 }
1451 
1452 bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1453   const Expr *Base = Obj;
1454   while (!isa<CXXThisExpr>(Base)) {
1455     // The result of a dynamic_cast can be null.
1456     if (isa<CXXDynamicCastExpr>(Base))
1457       return false;
1458 
1459     if (const auto *CE = dyn_cast<CastExpr>(Base)) {
1460       Base = CE->getSubExpr();
1461     } else if (const auto *PE = dyn_cast<ParenExpr>(Base)) {
1462       Base = PE->getSubExpr();
1463     } else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) {
1464       if (UO->getOpcode() == UO_Extension)
1465         Base = UO->getSubExpr();
1466       else
1467         return false;
1468     } else {
1469       return false;
1470     }
1471   }
1472   return true;
1473 }
1474 
1475 LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1476   LValue LV;
1477   if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E))
1478     LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true);
1479   else
1480     LV = EmitLValue(E);
1481   if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) {
1482     SanitizerSet SkippedChecks;
1483     if (const auto *ME = dyn_cast<MemberExpr>(E)) {
1484       bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase());
1485       if (IsBaseCXXThis)
1486         SkippedChecks.set(SanitizerKind::Alignment, true);
1487       if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
1488         SkippedChecks.set(SanitizerKind::Null, true);
1489     }
1490     EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(),
1491                   LV.getAlignment(), SkippedChecks);
1492   }
1493   return LV;
1494 }
1495 
1496 /// EmitLValue - Emit code to compute a designator that specifies the location
1497 /// of the expression.
1498 ///
1499 /// This can return one of two things: a simple address or a bitfield reference.
1500 /// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1501 /// an LLVM pointer type.
1502 ///
1503 /// If this returns a bitfield reference, nothing about the pointee type of the
1504 /// LLVM value is known: For example, it may not be a pointer to an integer.
1505 ///
1506 /// If this returns a normal address, and if the lvalue's C type is fixed size,
1507 /// this method guarantees that the returned pointer type will point to an LLVM
1508 /// type of the same size of the lvalue's type.  If the lvalue has a variable
1509 /// length type, this is not possible.
1510 ///
1511 LValue CodeGenFunction::EmitLValue(const Expr *E,
1512                                    KnownNonNull_t IsKnownNonNull) {
1513   LValue LV = EmitLValueHelper(E, IsKnownNonNull);
1514   if (IsKnownNonNull && !LV.isKnownNonNull())
1515     LV.setKnownNonNull();
1516   return LV;
1517 }
1518 
1519 static QualType getConstantExprReferredType(const FullExpr *E,
1520                                             const ASTContext &Ctx) {
1521   const Expr *SE = E->getSubExpr()->IgnoreImplicit();
1522   if (isa<OpaqueValueExpr>(SE))
1523     return SE->getType();
1524   return cast<CallExpr>(SE)->getCallReturnType(Ctx)->getPointeeType();
1525 }
1526 
1527 LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1528                                          KnownNonNull_t IsKnownNonNull) {
1529   ApplyDebugLocation DL(*this, E);
1530   switch (E->getStmtClass()) {
1531   default: return EmitUnsupportedLValue(E, "l-value expression");
1532 
1533   case Expr::ObjCPropertyRefExprClass:
1534     llvm_unreachable("cannot emit a property reference directly");
1535 
1536   case Expr::ObjCSelectorExprClass:
1537     return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
1538   case Expr::ObjCIsaExprClass:
1539     return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
1540   case Expr::BinaryOperatorClass:
1541     return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
1542   case Expr::CompoundAssignOperatorClass: {
1543     QualType Ty = E->getType();
1544     if (const AtomicType *AT = Ty->getAs<AtomicType>())
1545       Ty = AT->getValueType();
1546     if (!Ty->isAnyComplexType())
1547       return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1548     return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1549   }
1550   case Expr::CallExprClass:
1551   case Expr::CXXMemberCallExprClass:
1552   case Expr::CXXOperatorCallExprClass:
1553   case Expr::UserDefinedLiteralClass:
1554     return EmitCallExprLValue(cast<CallExpr>(E));
1555   case Expr::CXXRewrittenBinaryOperatorClass:
1556     return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm(),
1557                       IsKnownNonNull);
1558   case Expr::VAArgExprClass:
1559     return EmitVAArgExprLValue(cast<VAArgExpr>(E));
1560   case Expr::DeclRefExprClass:
1561     return EmitDeclRefLValue(cast<DeclRefExpr>(E));
1562   case Expr::ConstantExprClass: {
1563     const ConstantExpr *CE = cast<ConstantExpr>(E);
1564     if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
1565       QualType RetType = getConstantExprReferredType(CE, getContext());
1566       return MakeNaturalAlignAddrLValue(Result, RetType);
1567     }
1568     return EmitLValue(cast<ConstantExpr>(E)->getSubExpr(), IsKnownNonNull);
1569   }
1570   case Expr::ParenExprClass:
1571     return EmitLValue(cast<ParenExpr>(E)->getSubExpr(), IsKnownNonNull);
1572   case Expr::GenericSelectionExprClass:
1573     return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr(),
1574                       IsKnownNonNull);
1575   case Expr::PredefinedExprClass:
1576     return EmitPredefinedLValue(cast<PredefinedExpr>(E));
1577   case Expr::StringLiteralClass:
1578     return EmitStringLiteralLValue(cast<StringLiteral>(E));
1579   case Expr::ObjCEncodeExprClass:
1580     return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
1581   case Expr::PseudoObjectExprClass:
1582     return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
1583   case Expr::InitListExprClass:
1584     return EmitInitListLValue(cast<InitListExpr>(E));
1585   case Expr::CXXTemporaryObjectExprClass:
1586   case Expr::CXXConstructExprClass:
1587     return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
1588   case Expr::CXXBindTemporaryExprClass:
1589     return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
1590   case Expr::CXXUuidofExprClass:
1591     return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));
1592   case Expr::LambdaExprClass:
1593     return EmitAggExprToLValue(E);
1594 
1595   case Expr::ExprWithCleanupsClass: {
1596     const auto *cleanups = cast<ExprWithCleanups>(E);
1597     RunCleanupsScope Scope(*this);
1598     LValue LV = EmitLValue(cleanups->getSubExpr(), IsKnownNonNull);
1599     if (LV.isSimple()) {
1600       // Defend against branches out of gnu statement expressions surrounded by
1601       // cleanups.
1602       Address Addr = LV.getAddress(*this);
1603       llvm::Value *V = Addr.getPointer();
1604       Scope.ForceCleanup({&V});
1605       return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()),
1606                               LV.getType(), getContext(), LV.getBaseInfo(),
1607                               LV.getTBAAInfo());
1608     }
1609     // FIXME: Is it possible to create an ExprWithCleanups that produces a
1610     // bitfield lvalue or some other non-simple lvalue?
1611     return LV;
1612   }
1613 
1614   case Expr::CXXDefaultArgExprClass: {
1615     auto *DAE = cast<CXXDefaultArgExpr>(E);
1616     CXXDefaultArgExprScope Scope(*this, DAE);
1617     return EmitLValue(DAE->getExpr(), IsKnownNonNull);
1618   }
1619   case Expr::CXXDefaultInitExprClass: {
1620     auto *DIE = cast<CXXDefaultInitExpr>(E);
1621     CXXDefaultInitExprScope Scope(*this, DIE);
1622     return EmitLValue(DIE->getExpr(), IsKnownNonNull);
1623   }
1624   case Expr::CXXTypeidExprClass:
1625     return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
1626 
1627   case Expr::ObjCMessageExprClass:
1628     return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
1629   case Expr::ObjCIvarRefExprClass:
1630     return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
1631   case Expr::StmtExprClass:
1632     return EmitStmtExprLValue(cast<StmtExpr>(E));
1633   case Expr::UnaryOperatorClass:
1634     return EmitUnaryOpLValue(cast<UnaryOperator>(E));
1635   case Expr::ArraySubscriptExprClass:
1636     return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
1637   case Expr::MatrixSubscriptExprClass:
1638     return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E));
1639   case Expr::OMPArraySectionExprClass:
1640     return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E));
1641   case Expr::ExtVectorElementExprClass:
1642     return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
1643   case Expr::CXXThisExprClass:
1644     return MakeAddrLValue(LoadCXXThisAddress(), E->getType());
1645   case Expr::MemberExprClass:
1646     return EmitMemberExpr(cast<MemberExpr>(E));
1647   case Expr::CompoundLiteralExprClass:
1648     return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
1649   case Expr::ConditionalOperatorClass:
1650     return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
1651   case Expr::BinaryConditionalOperatorClass:
1652     return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
1653   case Expr::ChooseExprClass:
1654     return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(), IsKnownNonNull);
1655   case Expr::OpaqueValueExprClass:
1656     return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
1657   case Expr::SubstNonTypeTemplateParmExprClass:
1658     return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(),
1659                       IsKnownNonNull);
1660   case Expr::ImplicitCastExprClass:
1661   case Expr::CStyleCastExprClass:
1662   case Expr::CXXFunctionalCastExprClass:
1663   case Expr::CXXStaticCastExprClass:
1664   case Expr::CXXDynamicCastExprClass:
1665   case Expr::CXXReinterpretCastExprClass:
1666   case Expr::CXXConstCastExprClass:
1667   case Expr::CXXAddrspaceCastExprClass:
1668   case Expr::ObjCBridgedCastExprClass:
1669     return EmitCastLValue(cast<CastExpr>(E));
1670 
1671   case Expr::MaterializeTemporaryExprClass:
1672     return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
1673 
1674   case Expr::CoawaitExprClass:
1675     return EmitCoawaitLValue(cast<CoawaitExpr>(E));
1676   case Expr::CoyieldExprClass:
1677     return EmitCoyieldLValue(cast<CoyieldExpr>(E));
1678   }
1679 }
1680 
1681 /// Given an object of the given canonical type, can we safely copy a
1682 /// value out of it based on its initializer?
1683 static bool isConstantEmittableObjectType(QualType type) {
1684   assert(type.isCanonical());
1685   assert(!type->isReferenceType());
1686 
1687   // Must be const-qualified but non-volatile.
1688   Qualifiers qs = type.getLocalQualifiers();
1689   if (!qs.hasConst() || qs.hasVolatile()) return false;
1690 
1691   // Otherwise, all object types satisfy this except C++ classes with
1692   // mutable subobjects or non-trivial copy/destroy behavior.
1693   if (const auto *RT = dyn_cast<RecordType>(type))
1694     if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1695       if (RD->hasMutableFields() || !RD->isTrivial())
1696         return false;
1697 
1698   return true;
1699 }
1700 
1701 /// Can we constant-emit a load of a reference to a variable of the
1702 /// given type?  This is different from predicates like
1703 /// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1704 /// in situations that don't necessarily satisfy the language's rules
1705 /// for this (e.g. C++'s ODR-use rules).  For example, we want to able
1706 /// to do this with const float variables even if those variables
1707 /// aren't marked 'constexpr'.
1708 enum ConstantEmissionKind {
1709   CEK_None,
1710   CEK_AsReferenceOnly,
1711   CEK_AsValueOrReference,
1712   CEK_AsValueOnly
1713 };
1714 static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1715   type = type.getCanonicalType();
1716   if (const auto *ref = dyn_cast<ReferenceType>(type)) {
1717     if (isConstantEmittableObjectType(ref->getPointeeType()))
1718       return CEK_AsValueOrReference;
1719     return CEK_AsReferenceOnly;
1720   }
1721   if (isConstantEmittableObjectType(type))
1722     return CEK_AsValueOnly;
1723   return CEK_None;
1724 }
1725 
1726 /// Try to emit a reference to the given value without producing it as
1727 /// an l-value.  This is just an optimization, but it avoids us needing
1728 /// to emit global copies of variables if they're named without triggering
1729 /// a formal use in a context where we can't emit a direct reference to them,
1730 /// for instance if a block or lambda or a member of a local class uses a
1731 /// const int variable or constexpr variable from an enclosing function.
1732 CodeGenFunction::ConstantEmission
1733 CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
1734   ValueDecl *value = refExpr->getDecl();
1735 
1736   // The value needs to be an enum constant or a constant variable.
1737   ConstantEmissionKind CEK;
1738   if (isa<ParmVarDecl>(value)) {
1739     CEK = CEK_None;
1740   } else if (auto *var = dyn_cast<VarDecl>(value)) {
1741     CEK = checkVarTypeForConstantEmission(var->getType());
1742   } else if (isa<EnumConstantDecl>(value)) {
1743     CEK = CEK_AsValueOnly;
1744   } else {
1745     CEK = CEK_None;
1746   }
1747   if (CEK == CEK_None) return ConstantEmission();
1748 
1749   Expr::EvalResult result;
1750   bool resultIsReference;
1751   QualType resultType;
1752 
1753   // It's best to evaluate all the way as an r-value if that's permitted.
1754   if (CEK != CEK_AsReferenceOnly &&
1755       refExpr->EvaluateAsRValue(result, getContext())) {
1756     resultIsReference = false;
1757     resultType = refExpr->getType();
1758 
1759   // Otherwise, try to evaluate as an l-value.
1760   } else if (CEK != CEK_AsValueOnly &&
1761              refExpr->EvaluateAsLValue(result, getContext())) {
1762     resultIsReference = true;
1763     resultType = value->getType();
1764 
1765   // Failure.
1766   } else {
1767     return ConstantEmission();
1768   }
1769 
1770   // In any case, if the initializer has side-effects, abandon ship.
1771   if (result.HasSideEffects)
1772     return ConstantEmission();
1773 
1774   // In CUDA/HIP device compilation, a lambda may capture a reference variable
1775   // referencing a global host variable by copy. In this case the lambda should
1776   // make a copy of the value of the global host variable. The DRE of the
1777   // captured reference variable cannot be emitted as load from the host
1778   // global variable as compile time constant, since the host variable is not
1779   // accessible on device. The DRE of the captured reference variable has to be
1780   // loaded from captures.
1781   if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
1782       refExpr->refersToEnclosingVariableOrCapture()) {
1783     auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
1784     if (MD && MD->getParent()->isLambda() &&
1785         MD->getOverloadedOperator() == OO_Call) {
1786       const APValue::LValueBase &base = result.Val.getLValueBase();
1787       if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
1788         if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
1789           if (!VD->hasAttr<CUDADeviceAttr>()) {
1790             return ConstantEmission();
1791           }
1792         }
1793       }
1794     }
1795   }
1796 
1797   // Emit as a constant.
1798   auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(),
1799                                                result.Val, resultType);
1800 
1801   // Make sure we emit a debug reference to the global variable.
1802   // This should probably fire even for
1803   if (isa<VarDecl>(value)) {
1804     if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
1805       EmitDeclRefExprDbgValue(refExpr, result.Val);
1806   } else {
1807     assert(isa<EnumConstantDecl>(value));
1808     EmitDeclRefExprDbgValue(refExpr, result.Val);
1809   }
1810 
1811   // If we emitted a reference constant, we need to dereference that.
1812   if (resultIsReference)
1813     return ConstantEmission::forReference(C);
1814 
1815   return ConstantEmission::forValue(C);
1816 }
1817 
1818 static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
1819                                                         const MemberExpr *ME) {
1820   if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) {
1821     // Try to emit static variable member expressions as DREs.
1822     return DeclRefExpr::Create(
1823         CGF.getContext(), NestedNameSpecifierLoc(), SourceLocation(), VD,
1824         /*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(),
1825         ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse());
1826   }
1827   return nullptr;
1828 }
1829 
1830 CodeGenFunction::ConstantEmission
1831 CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
1832   if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, ME))
1833     return tryEmitAsConstant(DRE);
1834   return ConstantEmission();
1835 }
1836 
1837 llvm::Value *CodeGenFunction::emitScalarConstant(
1838     const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
1839   assert(Constant && "not a constant");
1840   if (Constant.isReference())
1841     return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E),
1842                             E->getExprLoc())
1843         .getScalarVal();
1844   return Constant.getValue();
1845 }
1846 
1847 llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
1848                                                SourceLocation Loc) {
1849   return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(),
1850                           lvalue.getType(), Loc, lvalue.getBaseInfo(),
1851                           lvalue.getTBAAInfo(), lvalue.isNontemporal());
1852 }
1853 
1854 static bool hasBooleanRepresentation(QualType Ty) {
1855   if (Ty->isBooleanType())
1856     return true;
1857 
1858   if (const EnumType *ET = Ty->getAs<EnumType>())
1859     return ET->getDecl()->getIntegerType()->isBooleanType();
1860 
1861   if (const AtomicType *AT = Ty->getAs<AtomicType>())
1862     return hasBooleanRepresentation(AT->getValueType());
1863 
1864   return false;
1865 }
1866 
1867 static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
1868                             llvm::APInt &Min, llvm::APInt &End,
1869                             bool StrictEnums, bool IsBool) {
1870   const EnumType *ET = Ty->getAs<EnumType>();
1871   bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
1872                                 ET && !ET->getDecl()->isFixed();
1873   if (!IsBool && !IsRegularCPlusPlusEnum)
1874     return false;
1875 
1876   if (IsBool) {
1877     Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);
1878     End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);
1879   } else {
1880     const EnumDecl *ED = ET->getDecl();
1881     ED->getValueRange(End, Min);
1882   }
1883   return true;
1884 }
1885 
1886 llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
1887   llvm::APInt Min, End;
1888   if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums,
1889                        hasBooleanRepresentation(Ty)))
1890     return nullptr;
1891 
1892   llvm::MDBuilder MDHelper(getLLVMContext());
1893   return MDHelper.createRange(Min, End);
1894 }
1895 
1896 bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
1897                                            SourceLocation Loc) {
1898   bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool);
1899   bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum);
1900   if (!HasBoolCheck && !HasEnumCheck)
1901     return false;
1902 
1903   bool IsBool = hasBooleanRepresentation(Ty) ||
1904                 NSAPI(CGM.getContext()).isObjCBOOLType(Ty);
1905   bool NeedsBoolCheck = HasBoolCheck && IsBool;
1906   bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>();
1907   if (!NeedsBoolCheck && !NeedsEnumCheck)
1908     return false;
1909 
1910   // Single-bit booleans don't need to be checked. Special-case this to avoid
1911   // a bit width mismatch when handling bitfield values. This is handled by
1912   // EmitFromMemory for the non-bitfield case.
1913   if (IsBool &&
1914       cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1)
1915     return false;
1916 
1917   llvm::APInt Min, End;
1918   if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
1919     return true;
1920 
1921   auto &Ctx = getLLVMContext();
1922   SanitizerScope SanScope(this);
1923   llvm::Value *Check;
1924   --End;
1925   if (!Min) {
1926     Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End));
1927   } else {
1928     llvm::Value *Upper =
1929         Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End));
1930     llvm::Value *Lower =
1931         Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min));
1932     Check = Builder.CreateAnd(Upper, Lower);
1933   }
1934   llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
1935                                   EmitCheckTypeDescriptor(Ty)};
1936   SanitizerMask Kind =
1937       NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1938   EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
1939             StaticArgs, EmitCheckValue(Value));
1940   return true;
1941 }
1942 
1943 llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
1944                                                QualType Ty,
1945                                                SourceLocation Loc,
1946                                                LValueBaseInfo BaseInfo,
1947                                                TBAAAccessInfo TBAAInfo,
1948                                                bool isNontemporal) {
1949   if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1950     if (GV->isThreadLocal())
1951       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
1952                               NotKnownNonNull);
1953 
1954   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1955     // Boolean vectors use `iN` as storage type.
1956     if (ClangVecTy->isExtVectorBoolType()) {
1957       llvm::Type *ValTy = ConvertType(Ty);
1958       unsigned ValNumElems =
1959           cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1960       // Load the `iP` storage object (P is the padded vector size).
1961       auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits");
1962       const auto *RawIntTy = RawIntV->getType();
1963       assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
1964       // Bitcast iP --> <P x i1>.
1965       auto *PaddedVecTy = llvm::FixedVectorType::get(
1966           Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1967       llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy);
1968       // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1969       V = emitBoolVecConversion(V, ValNumElems, "extractvec");
1970 
1971       return EmitFromMemory(V, Ty);
1972     }
1973 
1974     // Handle vectors of size 3 like size 4 for better performance.
1975     const llvm::Type *EltTy = Addr.getElementType();
1976     const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
1977 
1978     if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) {
1979 
1980       llvm::VectorType *vec4Ty =
1981           llvm::FixedVectorType::get(VTy->getElementType(), 4);
1982       Address Cast = Addr.withElementType(vec4Ty);
1983       // Now load value.
1984       llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
1985 
1986       // Shuffle vector to get vec3.
1987       V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec");
1988       return EmitFromMemory(V, Ty);
1989     }
1990   }
1991 
1992   // Atomic operations have to be done on integral types.
1993   LValue AtomicLValue =
1994       LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1995   if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) {
1996     return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();
1997   }
1998 
1999   llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
2000   if (isNontemporal) {
2001     llvm::MDNode *Node = llvm::MDNode::get(
2002         Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
2003     Load->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);
2004   }
2005 
2006   CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
2007 
2008   if (EmitScalarRangeCheck(Load, Ty, Loc)) {
2009     // In order to prevent the optimizer from throwing away the check, don't
2010     // attach range metadata to the load.
2011   } else if (CGM.getCodeGenOpts().OptimizationLevel > 0)
2012     if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
2013       Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
2014       Load->setMetadata(llvm::LLVMContext::MD_noundef,
2015                         llvm::MDNode::get(getLLVMContext(), std::nullopt));
2016     }
2017 
2018   return EmitFromMemory(Load, Ty);
2019 }
2020 
2021 llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
2022   // Bool has a different representation in memory than in registers.
2023   if (hasBooleanRepresentation(Ty)) {
2024     // This should really always be an i1, but sometimes it's already
2025     // an i8, and it's awkward to track those cases down.
2026     if (Value->getType()->isIntegerTy(1))
2027       return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool");
2028     assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
2029            "wrong value rep of bool");
2030   }
2031 
2032   return Value;
2033 }
2034 
2035 llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
2036   // Bool has a different representation in memory than in registers.
2037   if (hasBooleanRepresentation(Ty)) {
2038     assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
2039            "wrong value rep of bool");
2040     return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
2041   }
2042   if (Ty->isExtVectorBoolType()) {
2043     const auto *RawIntTy = Value->getType();
2044     // Bitcast iP --> <P x i1>.
2045     auto *PaddedVecTy = llvm::FixedVectorType::get(
2046         Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
2047     auto *V = Builder.CreateBitCast(Value, PaddedVecTy);
2048     // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
2049     llvm::Type *ValTy = ConvertType(Ty);
2050     unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
2051     return emitBoolVecConversion(V, ValNumElems, "extractvec");
2052   }
2053 
2054   return Value;
2055 }
2056 
2057 // Convert the pointer of \p Addr to a pointer to a vector (the value type of
2058 // MatrixType), if it points to a array (the memory type of MatrixType).
2059 static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF,
2060                                          bool IsVector = true) {
2061   auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
2062   if (ArrayTy && IsVector) {
2063     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
2064                                                 ArrayTy->getNumElements());
2065 
2066     return Addr.withElementType(VectorTy);
2067   }
2068   auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType());
2069   if (VectorTy && !IsVector) {
2070     auto *ArrayTy = llvm::ArrayType::get(
2071         VectorTy->getElementType(),
2072         cast<llvm::FixedVectorType>(VectorTy)->getNumElements());
2073 
2074     return Addr.withElementType(ArrayTy);
2075   }
2076 
2077   return Addr;
2078 }
2079 
2080 // Emit a store of a matrix LValue. This may require casting the original
2081 // pointer to memory address (ArrayType) to a pointer to the value type
2082 // (VectorType).
2083 static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
2084                                     bool isInit, CodeGenFunction &CGF) {
2085   Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF,
2086                                            value->getType()->isVectorTy());
2087   CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(),
2088                         lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit,
2089                         lvalue.isNontemporal());
2090 }
2091 
2092 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
2093                                         bool Volatile, QualType Ty,
2094                                         LValueBaseInfo BaseInfo,
2095                                         TBAAAccessInfo TBAAInfo,
2096                                         bool isInit, bool isNontemporal) {
2097   if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
2098     if (GV->isThreadLocal())
2099       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
2100                               NotKnownNonNull);
2101 
2102   llvm::Type *SrcTy = Value->getType();
2103   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
2104     auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
2105     if (VecTy && ClangVecTy->isExtVectorBoolType()) {
2106       auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType());
2107       // Expand to the memory bit width.
2108       unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
2109       // <N x i1> --> <P x i1>.
2110       Value = emitBoolVecConversion(Value, MemNumElems, "insertvec");
2111       // <P x i1> --> iP.
2112       Value = Builder.CreateBitCast(Value, MemIntTy);
2113     } else if (!CGM.getCodeGenOpts().PreserveVec3Type) {
2114       // Handle vec3 special.
2115       if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
2116         // Our source is a vec3, do a shuffle vector to make it a vec4.
2117         Value = Builder.CreateShuffleVector(Value, ArrayRef<int>{0, 1, 2, -1},
2118                                             "extractVec");
2119         SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4);
2120       }
2121       if (Addr.getElementType() != SrcTy) {
2122         Addr = Addr.withElementType(SrcTy);
2123       }
2124     }
2125   }
2126 
2127   Value = EmitToMemory(Value, Ty);
2128 
2129   LValue AtomicLValue =
2130       LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
2131   if (Ty->isAtomicType() ||
2132       (!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) {
2133     EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit);
2134     return;
2135   }
2136 
2137   llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
2138   if (isNontemporal) {
2139     llvm::MDNode *Node =
2140         llvm::MDNode::get(Store->getContext(),
2141                           llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
2142     Store->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);
2143   }
2144 
2145   CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2146 }
2147 
2148 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
2149                                         bool isInit) {
2150   if (lvalue.getType()->isConstantMatrixType()) {
2151     EmitStoreOfMatrixScalar(value, lvalue, isInit, *this);
2152     return;
2153   }
2154 
2155   EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(),
2156                     lvalue.getType(), lvalue.getBaseInfo(),
2157                     lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal());
2158 }
2159 
2160 // Emit a load of a LValue of matrix type. This may require casting the pointer
2161 // to memory address (ArrayType) to a pointer to the value type (VectorType).
2162 static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
2163                                      CodeGenFunction &CGF) {
2164   assert(LV.getType()->isConstantMatrixType());
2165   Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF);
2166   LV.setAddress(Addr);
2167   return RValue::get(CGF.EmitLoadOfScalar(LV, Loc));
2168 }
2169 
2170 /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
2171 /// method emits the address of the lvalue, then loads the result as an rvalue,
2172 /// returning the rvalue.
2173 RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
2174   if (LV.isObjCWeak()) {
2175     // load of a __weak object.
2176     Address AddrWeakObj = LV.getAddress(*this);
2177     return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
2178                                                              AddrWeakObj));
2179   }
2180   if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
2181     // In MRC mode, we do a load+autorelease.
2182     if (!getLangOpts().ObjCAutoRefCount) {
2183       return RValue::get(EmitARCLoadWeak(LV.getAddress(*this)));
2184     }
2185 
2186     // In ARC mode, we load retained and then consume the value.
2187     llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this));
2188     Object = EmitObjCConsumeObject(LV.getType(), Object);
2189     return RValue::get(Object);
2190   }
2191 
2192   if (LV.isSimple()) {
2193     assert(!LV.getType()->isFunctionType());
2194 
2195     if (LV.getType()->isConstantMatrixType())
2196       return EmitLoadOfMatrixLValue(LV, Loc, *this);
2197 
2198     // Everything needs a load.
2199     return RValue::get(EmitLoadOfScalar(LV, Loc));
2200   }
2201 
2202   if (LV.isVectorElt()) {
2203     llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(),
2204                                               LV.isVolatileQualified());
2205     return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
2206                                                     "vecext"));
2207   }
2208 
2209   // If this is a reference to a subset of the elements of a vector, either
2210   // shuffle the input or extract/insert them as appropriate.
2211   if (LV.isExtVectorElt()) {
2212     return EmitLoadOfExtVectorElementLValue(LV);
2213   }
2214 
2215   // Global Register variables always invoke intrinsics
2216   if (LV.isGlobalReg())
2217     return EmitLoadOfGlobalRegLValue(LV);
2218 
2219   if (LV.isMatrixElt()) {
2220     llvm::Value *Idx = LV.getMatrixIdx();
2221     if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2222       const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();
2223       llvm::MatrixBuilder MB(Builder);
2224       MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2225     }
2226     llvm::LoadInst *Load =
2227         Builder.CreateLoad(LV.getMatrixAddress(), LV.isVolatileQualified());
2228     return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext"));
2229   }
2230 
2231   assert(LV.isBitField() && "Unknown LValue type!");
2232   return EmitLoadOfBitfieldLValue(LV, Loc);
2233 }
2234 
2235 RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2236                                                  SourceLocation Loc) {
2237   const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2238 
2239   // Get the output type.
2240   llvm::Type *ResLTy = ConvertType(LV.getType());
2241 
2242   Address Ptr = LV.getBitFieldAddress();
2243   llvm::Value *Val =
2244       Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load");
2245 
2246   bool UseVolatile = LV.isVolatileQualified() &&
2247                      Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2248   const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2249   const unsigned StorageSize =
2250       UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2251   if (Info.IsSigned) {
2252     assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2253     unsigned HighBits = StorageSize - Offset - Info.Size;
2254     if (HighBits)
2255       Val = Builder.CreateShl(Val, HighBits, "bf.shl");
2256     if (Offset + HighBits)
2257       Val = Builder.CreateAShr(Val, Offset + HighBits, "bf.ashr");
2258   } else {
2259     if (Offset)
2260       Val = Builder.CreateLShr(Val, Offset, "bf.lshr");
2261     if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2262       Val = Builder.CreateAnd(
2263           Val, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), "bf.clear");
2264   }
2265   Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast");
2266   EmitScalarRangeCheck(Val, LV.getType(), Loc);
2267   return RValue::get(Val);
2268 }
2269 
2270 // If this is a reference to a subset of the elements of a vector, create an
2271 // appropriate shufflevector.
2272 RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2273   llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(),
2274                                         LV.isVolatileQualified());
2275 
2276   // HLSL allows treating scalars as one-element vectors. Converting the scalar
2277   // IR value to a vector here allows the rest of codegen to behave as normal.
2278   if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) {
2279     llvm::Type *DstTy = llvm::FixedVectorType::get(Vec->getType(), 1);
2280     llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
2281     Vec = Builder.CreateInsertElement(DstTy, Vec, Zero, "cast.splat");
2282   }
2283 
2284   const llvm::Constant *Elts = LV.getExtVectorElts();
2285 
2286   // If the result of the expression is a non-vector type, we must be extracting
2287   // a single element.  Just codegen as an extractelement.
2288   const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2289   if (!ExprVT) {
2290     unsigned InIdx = getAccessedFieldNo(0, Elts);
2291     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2292     return RValue::get(Builder.CreateExtractElement(Vec, Elt));
2293   }
2294 
2295   // Always use shuffle vector to try to retain the original program structure
2296   unsigned NumResultElts = ExprVT->getNumElements();
2297 
2298   SmallVector<int, 4> Mask;
2299   for (unsigned i = 0; i != NumResultElts; ++i)
2300     Mask.push_back(getAccessedFieldNo(i, Elts));
2301 
2302   Vec = Builder.CreateShuffleVector(Vec, Mask);
2303   return RValue::get(Vec);
2304 }
2305 
2306 /// Generates lvalue for partial ext_vector access.
2307 Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2308   Address VectorAddress = LV.getExtVectorAddress();
2309   QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2310   llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT);
2311 
2312   Address CastToPointerElement = VectorAddress.withElementType(VectorElementTy);
2313 
2314   const llvm::Constant *Elts = LV.getExtVectorElts();
2315   unsigned ix = getAccessedFieldNo(0, Elts);
2316 
2317   Address VectorBasePtrPlusIx =
2318     Builder.CreateConstInBoundsGEP(CastToPointerElement, ix,
2319                                    "vector.elt");
2320 
2321   return VectorBasePtrPlusIx;
2322 }
2323 
2324 /// Load of global gamed gegisters are always calls to intrinsics.
2325 RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2326   assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2327          "Bad type for register variable");
2328   llvm::MDNode *RegName = cast<llvm::MDNode>(
2329       cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata());
2330 
2331   // We accept integer and pointer types only
2332   llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType());
2333   llvm::Type *Ty = OrigTy;
2334   if (OrigTy->isPointerTy())
2335     Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2336   llvm::Type *Types[] = { Ty };
2337 
2338   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
2339   llvm::Value *Call = Builder.CreateCall(
2340       F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));
2341   if (OrigTy->isPointerTy())
2342     Call = Builder.CreateIntToPtr(Call, OrigTy);
2343   return RValue::get(Call);
2344 }
2345 
2346 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
2347 /// lvalue, where both are guaranteed to the have the same type, and that type
2348 /// is 'Ty'.
2349 void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2350                                              bool isInit) {
2351   if (!Dst.isSimple()) {
2352     if (Dst.isVectorElt()) {
2353       // Read/modify/write the vector, inserting the new element.
2354       llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
2355                                             Dst.isVolatileQualified());
2356       auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
2357       if (IRStoreTy) {
2358         auto *IRVecTy = llvm::FixedVectorType::get(
2359             Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
2360         Vec = Builder.CreateBitCast(Vec, IRVecTy);
2361         // iN --> <N x i1>.
2362       }
2363       Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
2364                                         Dst.getVectorIdx(), "vecins");
2365       if (IRStoreTy) {
2366         // <N x i1> --> <iN>.
2367         Vec = Builder.CreateBitCast(Vec, IRStoreTy);
2368       }
2369       Builder.CreateStore(Vec, Dst.getVectorAddress(),
2370                           Dst.isVolatileQualified());
2371       return;
2372     }
2373 
2374     // If this is an update of extended vector elements, insert them as
2375     // appropriate.
2376     if (Dst.isExtVectorElt())
2377       return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2378 
2379     if (Dst.isGlobalReg())
2380       return EmitStoreThroughGlobalRegLValue(Src, Dst);
2381 
2382     if (Dst.isMatrixElt()) {
2383       llvm::Value *Idx = Dst.getMatrixIdx();
2384       if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2385         const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2386         llvm::MatrixBuilder MB(Builder);
2387         MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2388       }
2389       llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress());
2390       llvm::Value *Vec =
2391           Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins");
2392       Builder.CreateStore(Vec, Dst.getMatrixAddress(),
2393                           Dst.isVolatileQualified());
2394       return;
2395     }
2396 
2397     assert(Dst.isBitField() && "Unknown LValue type");
2398     return EmitStoreThroughBitfieldLValue(Src, Dst);
2399   }
2400 
2401   // There's special magic for assigning into an ARC-qualified l-value.
2402   if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2403     switch (Lifetime) {
2404     case Qualifiers::OCL_None:
2405       llvm_unreachable("present but none");
2406 
2407     case Qualifiers::OCL_ExplicitNone:
2408       // nothing special
2409       break;
2410 
2411     case Qualifiers::OCL_Strong:
2412       if (isInit) {
2413         Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal()));
2414         break;
2415       }
2416       EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
2417       return;
2418 
2419     case Qualifiers::OCL_Weak:
2420       if (isInit)
2421         // Initialize and then skip the primitive store.
2422         EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal());
2423       else
2424         EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(),
2425                          /*ignore*/ true);
2426       return;
2427 
2428     case Qualifiers::OCL_Autoreleasing:
2429       Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),
2430                                                      Src.getScalarVal()));
2431       // fall into the normal path
2432       break;
2433     }
2434   }
2435 
2436   if (Dst.isObjCWeak() && !Dst.isNonGC()) {
2437     // load of a __weak object.
2438     Address LvalueDst = Dst.getAddress(*this);
2439     llvm::Value *src = Src.getScalarVal();
2440      CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
2441     return;
2442   }
2443 
2444   if (Dst.isObjCStrong() && !Dst.isNonGC()) {
2445     // load of a __strong object.
2446     Address LvalueDst = Dst.getAddress(*this);
2447     llvm::Value *src = Src.getScalarVal();
2448     if (Dst.isObjCIvar()) {
2449       assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
2450       llvm::Type *ResultType = IntPtrTy;
2451       Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp());
2452       llvm::Value *RHS = dst.getPointer();
2453       RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
2454       llvm::Value *LHS =
2455         Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
2456                                "sub.ptr.lhs.cast");
2457       llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
2458       CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
2459                                               BytesBetween);
2460     } else if (Dst.isGlobalObjCRef()) {
2461       CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
2462                                                 Dst.isThreadLocalRef());
2463     }
2464     else
2465       CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
2466     return;
2467   }
2468 
2469   assert(Src.isScalar() && "Can't emit an agg store with this method");
2470   EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
2471 }
2472 
2473 void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
2474                                                      llvm::Value **Result) {
2475   const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
2476   llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
2477   Address Ptr = Dst.getBitFieldAddress();
2478 
2479   // Get the source value, truncated to the width of the bit-field.
2480   llvm::Value *SrcVal = Src.getScalarVal();
2481 
2482   // Cast the source to the storage type and shift it into place.
2483   SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(),
2484                                  /*isSigned=*/false);
2485   llvm::Value *MaskedVal = SrcVal;
2486 
2487   const bool UseVolatile =
2488       CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
2489       Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2490   const unsigned StorageSize =
2491       UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2492   const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2493   // See if there are other bits in the bitfield's storage we'll need to load
2494   // and mask together with source before storing.
2495   if (StorageSize != Info.Size) {
2496     assert(StorageSize > Info.Size && "Invalid bitfield size.");
2497     llvm::Value *Val =
2498         Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load");
2499 
2500     // Mask the source value as needed.
2501     if (!hasBooleanRepresentation(Dst.getType()))
2502       SrcVal = Builder.CreateAnd(
2503           SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.Size),
2504           "bf.value");
2505     MaskedVal = SrcVal;
2506     if (Offset)
2507       SrcVal = Builder.CreateShl(SrcVal, Offset, "bf.shl");
2508 
2509     // Mask out the original value.
2510     Val = Builder.CreateAnd(
2511         Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.Size),
2512         "bf.clear");
2513 
2514     // Or together the unchanged values and the source value.
2515     SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set");
2516   } else {
2517     assert(Offset == 0);
2518     // According to the AACPS:
2519     // When a volatile bit-field is written, and its container does not overlap
2520     // with any non-bit-field member, its container must be read exactly once
2521     // and written exactly once using the access width appropriate to the type
2522     // of the container. The two accesses are not atomic.
2523     if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) &&
2524         CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
2525       Builder.CreateLoad(Ptr, true, "bf.load");
2526   }
2527 
2528   // Write the new value back out.
2529   Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified());
2530 
2531   // Return the new value of the bit-field, if requested.
2532   if (Result) {
2533     llvm::Value *ResultVal = MaskedVal;
2534 
2535     // Sign extend the value if needed.
2536     if (Info.IsSigned) {
2537       assert(Info.Size <= StorageSize);
2538       unsigned HighBits = StorageSize - Info.Size;
2539       if (HighBits) {
2540         ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl");
2541         ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr");
2542       }
2543     }
2544 
2545     ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned,
2546                                       "bf.result.cast");
2547     *Result = EmitFromMemory(ResultVal, Dst.getType());
2548   }
2549 }
2550 
2551 void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
2552                                                                LValue Dst) {
2553   // HLSL allows storing to scalar values through ExtVector component LValues.
2554   // To support this we need to handle the case where the destination address is
2555   // a scalar.
2556   Address DstAddr = Dst.getExtVectorAddress();
2557   if (!DstAddr.getElementType()->isVectorTy()) {
2558     assert(!Dst.getType()->isVectorType() &&
2559            "this should only occur for non-vector l-values");
2560     Builder.CreateStore(Src.getScalarVal(), DstAddr, Dst.isVolatileQualified());
2561     return;
2562   }
2563 
2564   // This access turns into a read/modify/write of the vector.  Load the input
2565   // value now.
2566   llvm::Value *Vec = Builder.CreateLoad(DstAddr, Dst.isVolatileQualified());
2567   const llvm::Constant *Elts = Dst.getExtVectorElts();
2568 
2569   llvm::Value *SrcVal = Src.getScalarVal();
2570 
2571   if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
2572     unsigned NumSrcElts = VTy->getNumElements();
2573     unsigned NumDstElts =
2574         cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
2575     if (NumDstElts == NumSrcElts) {
2576       // Use shuffle vector is the src and destination are the same number of
2577       // elements and restore the vector mask since it is on the side it will be
2578       // stored.
2579       SmallVector<int, 4> Mask(NumDstElts);
2580       for (unsigned i = 0; i != NumSrcElts; ++i)
2581         Mask[getAccessedFieldNo(i, Elts)] = i;
2582 
2583       Vec = Builder.CreateShuffleVector(SrcVal, Mask);
2584     } else if (NumDstElts > NumSrcElts) {
2585       // Extended the source vector to the same length and then shuffle it
2586       // into the destination.
2587       // FIXME: since we're shuffling with undef, can we just use the indices
2588       //        into that?  This could be simpler.
2589       SmallVector<int, 4> ExtMask;
2590       for (unsigned i = 0; i != NumSrcElts; ++i)
2591         ExtMask.push_back(i);
2592       ExtMask.resize(NumDstElts, -1);
2593       llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask);
2594       // build identity
2595       SmallVector<int, 4> Mask;
2596       for (unsigned i = 0; i != NumDstElts; ++i)
2597         Mask.push_back(i);
2598 
2599       // When the vector size is odd and .odd or .hi is used, the last element
2600       // of the Elts constant array will be one past the size of the vector.
2601       // Ignore the last element here, if it is greater than the mask size.
2602       if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size())
2603         NumSrcElts--;
2604 
2605       // modify when what gets shuffled in
2606       for (unsigned i = 0; i != NumSrcElts; ++i)
2607         Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts;
2608       Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);
2609     } else {
2610       // We should never shorten the vector
2611       llvm_unreachable("unexpected shorten vector length");
2612     }
2613   } else {
2614     // If the Src is a scalar (not a vector), and the target is a vector it must
2615     // be updating one element.
2616     unsigned InIdx = getAccessedFieldNo(0, Elts);
2617     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2618     Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
2619   }
2620 
2621   Builder.CreateStore(Vec, Dst.getExtVectorAddress(),
2622                       Dst.isVolatileQualified());
2623 }
2624 
2625 /// Store of global named registers are always calls to intrinsics.
2626 void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
2627   assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
2628          "Bad type for register variable");
2629   llvm::MDNode *RegName = cast<llvm::MDNode>(
2630       cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata());
2631   assert(RegName && "Register LValue is not metadata");
2632 
2633   // We accept integer and pointer types only
2634   llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType());
2635   llvm::Type *Ty = OrigTy;
2636   if (OrigTy->isPointerTy())
2637     Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2638   llvm::Type *Types[] = { Ty };
2639 
2640   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
2641   llvm::Value *Value = Src.getScalarVal();
2642   if (OrigTy->isPointerTy())
2643     Value = Builder.CreatePtrToInt(Value, Ty);
2644   Builder.CreateCall(
2645       F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value});
2646 }
2647 
2648 // setObjCGCLValueClass - sets class of the lvalue for the purpose of
2649 // generating write-barries API. It is currently a global, ivar,
2650 // or neither.
2651 static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
2652                                  LValue &LV,
2653                                  bool IsMemberAccess=false) {
2654   if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
2655     return;
2656 
2657   if (isa<ObjCIvarRefExpr>(E)) {
2658     QualType ExpTy = E->getType();
2659     if (IsMemberAccess && ExpTy->isPointerType()) {
2660       // If ivar is a structure pointer, assigning to field of
2661       // this struct follows gcc's behavior and makes it a non-ivar
2662       // writer-barrier conservatively.
2663       ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2664       if (ExpTy->isRecordType()) {
2665         LV.setObjCIvar(false);
2666         return;
2667       }
2668     }
2669     LV.setObjCIvar(true);
2670     auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E));
2671     LV.setBaseIvarExp(Exp->getBase());
2672     LV.setObjCArray(E->getType()->isArrayType());
2673     return;
2674   }
2675 
2676   if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) {
2677     if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
2678       if (VD->hasGlobalStorage()) {
2679         LV.setGlobalObjCRef(true);
2680         LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
2681       }
2682     }
2683     LV.setObjCArray(E->getType()->isArrayType());
2684     return;
2685   }
2686 
2687   if (const auto *Exp = dyn_cast<UnaryOperator>(E)) {
2688     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2689     return;
2690   }
2691 
2692   if (const auto *Exp = dyn_cast<ParenExpr>(E)) {
2693     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2694     if (LV.isObjCIvar()) {
2695       // If cast is to a structure pointer, follow gcc's behavior and make it
2696       // a non-ivar write-barrier.
2697       QualType ExpTy = E->getType();
2698       if (ExpTy->isPointerType())
2699         ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2700       if (ExpTy->isRecordType())
2701         LV.setObjCIvar(false);
2702     }
2703     return;
2704   }
2705 
2706   if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) {
2707     setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
2708     return;
2709   }
2710 
2711   if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) {
2712     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2713     return;
2714   }
2715 
2716   if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) {
2717     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2718     return;
2719   }
2720 
2721   if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
2722     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2723     return;
2724   }
2725 
2726   if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
2727     setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
2728     if (LV.isObjCIvar() && !LV.isObjCArray())
2729       // Using array syntax to assigning to what an ivar points to is not
2730       // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
2731       LV.setObjCIvar(false);
2732     else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
2733       // Using array syntax to assigning to what global points to is not
2734       // same as assigning to the global itself. {id *G;} G[i] = 0;
2735       LV.setGlobalObjCRef(false);
2736     return;
2737   }
2738 
2739   if (const auto *Exp = dyn_cast<MemberExpr>(E)) {
2740     setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
2741     // We don't know if member is an 'ivar', but this flag is looked at
2742     // only in the context of LV.isObjCIvar().
2743     LV.setObjCArray(E->getType()->isArrayType());
2744     return;
2745   }
2746 }
2747 
2748 static LValue EmitThreadPrivateVarDeclLValue(
2749     CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
2750     llvm::Type *RealVarTy, SourceLocation Loc) {
2751   if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
2752     Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
2753         CGF, VD, Addr, Loc);
2754   else
2755     Addr =
2756         CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
2757 
2758   Addr = Addr.withElementType(RealVarTy);
2759   return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2760 }
2761 
2762 static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
2763                                            const VarDecl *VD, QualType T) {
2764   std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2765       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2766   // Return an invalid address if variable is MT_To (or MT_Enter starting with
2767   // OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link
2768   // and MT_To (or MT_Enter) with unified memory, return a valid address.
2769   if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2770                 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2771                !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
2772     return Address::invalid();
2773   assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2774           ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2775             *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2776            CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
2777          "Expected link clause OR to clause with unified memory enabled.");
2778   QualType PtrTy = CGF.getContext().getPointerType(VD->getType());
2779   Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
2780   return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>());
2781 }
2782 
2783 Address
2784 CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
2785                                      LValueBaseInfo *PointeeBaseInfo,
2786                                      TBAAAccessInfo *PointeeTBAAInfo) {
2787   llvm::LoadInst *Load =
2788       Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
2789   CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo());
2790 
2791   QualType PointeeType = RefLVal.getType()->getPointeeType();
2792   CharUnits Align = CGM.getNaturalTypeAlignment(
2793       PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
2794       /* forPointeeType= */ true);
2795   return Address(Load, ConvertTypeForMem(PointeeType), Align);
2796 }
2797 
2798 LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
2799   LValueBaseInfo PointeeBaseInfo;
2800   TBAAAccessInfo PointeeTBAAInfo;
2801   Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo,
2802                                             &PointeeTBAAInfo);
2803   return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(),
2804                         PointeeBaseInfo, PointeeTBAAInfo);
2805 }
2806 
2807 Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
2808                                            const PointerType *PtrTy,
2809                                            LValueBaseInfo *BaseInfo,
2810                                            TBAAAccessInfo *TBAAInfo) {
2811   llvm::Value *Addr = Builder.CreateLoad(Ptr);
2812   return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
2813                  CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
2814                                              TBAAInfo,
2815                                              /*forPointeeType=*/true));
2816 }
2817 
2818 LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
2819                                                 const PointerType *PtrTy) {
2820   LValueBaseInfo BaseInfo;
2821   TBAAAccessInfo TBAAInfo;
2822   Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo);
2823   return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
2824 }
2825 
2826 static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
2827                                       const Expr *E, const VarDecl *VD) {
2828   QualType T = E->getType();
2829 
2830   // If it's thread_local, emit a call to its wrapper function instead.
2831   if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
2832       CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
2833     return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T);
2834   // Check if the variable is marked as declare target with link clause in
2835   // device codegen.
2836   if (CGF.getLangOpts().OpenMPIsTargetDevice) {
2837     Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
2838     if (Addr.isValid())
2839       return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2840   }
2841 
2842   llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
2843 
2844   if (VD->getTLSKind() != VarDecl::TLS_None)
2845     V = CGF.Builder.CreateThreadLocalAddress(V);
2846 
2847   llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
2848   CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2849   Address Addr(V, RealVarTy, Alignment);
2850   // Emit reference to the private copy of the variable if it is an OpenMP
2851   // threadprivate variable.
2852   if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
2853       VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2854     return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
2855                                           E->getExprLoc());
2856   }
2857   LValue LV = VD->getType()->isReferenceType() ?
2858       CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2859                                     AlignmentSource::Decl) :
2860       CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2861   setObjCGCLValueClass(CGF.getContext(), E, LV);
2862   return LV;
2863 }
2864 
2865 static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM,
2866                                                GlobalDecl GD) {
2867   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2868   if (FD->hasAttr<WeakRefAttr>()) {
2869     ConstantAddress aliasee = CGM.GetWeakRefReference(FD);
2870     return aliasee.getPointer();
2871   }
2872 
2873   llvm::Constant *V = CGM.GetAddrOfFunction(GD);
2874   return V;
2875 }
2876 
2877 static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
2878                                      GlobalDecl GD) {
2879   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2880   llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD);
2881   CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
2882   return CGF.MakeAddrLValue(V, E->getType(), Alignment,
2883                             AlignmentSource::Decl);
2884 }
2885 
2886 static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
2887                                       llvm::Value *ThisValue) {
2888 
2889   return CGF.EmitLValueForLambdaField(FD, ThisValue);
2890 }
2891 
2892 /// Named Registers are named metadata pointing to the register name
2893 /// which will be read from/written to as an argument to the intrinsic
2894 /// @llvm.read/write_register.
2895 /// So far, only the name is being passed down, but other options such as
2896 /// register type, allocation type or even optimization options could be
2897 /// passed down via the metadata node.
2898 static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
2899   SmallString<64> Name("llvm.named.register.");
2900   AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
2901   assert(Asm->getLabel().size() < 64-Name.size() &&
2902       "Register name too big");
2903   Name.append(Asm->getLabel());
2904   llvm::NamedMDNode *M =
2905     CGM.getModule().getOrInsertNamedMetadata(Name);
2906   if (M->getNumOperands() == 0) {
2907     llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(),
2908                                               Asm->getLabel());
2909     llvm::Metadata *Ops[] = {Str};
2910     M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
2911   }
2912 
2913   CharUnits Alignment = CGM.getContext().getDeclAlign(VD);
2914 
2915   llvm::Value *Ptr =
2916     llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));
2917   return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType());
2918 }
2919 
2920 /// Determine whether we can emit a reference to \p VD from the current
2921 /// context, despite not necessarily having seen an odr-use of the variable in
2922 /// this context.
2923 static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
2924                                                const DeclRefExpr *E,
2925                                                const VarDecl *VD) {
2926   // For a variable declared in an enclosing scope, do not emit a spurious
2927   // reference even if we have a capture, as that will emit an unwarranted
2928   // reference to our capture state, and will likely generate worse code than
2929   // emitting a local copy.
2930   if (E->refersToEnclosingVariableOrCapture())
2931     return false;
2932 
2933   // For a local declaration declared in this function, we can always reference
2934   // it even if we don't have an odr-use.
2935   if (VD->hasLocalStorage()) {
2936     return VD->getDeclContext() ==
2937            dyn_cast_or_null<DeclContext>(CGF.CurCodeDecl);
2938   }
2939 
2940   // For a global declaration, we can emit a reference to it if we know
2941   // for sure that we are able to emit a definition of it.
2942   VD = VD->getDefinition(CGF.getContext());
2943   if (!VD)
2944     return false;
2945 
2946   // Don't emit a spurious reference if it might be to a variable that only
2947   // exists on a different device / target.
2948   // FIXME: This is unnecessarily broad. Check whether this would actually be a
2949   // cross-target reference.
2950   if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
2951       CGF.getLangOpts().OpenCL) {
2952     return false;
2953   }
2954 
2955   // We can emit a spurious reference only if the linkage implies that we'll
2956   // be emitting a non-interposable symbol that will be retained until link
2957   // time.
2958   switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
2959   case llvm::GlobalValue::ExternalLinkage:
2960   case llvm::GlobalValue::LinkOnceODRLinkage:
2961   case llvm::GlobalValue::WeakODRLinkage:
2962   case llvm::GlobalValue::InternalLinkage:
2963   case llvm::GlobalValue::PrivateLinkage:
2964     return true;
2965   default:
2966     return false;
2967   }
2968 }
2969 
2970 LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
2971   const NamedDecl *ND = E->getDecl();
2972   QualType T = E->getType();
2973 
2974   assert(E->isNonOdrUse() != NOUR_Unevaluated &&
2975          "should not emit an unevaluated operand");
2976 
2977   if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2978     // Global Named registers access via intrinsics only
2979     if (VD->getStorageClass() == SC_Register &&
2980         VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
2981       return EmitGlobalNamedRegister(VD, CGM);
2982 
2983     // If this DeclRefExpr does not constitute an odr-use of the variable,
2984     // we're not permitted to emit a reference to it in general, and it might
2985     // not be captured if capture would be necessary for a use. Emit the
2986     // constant value directly instead.
2987     if (E->isNonOdrUse() == NOUR_Constant &&
2988         (VD->getType()->isReferenceType() ||
2989          !canEmitSpuriousReferenceToVariable(*this, E, VD))) {
2990       VD->getAnyInitializer(VD);
2991       llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
2992           E->getLocation(), *VD->evaluateValue(), VD->getType());
2993       assert(Val && "failed to emit constant expression");
2994 
2995       Address Addr = Address::invalid();
2996       if (!VD->getType()->isReferenceType()) {
2997         // Spill the constant value to a global.
2998         Addr = CGM.createUnnamedGlobalFrom(*VD, Val,
2999                                            getContext().getDeclAlign(VD));
3000         llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());
3001         auto *PTy = llvm::PointerType::get(
3002             VarTy, getTypes().getTargetAddressSpace(VD->getType()));
3003         Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
3004       } else {
3005         // Should we be using the alignment of the constant pointer we emitted?
3006         CharUnits Alignment =
3007             CGM.getNaturalTypeAlignment(E->getType(),
3008                                         /* BaseInfo= */ nullptr,
3009                                         /* TBAAInfo= */ nullptr,
3010                                         /* forPointeeType= */ true);
3011         Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
3012       }
3013       return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
3014     }
3015 
3016     // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
3017 
3018     // Check for captured variables.
3019     if (E->refersToEnclosingVariableOrCapture()) {
3020       VD = VD->getCanonicalDecl();
3021       if (auto *FD = LambdaCaptureFields.lookup(VD))
3022         return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
3023       if (CapturedStmtInfo) {
3024         auto I = LocalDeclMap.find(VD);
3025         if (I != LocalDeclMap.end()) {
3026           LValue CapLVal;
3027           if (VD->getType()->isReferenceType())
3028             CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(),
3029                                                 AlignmentSource::Decl);
3030           else
3031             CapLVal = MakeAddrLValue(I->second, T);
3032           // Mark lvalue as nontemporal if the variable is marked as nontemporal
3033           // in simd context.
3034           if (getLangOpts().OpenMP &&
3035               CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3036             CapLVal.setNontemporal(/*Value=*/true);
3037           return CapLVal;
3038         }
3039         LValue CapLVal =
3040             EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
3041                                     CapturedStmtInfo->getContextValue());
3042         Address LValueAddress = CapLVal.getAddress(*this);
3043         CapLVal = MakeAddrLValue(
3044             Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
3045                     getContext().getDeclAlign(VD)),
3046             CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl),
3047             CapLVal.getTBAAInfo());
3048         // Mark lvalue as nontemporal if the variable is marked as nontemporal
3049         // in simd context.
3050         if (getLangOpts().OpenMP &&
3051             CGM.getOpenMPRuntime().isNontemporalDecl(VD))
3052           CapLVal.setNontemporal(/*Value=*/true);
3053         return CapLVal;
3054       }
3055 
3056       assert(isa<BlockDecl>(CurCodeDecl));
3057       Address addr = GetAddrOfBlockDecl(VD);
3058       return MakeAddrLValue(addr, T, AlignmentSource::Decl);
3059     }
3060   }
3061 
3062   // FIXME: We should be able to assert this for FunctionDecls as well!
3063   // FIXME: We should be able to assert this for all DeclRefExprs, not just
3064   // those with a valid source location.
3065   assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
3066           !E->getLocation().isValid()) &&
3067          "Should not use decl without marking it used!");
3068 
3069   if (ND->hasAttr<WeakRefAttr>()) {
3070     const auto *VD = cast<ValueDecl>(ND);
3071     ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
3072     return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl);
3073   }
3074 
3075   if (const auto *VD = dyn_cast<VarDecl>(ND)) {
3076     // Check if this is a global variable.
3077     if (VD->hasLinkage() || VD->isStaticDataMember())
3078       return EmitGlobalVarDeclLValue(*this, E, VD);
3079 
3080     Address addr = Address::invalid();
3081 
3082     // The variable should generally be present in the local decl map.
3083     auto iter = LocalDeclMap.find(VD);
3084     if (iter != LocalDeclMap.end()) {
3085       addr = iter->second;
3086 
3087     // Otherwise, it might be static local we haven't emitted yet for
3088     // some reason; most likely, because it's in an outer function.
3089     } else if (VD->isStaticLocal()) {
3090       llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
3091           *VD, CGM.getLLVMLinkageVarDefinition(VD));
3092       addr = Address(
3093           var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
3094 
3095     // No other cases for now.
3096     } else {
3097       llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
3098     }
3099 
3100     // Handle threadlocal function locals.
3101     if (VD->getTLSKind() != VarDecl::TLS_None)
3102       addr = addr.withPointer(
3103           Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull);
3104 
3105     // Check for OpenMP threadprivate variables.
3106     if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
3107         VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
3108       return EmitThreadPrivateVarDeclLValue(
3109           *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()),
3110           E->getExprLoc());
3111     }
3112 
3113     // Drill into block byref variables.
3114     bool isBlockByref = VD->isEscapingByref();
3115     if (isBlockByref) {
3116       addr = emitBlockByrefAddress(addr, VD);
3117     }
3118 
3119     // Drill into reference types.
3120     LValue LV = VD->getType()->isReferenceType() ?
3121         EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) :
3122         MakeAddrLValue(addr, T, AlignmentSource::Decl);
3123 
3124     bool isLocalStorage = VD->hasLocalStorage();
3125 
3126     bool NonGCable = isLocalStorage &&
3127                      !VD->getType()->isReferenceType() &&
3128                      !isBlockByref;
3129     if (NonGCable) {
3130       LV.getQuals().removeObjCGCAttr();
3131       LV.setNonGC(true);
3132     }
3133 
3134     bool isImpreciseLifetime =
3135       (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
3136     if (isImpreciseLifetime)
3137       LV.setARCPreciseLifetime(ARCImpreciseLifetime);
3138     setObjCGCLValueClass(getContext(), E, LV);
3139     return LV;
3140   }
3141 
3142   if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
3143     LValue LV = EmitFunctionDeclLValue(*this, E, FD);
3144 
3145     // Emit debuginfo for the function declaration if the target wants to.
3146     if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) {
3147       if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) {
3148         auto *Fn =
3149             cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts());
3150         if (!Fn->getSubprogram())
3151           DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn);
3152       }
3153     }
3154 
3155     return LV;
3156   }
3157 
3158   // FIXME: While we're emitting a binding from an enclosing scope, all other
3159   // DeclRefExprs we see should be implicitly treated as if they also refer to
3160   // an enclosing scope.
3161   if (const auto *BD = dyn_cast<BindingDecl>(ND)) {
3162     if (E->refersToEnclosingVariableOrCapture()) {
3163       auto *FD = LambdaCaptureFields.lookup(BD);
3164       return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
3165     }
3166     return EmitLValue(BD->getBinding());
3167   }
3168 
3169   // We can form DeclRefExprs naming GUID declarations when reconstituting
3170   // non-type template parameters into expressions.
3171   if (const auto *GD = dyn_cast<MSGuidDecl>(ND))
3172     return MakeAddrLValue(CGM.GetAddrOfMSGuidDecl(GD), T,
3173                           AlignmentSource::Decl);
3174 
3175   if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) {
3176     auto ATPO = CGM.GetAddrOfTemplateParamObject(TPO);
3177     auto AS = getLangASFromTargetAS(ATPO.getAddressSpace());
3178 
3179     if (AS != T.getAddressSpace()) {
3180       auto TargetAS = getContext().getTargetAddressSpace(T.getAddressSpace());
3181       auto PtrTy = ATPO.getElementType()->getPointerTo(TargetAS);
3182       auto ASC = getTargetHooks().performAddrSpaceCast(
3183           CGM, ATPO.getPointer(), AS, T.getAddressSpace(), PtrTy);
3184       ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
3185     }
3186 
3187     return MakeAddrLValue(ATPO, T, AlignmentSource::Decl);
3188   }
3189 
3190   llvm_unreachable("Unhandled DeclRefExpr");
3191 }
3192 
3193 LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
3194   // __extension__ doesn't affect lvalue-ness.
3195   if (E->getOpcode() == UO_Extension)
3196     return EmitLValue(E->getSubExpr());
3197 
3198   QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
3199   switch (E->getOpcode()) {
3200   default: llvm_unreachable("Unknown unary operator lvalue!");
3201   case UO_Deref: {
3202     QualType T = E->getSubExpr()->getType()->getPointeeType();
3203     assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
3204 
3205     LValueBaseInfo BaseInfo;
3206     TBAAAccessInfo TBAAInfo;
3207     Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &BaseInfo,
3208                                             &TBAAInfo);
3209     LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
3210     LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
3211 
3212     // We should not generate __weak write barrier on indirect reference
3213     // of a pointer to object; as in void foo (__weak id *param); *param = 0;
3214     // But, we continue to generate __strong write barrier on indirect write
3215     // into a pointer to object.
3216     if (getLangOpts().ObjC &&
3217         getLangOpts().getGC() != LangOptions::NonGC &&
3218         LV.isObjCWeak())
3219       LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
3220     return LV;
3221   }
3222   case UO_Real:
3223   case UO_Imag: {
3224     LValue LV = EmitLValue(E->getSubExpr());
3225     assert(LV.isSimple() && "real/imag on non-ordinary l-value");
3226 
3227     // __real is valid on scalars.  This is a faster way of testing that.
3228     // __imag can only produce an rvalue on scalars.
3229     if (E->getOpcode() == UO_Real &&
3230         !LV.getAddress(*this).getElementType()->isStructTy()) {
3231       assert(E->getSubExpr()->getType()->isArithmeticType());
3232       return LV;
3233     }
3234 
3235     QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3236 
3237     Address Component =
3238         (E->getOpcode() == UO_Real
3239              ? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType())
3240              : emitAddrOfImagComponent(LV.getAddress(*this), LV.getType()));
3241     LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(),
3242                                    CGM.getTBAAInfoForSubobject(LV, T));
3243     ElemLV.getQuals().addQualifiers(LV.getQuals());
3244     return ElemLV;
3245   }
3246   case UO_PreInc:
3247   case UO_PreDec: {
3248     LValue LV = EmitLValue(E->getSubExpr());
3249     bool isInc = E->getOpcode() == UO_PreInc;
3250 
3251     if (E->getType()->isAnyComplexType())
3252       EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
3253     else
3254       EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
3255     return LV;
3256   }
3257   }
3258 }
3259 
3260 LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
3261   return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
3262                         E->getType(), AlignmentSource::Decl);
3263 }
3264 
3265 LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
3266   return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
3267                         E->getType(), AlignmentSource::Decl);
3268 }
3269 
3270 LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
3271   auto SL = E->getFunctionName();
3272   assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3273   StringRef FnName = CurFn->getName();
3274   if (FnName.starts_with("\01"))
3275     FnName = FnName.substr(1);
3276   StringRef NameItems[] = {
3277       PredefinedExpr::getIdentKindName(E->getIdentKind()), FnName};
3278   std::string GVName = llvm::join(NameItems, NameItems + 2, ".");
3279   if (auto *BD = dyn_cast_or_null<BlockDecl>(CurCodeDecl)) {
3280     std::string Name = std::string(SL->getString());
3281     if (!Name.empty()) {
3282       unsigned Discriminator =
3283           CGM.getCXXABI().getMangleContext().getBlockId(BD, true);
3284       if (Discriminator)
3285         Name += "_" + Twine(Discriminator + 1).str();
3286       auto C = CGM.GetAddrOfConstantCString(Name, GVName.c_str());
3287       return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3288     } else {
3289       auto C =
3290           CGM.GetAddrOfConstantCString(std::string(FnName), GVName.c_str());
3291       return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3292     }
3293   }
3294   auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName);
3295   return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3296 }
3297 
3298 /// Emit a type description suitable for use by a runtime sanitizer library. The
3299 /// format of a type descriptor is
3300 ///
3301 /// \code
3302 ///   { i16 TypeKind, i16 TypeInfo }
3303 /// \endcode
3304 ///
3305 /// followed by an array of i8 containing the type name. TypeKind is 0 for an
3306 /// integer, 1 for a floating point value, and -1 for anything else.
3307 llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
3308   // Only emit each type's descriptor once.
3309   if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))
3310     return C;
3311 
3312   uint16_t TypeKind = -1;
3313   uint16_t TypeInfo = 0;
3314 
3315   if (T->isIntegerType()) {
3316     TypeKind = 0;
3317     TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
3318                (T->isSignedIntegerType() ? 1 : 0);
3319   } else if (T->isFloatingType()) {
3320     TypeKind = 1;
3321     TypeInfo = getContext().getTypeSize(T);
3322   }
3323 
3324   // Format the type name as if for a diagnostic, including quotes and
3325   // optionally an 'aka'.
3326   SmallString<32> Buffer;
3327   CGM.getDiags().ConvertArgToString(
3328       DiagnosticsEngine::ak_qualtype, (intptr_t)T.getAsOpaquePtr(), StringRef(),
3329       StringRef(), std::nullopt, Buffer, std::nullopt);
3330 
3331   llvm::Constant *Components[] = {
3332     Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
3333     llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
3334   };
3335   llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
3336 
3337   auto *GV = new llvm::GlobalVariable(
3338       CGM.getModule(), Descriptor->getType(),
3339       /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3340   GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3341   CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
3342 
3343   // Remember the descriptor for this type.
3344   CGM.setTypeDescriptorInMap(T, GV);
3345 
3346   return GV;
3347 }
3348 
3349 llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
3350   llvm::Type *TargetTy = IntPtrTy;
3351 
3352   if (V->getType() == TargetTy)
3353     return V;
3354 
3355   // Floating-point types which fit into intptr_t are bitcast to integers
3356   // and then passed directly (after zero-extension, if necessary).
3357   if (V->getType()->isFloatingPointTy()) {
3358     unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
3359     if (Bits <= TargetTy->getIntegerBitWidth())
3360       V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(),
3361                                                          Bits));
3362   }
3363 
3364   // Integers which fit in intptr_t are zero-extended and passed directly.
3365   if (V->getType()->isIntegerTy() &&
3366       V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3367     return Builder.CreateZExt(V, TargetTy);
3368 
3369   // Pointers are passed directly, everything else is passed by address.
3370   if (!V->getType()->isPointerTy()) {
3371     Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
3372     Builder.CreateStore(V, Ptr);
3373     V = Ptr.getPointer();
3374   }
3375   return Builder.CreatePtrToInt(V, TargetTy);
3376 }
3377 
3378 /// Emit a representation of a SourceLocation for passing to a handler
3379 /// in a sanitizer runtime library. The format for this data is:
3380 /// \code
3381 ///   struct SourceLocation {
3382 ///     const char *Filename;
3383 ///     int32_t Line, Column;
3384 ///   };
3385 /// \endcode
3386 /// For an invalid SourceLocation, the Filename pointer is null.
3387 llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
3388   llvm::Constant *Filename;
3389   int Line, Column;
3390 
3391   PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
3392   if (PLoc.isValid()) {
3393     StringRef FilenameString = PLoc.getFilename();
3394 
3395     int PathComponentsToStrip =
3396         CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
3397     if (PathComponentsToStrip < 0) {
3398       assert(PathComponentsToStrip != INT_MIN);
3399       int PathComponentsToKeep = -PathComponentsToStrip;
3400       auto I = llvm::sys::path::rbegin(FilenameString);
3401       auto E = llvm::sys::path::rend(FilenameString);
3402       while (I != E && --PathComponentsToKeep)
3403         ++I;
3404 
3405       FilenameString = FilenameString.substr(I - E);
3406     } else if (PathComponentsToStrip > 0) {
3407       auto I = llvm::sys::path::begin(FilenameString);
3408       auto E = llvm::sys::path::end(FilenameString);
3409       while (I != E && PathComponentsToStrip--)
3410         ++I;
3411 
3412       if (I != E)
3413         FilenameString =
3414             FilenameString.substr(I - llvm::sys::path::begin(FilenameString));
3415       else
3416         FilenameString = llvm::sys::path::filename(FilenameString);
3417     }
3418 
3419     auto FilenameGV =
3420         CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src");
3421     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
3422         cast<llvm::GlobalVariable>(
3423             FilenameGV.getPointer()->stripPointerCasts()));
3424     Filename = FilenameGV.getPointer();
3425     Line = PLoc.getLine();
3426     Column = PLoc.getColumn();
3427   } else {
3428     Filename = llvm::Constant::getNullValue(Int8PtrTy);
3429     Line = Column = 0;
3430   }
3431 
3432   llvm::Constant *Data[] = {Filename, Builder.getInt32(Line),
3433                             Builder.getInt32(Column)};
3434 
3435   return llvm::ConstantStruct::getAnon(Data);
3436 }
3437 
3438 namespace {
3439 /// Specify under what conditions this check can be recovered
3440 enum class CheckRecoverableKind {
3441   /// Always terminate program execution if this check fails.
3442   Unrecoverable,
3443   /// Check supports recovering, runtime has both fatal (noreturn) and
3444   /// non-fatal handlers for this check.
3445   Recoverable,
3446   /// Runtime conditionally aborts, always need to support recovery.
3447   AlwaysRecoverable
3448 };
3449 }
3450 
3451 static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) {
3452   assert(Kind.countPopulation() == 1);
3453   if (Kind == SanitizerKind::Vptr)
3454     return CheckRecoverableKind::AlwaysRecoverable;
3455   else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3456     return CheckRecoverableKind::Unrecoverable;
3457   else
3458     return CheckRecoverableKind::Recoverable;
3459 }
3460 
3461 namespace {
3462 struct SanitizerHandlerInfo {
3463   char const *const Name;
3464   unsigned Version;
3465 };
3466 }
3467 
3468 const SanitizerHandlerInfo SanitizerHandlers[] = {
3469 #define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3470     LIST_SANITIZER_CHECKS
3471 #undef SANITIZER_CHECK
3472 };
3473 
3474 static void emitCheckHandlerCall(CodeGenFunction &CGF,
3475                                  llvm::FunctionType *FnType,
3476                                  ArrayRef<llvm::Value *> FnArgs,
3477                                  SanitizerHandler CheckHandler,
3478                                  CheckRecoverableKind RecoverKind, bool IsFatal,
3479                                  llvm::BasicBlock *ContBB) {
3480   assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3481   std::optional<ApplyDebugLocation> DL;
3482   if (!CGF.Builder.getCurrentDebugLocation()) {
3483     // Ensure that the call has at least an artificial debug location.
3484     DL.emplace(CGF, SourceLocation());
3485   }
3486   bool NeedsAbortSuffix =
3487       IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3488   bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
3489   const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
3490   const StringRef CheckName = CheckInfo.Name;
3491   std::string FnName = "__ubsan_handle_" + CheckName.str();
3492   if (CheckInfo.Version && !MinimalRuntime)
3493     FnName += "_v" + llvm::utostr(CheckInfo.Version);
3494   if (MinimalRuntime)
3495     FnName += "_minimal";
3496   if (NeedsAbortSuffix)
3497     FnName += "_abort";
3498   bool MayReturn =
3499       !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3500 
3501   llvm::AttrBuilder B(CGF.getLLVMContext());
3502   if (!MayReturn) {
3503     B.addAttribute(llvm::Attribute::NoReturn)
3504         .addAttribute(llvm::Attribute::NoUnwind);
3505   }
3506   B.addUWTableAttr(llvm::UWTableKind::Default);
3507 
3508   llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
3509       FnType, FnName,
3510       llvm::AttributeList::get(CGF.getLLVMContext(),
3511                                llvm::AttributeList::FunctionIndex, B),
3512       /*Local=*/true);
3513   llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs);
3514   if (!MayReturn) {
3515     HandlerCall->setDoesNotReturn();
3516     CGF.Builder.CreateUnreachable();
3517   } else {
3518     CGF.Builder.CreateBr(ContBB);
3519   }
3520 }
3521 
3522 void CodeGenFunction::EmitCheck(
3523     ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3524     SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
3525     ArrayRef<llvm::Value *> DynamicArgs) {
3526   assert(IsSanitizerScope);
3527   assert(Checked.size() > 0);
3528   assert(CheckHandler >= 0 &&
3529          size_t(CheckHandler) < std::size(SanitizerHandlers));
3530   const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
3531 
3532   llvm::Value *FatalCond = nullptr;
3533   llvm::Value *RecoverableCond = nullptr;
3534   llvm::Value *TrapCond = nullptr;
3535   for (int i = 0, n = Checked.size(); i < n; ++i) {
3536     llvm::Value *Check = Checked[i].first;
3537     // -fsanitize-trap= overrides -fsanitize-recover=.
3538     llvm::Value *&Cond =
3539         CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second)
3540             ? TrapCond
3541             : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second)
3542                   ? RecoverableCond
3543                   : FatalCond;
3544     Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check;
3545   }
3546 
3547   if (TrapCond)
3548     EmitTrapCheck(TrapCond, CheckHandler);
3549   if (!FatalCond && !RecoverableCond)
3550     return;
3551 
3552   llvm::Value *JointCond;
3553   if (FatalCond && RecoverableCond)
3554     JointCond = Builder.CreateAnd(FatalCond, RecoverableCond);
3555   else
3556     JointCond = FatalCond ? FatalCond : RecoverableCond;
3557   assert(JointCond);
3558 
3559   CheckRecoverableKind RecoverKind = getRecoverableKind(Checked[0].second);
3560   assert(SanOpts.has(Checked[0].second));
3561 #ifndef NDEBUG
3562   for (int i = 1, n = Checked.size(); i < n; ++i) {
3563     assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
3564            "All recoverable kinds in a single check must be same!");
3565     assert(SanOpts.has(Checked[i].second));
3566   }
3567 #endif
3568 
3569   llvm::BasicBlock *Cont = createBasicBlock("cont");
3570   llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName);
3571   llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers);
3572   // Give hint that we very much don't expect to execute the handler
3573   // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
3574   llvm::MDBuilder MDHelper(getLLVMContext());
3575   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3576   Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);
3577   EmitBlock(Handlers);
3578 
3579   // Handler functions take an i8* pointing to the (handler-specific) static
3580   // information block, followed by a sequence of intptr_t arguments
3581   // representing operand values.
3582   SmallVector<llvm::Value *, 4> Args;
3583   SmallVector<llvm::Type *, 4> ArgTypes;
3584   if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
3585     Args.reserve(DynamicArgs.size() + 1);
3586     ArgTypes.reserve(DynamicArgs.size() + 1);
3587 
3588     // Emit handler arguments and create handler function type.
3589     if (!StaticArgs.empty()) {
3590       llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3591       auto *InfoPtr = new llvm::GlobalVariable(
3592           CGM.getModule(), Info->getType(), false,
3593           llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr,
3594           llvm::GlobalVariable::NotThreadLocal,
3595           CGM.getDataLayout().getDefaultGlobalsAddressSpace());
3596       InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3597       CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
3598       Args.push_back(InfoPtr);
3599       ArgTypes.push_back(Args.back()->getType());
3600     }
3601 
3602     for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3603       Args.push_back(EmitCheckValue(DynamicArgs[i]));
3604       ArgTypes.push_back(IntPtrTy);
3605     }
3606   }
3607 
3608   llvm::FunctionType *FnType =
3609     llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);
3610 
3611   if (!FatalCond || !RecoverableCond) {
3612     // Simple case: we need to generate a single handler call, either
3613     // fatal, or non-fatal.
3614     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,
3615                          (FatalCond != nullptr), Cont);
3616   } else {
3617     // Emit two handler calls: first one for set of unrecoverable checks,
3618     // another one for recoverable.
3619     llvm::BasicBlock *NonFatalHandlerBB =
3620         createBasicBlock("non_fatal." + CheckName);
3621     llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName);
3622     Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
3623     EmitBlock(FatalHandlerBB);
3624     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,
3625                          NonFatalHandlerBB);
3626     EmitBlock(NonFatalHandlerBB);
3627     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,
3628                          Cont);
3629   }
3630 
3631   EmitBlock(Cont);
3632 }
3633 
3634 void CodeGenFunction::EmitCfiSlowPathCheck(
3635     SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3636     llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) {
3637   llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");
3638 
3639   llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");
3640   llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);
3641 
3642   llvm::MDBuilder MDHelper(getLLVMContext());
3643   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3644   BI->setMetadata(llvm::LLVMContext::MD_prof, Node);
3645 
3646   EmitBlock(CheckBB);
3647 
3648   bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Kind);
3649 
3650   llvm::CallInst *CheckCall;
3651   llvm::FunctionCallee SlowPathFn;
3652   if (WithDiag) {
3653     llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3654     auto *InfoPtr =
3655         new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
3656                                  llvm::GlobalVariable::PrivateLinkage, Info);
3657     InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3658     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
3659 
3660     SlowPathFn = CGM.getModule().getOrInsertFunction(
3661         "__cfi_slowpath_diag",
3662         llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy},
3663                                 false));
3664     CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr, InfoPtr});
3665   } else {
3666     SlowPathFn = CGM.getModule().getOrInsertFunction(
3667         "__cfi_slowpath",
3668         llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false));
3669     CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
3670   }
3671 
3672   CGM.setDSOLocal(
3673       cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));
3674   CheckCall->setDoesNotThrow();
3675 
3676   EmitBlock(Cont);
3677 }
3678 
3679 // Emit a stub for __cfi_check function so that the linker knows about this
3680 // symbol in LTO mode.
3681 void CodeGenFunction::EmitCfiCheckStub() {
3682   llvm::Module *M = &CGM.getModule();
3683   auto &Ctx = M->getContext();
3684   llvm::Function *F = llvm::Function::Create(
3685       llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false),
3686       llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M);
3687   F->setAlignment(llvm::Align(4096));
3688   CGM.setDSOLocal(F);
3689   llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F);
3690   // CrossDSOCFI pass is not executed if there is no executable code.
3691   SmallVector<llvm::Value*> Args{F->getArg(2), F->getArg(1)};
3692   llvm::CallInst::Create(M->getFunction("__cfi_check_fail"), Args, "", BB);
3693   llvm::ReturnInst::Create(Ctx, nullptr, BB);
3694 }
3695 
3696 // This function is basically a switch over the CFI failure kind, which is
3697 // extracted from CFICheckFailData (1st function argument). Each case is either
3698 // llvm.trap or a call to one of the two runtime handlers, based on
3699 // -fsanitize-trap and -fsanitize-recover settings.  Default case (invalid
3700 // failure kind) traps, but this should really never happen.  CFICheckFailData
3701 // can be nullptr if the calling module has -fsanitize-trap behavior for this
3702 // check kind; in this case __cfi_check_fail traps as well.
3703 void CodeGenFunction::EmitCfiCheckFail() {
3704   SanitizerScope SanScope(this);
3705   FunctionArgList Args;
3706   ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy,
3707                             ImplicitParamKind::Other);
3708   ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy,
3709                             ImplicitParamKind::Other);
3710   Args.push_back(&ArgData);
3711   Args.push_back(&ArgAddr);
3712 
3713   const CGFunctionInfo &FI =
3714     CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, Args);
3715 
3716   llvm::Function *F = llvm::Function::Create(
3717       llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false),
3718       llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule());
3719 
3720   CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);
3721   CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
3722   F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3723 
3724   StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args,
3725                 SourceLocation());
3726 
3727   // This function is not affected by NoSanitizeList. This function does
3728   // not have a source location, but "src:*" would still apply. Revert any
3729   // changes to SanOpts made in StartFunction.
3730   SanOpts = CGM.getLangOpts().Sanitize;
3731 
3732   llvm::Value *Data =
3733       EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,
3734                        CGM.getContext().VoidPtrTy, ArgData.getLocation());
3735   llvm::Value *Addr =
3736       EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,
3737                        CGM.getContext().VoidPtrTy, ArgAddr.getLocation());
3738 
3739   // Data == nullptr means the calling module has trap behaviour for this check.
3740   llvm::Value *DataIsNotNullPtr =
3741       Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy));
3742   EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
3743 
3744   llvm::StructType *SourceLocationTy =
3745       llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty);
3746   llvm::StructType *CfiCheckFailDataTy =
3747       llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy);
3748 
3749   llvm::Value *V = Builder.CreateConstGEP2_32(
3750       CfiCheckFailDataTy,
3751       Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
3752       0);
3753 
3754   Address CheckKindAddr(V, Int8Ty, getIntAlign());
3755   llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
3756 
3757   llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3758       CGM.getLLVMContext(),
3759       llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
3760   llvm::Value *ValidVtable = Builder.CreateZExt(
3761       Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
3762                          {Addr, AllVtables}),
3763       IntPtrTy);
3764 
3765   const std::pair<int, SanitizerMask> CheckKinds[] = {
3766       {CFITCK_VCall, SanitizerKind::CFIVCall},
3767       {CFITCK_NVCall, SanitizerKind::CFINVCall},
3768       {CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast},
3769       {CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast},
3770       {CFITCK_ICall, SanitizerKind::CFIICall}};
3771 
3772   SmallVector<std::pair<llvm::Value *, SanitizerMask>, 5> Checks;
3773   for (auto CheckKindMaskPair : CheckKinds) {
3774     int Kind = CheckKindMaskPair.first;
3775     SanitizerMask Mask = CheckKindMaskPair.second;
3776     llvm::Value *Cond =
3777         Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind));
3778     if (CGM.getLangOpts().Sanitize.has(Mask))
3779       EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
3780                 {Data, Addr, ValidVtable});
3781     else
3782       EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail);
3783   }
3784 
3785   FinishFunction();
3786   // The only reference to this function will be created during LTO link.
3787   // Make sure it survives until then.
3788   CGM.addUsedGlobal(F);
3789 }
3790 
3791 void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
3792   if (SanOpts.has(SanitizerKind::Unreachable)) {
3793     SanitizerScope SanScope(this);
3794     EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
3795                              SanitizerKind::Unreachable),
3796               SanitizerHandler::BuiltinUnreachable,
3797               EmitCheckSourceLocation(Loc), std::nullopt);
3798   }
3799   Builder.CreateUnreachable();
3800 }
3801 
3802 void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
3803                                     SanitizerHandler CheckHandlerID) {
3804   llvm::BasicBlock *Cont = createBasicBlock("cont");
3805 
3806   // If we're optimizing, collapse all calls to trap down to just one per
3807   // check-type per function to save on code size.
3808   if (TrapBBs.size() <= CheckHandlerID)
3809     TrapBBs.resize(CheckHandlerID + 1);
3810 
3811   llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3812 
3813   if (!ClSanitizeDebugDeoptimization &&
3814       CGM.getCodeGenOpts().OptimizationLevel && TrapBB &&
3815       (!CurCodeDecl || !CurCodeDecl->hasAttr<OptimizeNoneAttr>())) {
3816     auto Call = TrapBB->begin();
3817     assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
3818 
3819     Call->applyMergedLocation(Call->getDebugLoc(),
3820                               Builder.getCurrentDebugLocation());
3821     Builder.CreateCondBr(Checked, Cont, TrapBB);
3822   } else {
3823     TrapBB = createBasicBlock("trap");
3824     Builder.CreateCondBr(Checked, Cont, TrapBB);
3825     EmitBlock(TrapBB);
3826 
3827     llvm::CallInst *TrapCall = Builder.CreateCall(
3828         CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),
3829         llvm::ConstantInt::get(CGM.Int8Ty, ClSanitizeDebugDeoptimization
3830                                                ? TrapBB->getParent()->size()
3831                                                : CheckHandlerID));
3832 
3833     if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3834       auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3835                                     CGM.getCodeGenOpts().TrapFuncName);
3836       TrapCall->addFnAttr(A);
3837     }
3838     TrapCall->setDoesNotReturn();
3839     TrapCall->setDoesNotThrow();
3840     Builder.CreateUnreachable();
3841   }
3842 
3843   EmitBlock(Cont);
3844 }
3845 
3846 llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
3847   llvm::CallInst *TrapCall =
3848       Builder.CreateCall(CGM.getIntrinsic(IntrID));
3849 
3850   if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3851     auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3852                                   CGM.getCodeGenOpts().TrapFuncName);
3853     TrapCall->addFnAttr(A);
3854   }
3855 
3856   return TrapCall;
3857 }
3858 
3859 Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
3860                                                  LValueBaseInfo *BaseInfo,
3861                                                  TBAAAccessInfo *TBAAInfo) {
3862   assert(E->getType()->isArrayType() &&
3863          "Array to pointer decay must have array source type!");
3864 
3865   // Expressions of array type can't be bitfields or vector elements.
3866   LValue LV = EmitLValue(E);
3867   Address Addr = LV.getAddress(*this);
3868 
3869   // If the array type was an incomplete type, we need to make sure
3870   // the decay ends up being the right type.
3871   llvm::Type *NewTy = ConvertType(E->getType());
3872   Addr = Addr.withElementType(NewTy);
3873 
3874   // Note that VLA pointers are always decayed, so we don't need to do
3875   // anything here.
3876   if (!E->getType()->isVariableArrayType()) {
3877     assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3878            "Expected pointer to array");
3879     Addr = Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3880   }
3881 
3882   // The result of this decay conversion points to an array element within the
3883   // base lvalue. However, since TBAA currently does not support representing
3884   // accesses to elements of member arrays, we conservatively represent accesses
3885   // to the pointee object as if it had no any base lvalue specified.
3886   // TODO: Support TBAA for member arrays.
3887   QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
3888   if (BaseInfo) *BaseInfo = LV.getBaseInfo();
3889   if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType);
3890 
3891   return Addr.withElementType(ConvertTypeForMem(EltType));
3892 }
3893 
3894 /// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
3895 /// array to pointer, return the array subexpression.
3896 static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
3897   // If this isn't just an array->pointer decay, bail out.
3898   const auto *CE = dyn_cast<CastExpr>(E);
3899   if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
3900     return nullptr;
3901 
3902   // If this is a decay from variable width array, bail out.
3903   const Expr *SubExpr = CE->getSubExpr();
3904   if (SubExpr->getType()->isVariableArrayType())
3905     return nullptr;
3906 
3907   return SubExpr;
3908 }
3909 
3910 static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
3911                                           llvm::Type *elemType,
3912                                           llvm::Value *ptr,
3913                                           ArrayRef<llvm::Value*> indices,
3914                                           bool inbounds,
3915                                           bool signedIndices,
3916                                           SourceLocation loc,
3917                                     const llvm::Twine &name = "arrayidx") {
3918   if (inbounds) {
3919     return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices,
3920                                       CodeGenFunction::NotSubtraction, loc,
3921                                       name);
3922   } else {
3923     return CGF.Builder.CreateGEP(elemType, ptr, indices, name);
3924   }
3925 }
3926 
3927 static CharUnits getArrayElementAlign(CharUnits arrayAlign,
3928                                       llvm::Value *idx,
3929                                       CharUnits eltSize) {
3930   // If we have a constant index, we can use the exact offset of the
3931   // element we're accessing.
3932   if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
3933     CharUnits offset = constantIdx->getZExtValue() * eltSize;
3934     return arrayAlign.alignmentAtOffset(offset);
3935 
3936   // Otherwise, use the worst-case alignment for any element.
3937   } else {
3938     return arrayAlign.alignmentOfArrayElement(eltSize);
3939   }
3940 }
3941 
3942 static QualType getFixedSizeElementType(const ASTContext &ctx,
3943                                         const VariableArrayType *vla) {
3944   QualType eltType;
3945   do {
3946     eltType = vla->getElementType();
3947   } while ((vla = ctx.getAsVariableArrayType(eltType)));
3948   return eltType;
3949 }
3950 
3951 static bool hasBPFPreserveStaticOffset(const RecordDecl *D) {
3952   return D && D->hasAttr<BPFPreserveStaticOffsetAttr>();
3953 }
3954 
3955 static bool hasBPFPreserveStaticOffset(const Expr *E) {
3956   if (!E)
3957     return false;
3958   QualType PointeeType = E->getType()->getPointeeType();
3959   if (PointeeType.isNull())
3960     return false;
3961   if (const auto *BaseDecl = PointeeType->getAsRecordDecl())
3962     return hasBPFPreserveStaticOffset(BaseDecl);
3963   return false;
3964 }
3965 
3966 // Wraps Addr with a call to llvm.preserve.static.offset intrinsic.
3967 static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
3968                                                Address &Addr) {
3969   if (!CGF.getTarget().getTriple().isBPF())
3970     return Addr;
3971 
3972   llvm::Function *Fn =
3973       CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset);
3974   llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()});
3975   return Address(Call, Addr.getElementType(), Addr.getAlignment());
3976 }
3977 
3978 /// Given an array base, check whether its member access belongs to a record
3979 /// with preserve_access_index attribute or not.
3980 static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
3981   if (!ArrayBase || !CGF.getDebugInfo())
3982     return false;
3983 
3984   // Only support base as either a MemberExpr or DeclRefExpr.
3985   // DeclRefExpr to cover cases like:
3986   //    struct s { int a; int b[10]; };
3987   //    struct s *p;
3988   //    p[1].a
3989   // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
3990   // p->b[5] is a MemberExpr example.
3991   const Expr *E = ArrayBase->IgnoreImpCasts();
3992   if (const auto *ME = dyn_cast<MemberExpr>(E))
3993     return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3994 
3995   if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
3996     const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());
3997     if (!VarDef)
3998       return false;
3999 
4000     const auto *PtrT = VarDef->getType()->getAs<PointerType>();
4001     if (!PtrT)
4002       return false;
4003 
4004     const auto *PointeeT = PtrT->getPointeeType()
4005                              ->getUnqualifiedDesugaredType();
4006     if (const auto *RecT = dyn_cast<RecordType>(PointeeT))
4007       return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
4008     return false;
4009   }
4010 
4011   return false;
4012 }
4013 
4014 static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
4015                                      ArrayRef<llvm::Value *> indices,
4016                                      QualType eltType, bool inbounds,
4017                                      bool signedIndices, SourceLocation loc,
4018                                      QualType *arrayType = nullptr,
4019                                      const Expr *Base = nullptr,
4020                                      const llvm::Twine &name = "arrayidx") {
4021   // All the indices except that last must be zero.
4022 #ifndef NDEBUG
4023   for (auto *idx : indices.drop_back())
4024     assert(isa<llvm::ConstantInt>(idx) &&
4025            cast<llvm::ConstantInt>(idx)->isZero());
4026 #endif
4027 
4028   // Determine the element size of the statically-sized base.  This is
4029   // the thing that the indices are expressed in terms of.
4030   if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) {
4031     eltType = getFixedSizeElementType(CGF.getContext(), vla);
4032   }
4033 
4034   // We can use that to compute the best alignment of the element.
4035   CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
4036   CharUnits eltAlign =
4037     getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
4038 
4039   if (hasBPFPreserveStaticOffset(Base))
4040     addr = wrapWithBPFPreserveStaticOffset(CGF, addr);
4041 
4042   llvm::Value *eltPtr;
4043   auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
4044   if (!LastIndex ||
4045       (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) {
4046     eltPtr = emitArraySubscriptGEP(
4047         CGF, addr.getElementType(), addr.getPointer(), indices, inbounds,
4048         signedIndices, loc, name);
4049   } else {
4050     // Remember the original array subscript for bpf target
4051     unsigned idx = LastIndex->getZExtValue();
4052     llvm::DIType *DbgInfo = nullptr;
4053     if (arrayType)
4054       DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);
4055     eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(),
4056                                                         addr.getPointer(),
4057                                                         indices.size() - 1,
4058                                                         idx, DbgInfo);
4059   }
4060 
4061   return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
4062 }
4063 
4064 /// The offset of a field from the beginning of the record.
4065 static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,
4066                                  const FieldDecl *FD, int64_t &Offset) {
4067   ASTContext &Ctx = CGF.getContext();
4068   const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
4069   unsigned FieldNo = 0;
4070 
4071   for (const Decl *D : RD->decls()) {
4072     if (const auto *Record = dyn_cast<RecordDecl>(D))
4073       if (getFieldOffsetInBits(CGF, Record, FD, Offset)) {
4074         Offset += Layout.getFieldOffset(FieldNo);
4075         return true;
4076       }
4077 
4078     if (const auto *Field = dyn_cast<FieldDecl>(D))
4079       if (FD == Field) {
4080         Offset += Layout.getFieldOffset(FieldNo);
4081         return true;
4082       }
4083 
4084     if (isa<FieldDecl>(D))
4085       ++FieldNo;
4086   }
4087 
4088   return false;
4089 }
4090 
4091 /// Returns the relative offset difference between \p FD1 and \p FD2.
4092 /// \code
4093 ///   offsetof(struct foo, FD1) - offsetof(struct foo, FD2)
4094 /// \endcode
4095 /// Both fields must be within the same struct.
4096 static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF,
4097                                                         const FieldDecl *FD1,
4098                                                         const FieldDecl *FD2) {
4099   const RecordDecl *FD1OuterRec =
4100       FD1->getParent()->getOuterLexicalRecordContext();
4101   const RecordDecl *FD2OuterRec =
4102       FD2->getParent()->getOuterLexicalRecordContext();
4103 
4104   if (FD1OuterRec != FD2OuterRec)
4105     // Fields must be within the same RecordDecl.
4106     return std::optional<int64_t>();
4107 
4108   int64_t FD1Offset = 0;
4109   if (!getFieldOffsetInBits(CGF, FD1OuterRec, FD1, FD1Offset))
4110     return std::optional<int64_t>();
4111 
4112   int64_t FD2Offset = 0;
4113   if (!getFieldOffsetInBits(CGF, FD2OuterRec, FD2, FD2Offset))
4114     return std::optional<int64_t>();
4115 
4116   return std::make_optional<int64_t>(FD1Offset - FD2Offset);
4117 }
4118 
4119 LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
4120                                                bool Accessed) {
4121   // The index must always be an integer, which is not an aggregate.  Emit it
4122   // in lexical order (this complexity is, sadly, required by C++17).
4123   llvm::Value *IdxPre =
4124       (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr;
4125   bool SignedIndices = false;
4126   auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
4127     auto *Idx = IdxPre;
4128     if (E->getLHS() != E->getIdx()) {
4129       assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
4130       Idx = EmitScalarExpr(E->getIdx());
4131     }
4132 
4133     QualType IdxTy = E->getIdx()->getType();
4134     bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
4135     SignedIndices |= IdxSigned;
4136 
4137     if (SanOpts.has(SanitizerKind::ArrayBounds))
4138       EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed);
4139 
4140     // Extend or truncate the index type to 32 or 64-bits.
4141     if (Promote && Idx->getType() != IntPtrTy)
4142       Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
4143 
4144     return Idx;
4145   };
4146   IdxPre = nullptr;
4147 
4148   // If the base is a vector type, then we are forming a vector element lvalue
4149   // with this subscript.
4150   if (E->getBase()->getType()->isVectorType() &&
4151       !isa<ExtVectorElementExpr>(E->getBase())) {
4152     // Emit the vector as an lvalue to get its address.
4153     LValue LHS = EmitLValue(E->getBase());
4154     auto *Idx = EmitIdxAfterBase(/*Promote*/false);
4155     assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
4156     return LValue::MakeVectorElt(LHS.getAddress(*this), Idx,
4157                                  E->getBase()->getType(), LHS.getBaseInfo(),
4158                                  TBAAAccessInfo());
4159   }
4160 
4161   // All the other cases basically behave like simple offsetting.
4162 
4163   // Handle the extvector case we ignored above.
4164   if (isa<ExtVectorElementExpr>(E->getBase())) {
4165     LValue LV = EmitLValue(E->getBase());
4166     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4167     Address Addr = EmitExtVectorElementLValue(LV);
4168 
4169     QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
4170     Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true,
4171                                  SignedIndices, E->getExprLoc());
4172     return MakeAddrLValue(Addr, EltType, LV.getBaseInfo(),
4173                           CGM.getTBAAInfoForSubobject(LV, EltType));
4174   }
4175 
4176   LValueBaseInfo EltBaseInfo;
4177   TBAAAccessInfo EltTBAAInfo;
4178   Address Addr = Address::invalid();
4179   if (const VariableArrayType *vla =
4180            getContext().getAsVariableArrayType(E->getType())) {
4181     // The base must be a pointer, which is not an aggregate.  Emit
4182     // it.  It needs to be emitted first in case it's what captures
4183     // the VLA bounds.
4184     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
4185     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4186 
4187     // The element count here is the total number of non-VLA elements.
4188     llvm::Value *numElements = getVLASize(vla).NumElts;
4189 
4190     // Effectively, the multiply by the VLA size is part of the GEP.
4191     // GEP indexes are signed, and scaling an index isn't permitted to
4192     // signed-overflow, so we use the same semantics for our explicit
4193     // multiply.  We suppress this if overflow is not undefined behavior.
4194     if (getLangOpts().isSignedOverflowDefined()) {
4195       Idx = Builder.CreateMul(Idx, numElements);
4196     } else {
4197       Idx = Builder.CreateNSWMul(Idx, numElements);
4198     }
4199 
4200     Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
4201                                  !getLangOpts().isSignedOverflowDefined(),
4202                                  SignedIndices, E->getExprLoc());
4203 
4204   } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
4205     // Indexing over an interface, as in "NSString *P; P[4];"
4206 
4207     // Emit the base pointer.
4208     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
4209     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4210 
4211     CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);
4212     llvm::Value *InterfaceSizeVal =
4213         llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());
4214 
4215     llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal);
4216 
4217     // We don't necessarily build correct LLVM struct types for ObjC
4218     // interfaces, so we can't rely on GEP to do this scaling
4219     // correctly, so we need to cast to i8*.  FIXME: is this actually
4220     // true?  A lot of other things in the fragile ABI would break...
4221     llvm::Type *OrigBaseElemTy = Addr.getElementType();
4222 
4223     // Do the GEP.
4224     CharUnits EltAlign =
4225       getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
4226     llvm::Value *EltPtr =
4227         emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx,
4228                               false, SignedIndices, E->getExprLoc());
4229     Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
4230   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
4231     // If this is A[i] where A is an array, the frontend will have decayed the
4232     // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
4233     // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4234     // "gep x, i" here.  Emit one "gep A, 0, i".
4235     assert(Array->getType()->isArrayType() &&
4236            "Array to pointer decay must have array source type!");
4237     LValue ArrayLV;
4238     // For simple multidimensional array indexing, set the 'accessed' flag for
4239     // better bounds-checking of the base expression.
4240     if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4241       ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
4242     else
4243       ArrayLV = EmitLValue(Array);
4244     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4245 
4246     if (SanOpts.has(SanitizerKind::ArrayBounds)) {
4247       // If the array being accessed has a "counted_by" attribute, generate
4248       // bounds checking code. The "count" field is at the top level of the
4249       // struct or in an anonymous struct, that's also at the top level. Future
4250       // expansions may allow the "count" to reside at any place in the struct,
4251       // but the value of "counted_by" will be a "simple" path to the count,
4252       // i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or
4253       // similar to emit the correct GEP.
4254       const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
4255           getLangOpts().getStrictFlexArraysLevel();
4256 
4257       if (const auto *ME = dyn_cast<MemberExpr>(Array);
4258           ME &&
4259           ME->isFlexibleArrayMemberLike(getContext(), StrictFlexArraysLevel) &&
4260           ME->getMemberDecl()->hasAttr<CountedByAttr>()) {
4261         const FieldDecl *FAMDecl = dyn_cast<FieldDecl>(ME->getMemberDecl());
4262         if (const FieldDecl *CountFD = FindCountedByField(FAMDecl)) {
4263           if (std::optional<int64_t> Diff =
4264                   getOffsetDifferenceInBits(*this, CountFD, FAMDecl)) {
4265             CharUnits OffsetDiff = CGM.getContext().toCharUnitsFromBits(*Diff);
4266 
4267             // Create a GEP with a byte offset between the FAM and count and
4268             // use that to load the count value.
4269             Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(
4270                 ArrayLV.getAddress(*this), Int8PtrTy, Int8Ty);
4271 
4272             llvm::Type *CountTy = ConvertType(CountFD->getType());
4273             llvm::Value *Res = Builder.CreateInBoundsGEP(
4274                 Int8Ty, Addr.getPointer(),
4275                 Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep");
4276             Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(),
4277                                             ".counted_by.load");
4278 
4279             // Now emit the bounds checking.
4280             EmitBoundsCheckImpl(E, Res, Idx, E->getIdx()->getType(),
4281                                 Array->getType(), Accessed);
4282           }
4283         }
4284       }
4285     }
4286 
4287     // Propagate the alignment from the array itself to the result.
4288     QualType arrayType = Array->getType();
4289     Addr = emitArraySubscriptGEP(
4290         *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
4291         E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices,
4292         E->getExprLoc(), &arrayType, E->getBase());
4293     EltBaseInfo = ArrayLV.getBaseInfo();
4294     EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
4295   } else {
4296     // The base must be a pointer; emit it with an estimate of its alignment.
4297     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
4298     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
4299     QualType ptrType = E->getBase()->getType();
4300     Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
4301                                  !getLangOpts().isSignedOverflowDefined(),
4302                                  SignedIndices, E->getExprLoc(), &ptrType,
4303                                  E->getBase());
4304   }
4305 
4306   LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
4307 
4308   if (getLangOpts().ObjC &&
4309       getLangOpts().getGC() != LangOptions::NonGC) {
4310     LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
4311     setObjCGCLValueClass(getContext(), E, LV);
4312   }
4313   return LV;
4314 }
4315 
4316 LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
4317   assert(
4318       !E->isIncomplete() &&
4319       "incomplete matrix subscript expressions should be rejected during Sema");
4320   LValue Base = EmitLValue(E->getBase());
4321   llvm::Value *RowIdx = EmitScalarExpr(E->getRowIdx());
4322   llvm::Value *ColIdx = EmitScalarExpr(E->getColumnIdx());
4323   llvm::Value *NumRows = Builder.getIntN(
4324       RowIdx->getType()->getScalarSizeInBits(),
4325       E->getBase()->getType()->castAs<ConstantMatrixType>()->getNumRows());
4326   llvm::Value *FinalIdx =
4327       Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx);
4328   return LValue::MakeMatrixElt(
4329       MaybeConvertMatrixAddress(Base.getAddress(*this), *this), FinalIdx,
4330       E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo());
4331 }
4332 
4333 static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
4334                                        LValueBaseInfo &BaseInfo,
4335                                        TBAAAccessInfo &TBAAInfo,
4336                                        QualType BaseTy, QualType ElTy,
4337                                        bool IsLowerBound) {
4338   LValue BaseLVal;
4339   if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) {
4340     BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound);
4341     if (BaseTy->isArrayType()) {
4342       Address Addr = BaseLVal.getAddress(CGF);
4343       BaseInfo = BaseLVal.getBaseInfo();
4344 
4345       // If the array type was an incomplete type, we need to make sure
4346       // the decay ends up being the right type.
4347       llvm::Type *NewTy = CGF.ConvertType(BaseTy);
4348       Addr = Addr.withElementType(NewTy);
4349 
4350       // Note that VLA pointers are always decayed, so we don't need to do
4351       // anything here.
4352       if (!BaseTy->isVariableArrayType()) {
4353         assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
4354                "Expected pointer to array");
4355         Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
4356       }
4357 
4358       return Addr.withElementType(CGF.ConvertTypeForMem(ElTy));
4359     }
4360     LValueBaseInfo TypeBaseInfo;
4361     TBAAAccessInfo TypeTBAAInfo;
4362     CharUnits Align =
4363         CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo);
4364     BaseInfo.mergeForCast(TypeBaseInfo);
4365     TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo);
4366     return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)),
4367                    CGF.ConvertTypeForMem(ElTy), Align);
4368   }
4369   return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
4370 }
4371 
4372 LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
4373                                                 bool IsLowerBound) {
4374   QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(E->getBase());
4375   QualType ResultExprTy;
4376   if (auto *AT = getContext().getAsArrayType(BaseTy))
4377     ResultExprTy = AT->getElementType();
4378   else
4379     ResultExprTy = BaseTy->getPointeeType();
4380   llvm::Value *Idx = nullptr;
4381   if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
4382     // Requesting lower bound or upper bound, but without provided length and
4383     // without ':' symbol for the default length -> length = 1.
4384     // Idx = LowerBound ?: 0;
4385     if (auto *LowerBound = E->getLowerBound()) {
4386       Idx = Builder.CreateIntCast(
4387           EmitScalarExpr(LowerBound), IntPtrTy,
4388           LowerBound->getType()->hasSignedIntegerRepresentation());
4389     } else
4390       Idx = llvm::ConstantInt::getNullValue(IntPtrTy);
4391   } else {
4392     // Try to emit length or lower bound as constant. If this is possible, 1
4393     // is subtracted from constant length or lower bound. Otherwise, emit LLVM
4394     // IR (LB + Len) - 1.
4395     auto &C = CGM.getContext();
4396     auto *Length = E->getLength();
4397     llvm::APSInt ConstLength;
4398     if (Length) {
4399       // Idx = LowerBound + Length - 1;
4400       if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) {
4401         ConstLength = CL->zextOrTrunc(PointerWidthInBits);
4402         Length = nullptr;
4403       }
4404       auto *LowerBound = E->getLowerBound();
4405       llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
4406       if (LowerBound) {
4407         if (std::optional<llvm::APSInt> LB =
4408                 LowerBound->getIntegerConstantExpr(C)) {
4409           ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits);
4410           LowerBound = nullptr;
4411         }
4412       }
4413       if (!Length)
4414         --ConstLength;
4415       else if (!LowerBound)
4416         --ConstLowerBound;
4417 
4418       if (Length || LowerBound) {
4419         auto *LowerBoundVal =
4420             LowerBound
4421                 ? Builder.CreateIntCast(
4422                       EmitScalarExpr(LowerBound), IntPtrTy,
4423                       LowerBound->getType()->hasSignedIntegerRepresentation())
4424                 : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound);
4425         auto *LengthVal =
4426             Length
4427                 ? Builder.CreateIntCast(
4428                       EmitScalarExpr(Length), IntPtrTy,
4429                       Length->getType()->hasSignedIntegerRepresentation())
4430                 : llvm::ConstantInt::get(IntPtrTy, ConstLength);
4431         Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",
4432                                 /*HasNUW=*/false,
4433                                 !getLangOpts().isSignedOverflowDefined());
4434         if (Length && LowerBound) {
4435           Idx = Builder.CreateSub(
4436               Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",
4437               /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4438         }
4439       } else
4440         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);
4441     } else {
4442       // Idx = ArraySize - 1;
4443       QualType ArrayTy = BaseTy->isPointerType()
4444                              ? E->getBase()->IgnoreParenImpCasts()->getType()
4445                              : BaseTy;
4446       if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) {
4447         Length = VAT->getSizeExpr();
4448         if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) {
4449           ConstLength = *L;
4450           Length = nullptr;
4451         }
4452       } else {
4453         auto *CAT = C.getAsConstantArrayType(ArrayTy);
4454         assert(CAT && "unexpected type for array initializer");
4455         ConstLength = CAT->getSize();
4456       }
4457       if (Length) {
4458         auto *LengthVal = Builder.CreateIntCast(
4459             EmitScalarExpr(Length), IntPtrTy,
4460             Length->getType()->hasSignedIntegerRepresentation());
4461         Idx = Builder.CreateSub(
4462             LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",
4463             /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4464       } else {
4465         ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
4466         --ConstLength;
4467         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength);
4468       }
4469     }
4470   }
4471   assert(Idx);
4472 
4473   Address EltPtr = Address::invalid();
4474   LValueBaseInfo BaseInfo;
4475   TBAAAccessInfo TBAAInfo;
4476   if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) {
4477     // The base must be a pointer, which is not an aggregate.  Emit
4478     // it.  It needs to be emitted first in case it's what captures
4479     // the VLA bounds.
4480     Address Base =
4481         emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo,
4482                                 BaseTy, VLA->getElementType(), IsLowerBound);
4483     // The element count here is the total number of non-VLA elements.
4484     llvm::Value *NumElements = getVLASize(VLA).NumElts;
4485 
4486     // Effectively, the multiply by the VLA size is part of the GEP.
4487     // GEP indexes are signed, and scaling an index isn't permitted to
4488     // signed-overflow, so we use the same semantics for our explicit
4489     // multiply.  We suppress this if overflow is not undefined behavior.
4490     if (getLangOpts().isSignedOverflowDefined())
4491       Idx = Builder.CreateMul(Idx, NumElements);
4492     else
4493       Idx = Builder.CreateNSWMul(Idx, NumElements);
4494     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),
4495                                    !getLangOpts().isSignedOverflowDefined(),
4496                                    /*signedIndices=*/false, E->getExprLoc());
4497   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
4498     // If this is A[i] where A is an array, the frontend will have decayed the
4499     // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
4500     // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4501     // "gep x, i" here.  Emit one "gep A, 0, i".
4502     assert(Array->getType()->isArrayType() &&
4503            "Array to pointer decay must have array source type!");
4504     LValue ArrayLV;
4505     // For simple multidimensional array indexing, set the 'accessed' flag for
4506     // better bounds-checking of the base expression.
4507     if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4508       ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
4509     else
4510       ArrayLV = EmitLValue(Array);
4511 
4512     // Propagate the alignment from the array itself to the result.
4513     EltPtr = emitArraySubscriptGEP(
4514         *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
4515         ResultExprTy, !getLangOpts().isSignedOverflowDefined(),
4516         /*signedIndices=*/false, E->getExprLoc());
4517     BaseInfo = ArrayLV.getBaseInfo();
4518     TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
4519   } else {
4520     Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo,
4521                                            TBAAInfo, BaseTy, ResultExprTy,
4522                                            IsLowerBound);
4523     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
4524                                    !getLangOpts().isSignedOverflowDefined(),
4525                                    /*signedIndices=*/false, E->getExprLoc());
4526   }
4527 
4528   return MakeAddrLValue(EltPtr, ResultExprTy, BaseInfo, TBAAInfo);
4529 }
4530 
4531 LValue CodeGenFunction::
4532 EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
4533   // Emit the base vector as an l-value.
4534   LValue Base;
4535 
4536   // ExtVectorElementExpr's base can either be a vector or pointer to vector.
4537   if (E->isArrow()) {
4538     // If it is a pointer to a vector, emit the address and form an lvalue with
4539     // it.
4540     LValueBaseInfo BaseInfo;
4541     TBAAAccessInfo TBAAInfo;
4542     Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo);
4543     const auto *PT = E->getBase()->getType()->castAs<PointerType>();
4544     Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo);
4545     Base.getQuals().removeObjCGCAttr();
4546   } else if (E->getBase()->isGLValue()) {
4547     // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
4548     // emit the base as an lvalue.
4549     assert(E->getBase()->getType()->isVectorType());
4550     Base = EmitLValue(E->getBase());
4551   } else {
4552     // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
4553     assert(E->getBase()->getType()->isVectorType() &&
4554            "Result must be a vector");
4555     llvm::Value *Vec = EmitScalarExpr(E->getBase());
4556 
4557     // Store the vector to memory (because LValue wants an address).
4558     Address VecMem = CreateMemTemp(E->getBase()->getType());
4559     Builder.CreateStore(Vec, VecMem);
4560     Base = MakeAddrLValue(VecMem, E->getBase()->getType(),
4561                           AlignmentSource::Decl);
4562   }
4563 
4564   QualType type =
4565     E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
4566 
4567   // Encode the element access list into a vector of unsigned indices.
4568   SmallVector<uint32_t, 4> Indices;
4569   E->getEncodedElementAccess(Indices);
4570 
4571   if (Base.isSimple()) {
4572     llvm::Constant *CV =
4573         llvm::ConstantDataVector::get(getLLVMContext(), Indices);
4574     return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type,
4575                                     Base.getBaseInfo(), TBAAAccessInfo());
4576   }
4577   assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
4578 
4579   llvm::Constant *BaseElts = Base.getExtVectorElts();
4580   SmallVector<llvm::Constant *, 4> CElts;
4581 
4582   for (unsigned i = 0, e = Indices.size(); i != e; ++i)
4583     CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
4584   llvm::Constant *CV = llvm::ConstantVector::get(CElts);
4585   return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,
4586                                   Base.getBaseInfo(), TBAAAccessInfo());
4587 }
4588 
4589 LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
4590   if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, E)) {
4591     EmitIgnoredExpr(E->getBase());
4592     return EmitDeclRefLValue(DRE);
4593   }
4594 
4595   Expr *BaseExpr = E->getBase();
4596   // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
4597   LValue BaseLV;
4598   if (E->isArrow()) {
4599     LValueBaseInfo BaseInfo;
4600     TBAAAccessInfo TBAAInfo;
4601     Address Addr = EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
4602     QualType PtrTy = BaseExpr->getType()->getPointeeType();
4603     SanitizerSet SkippedChecks;
4604     bool IsBaseCXXThis = IsWrappedCXXThis(BaseExpr);
4605     if (IsBaseCXXThis)
4606       SkippedChecks.set(SanitizerKind::Alignment, true);
4607     if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
4608       SkippedChecks.set(SanitizerKind::Null, true);
4609     EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy,
4610                   /*Alignment=*/CharUnits::Zero(), SkippedChecks);
4611     BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
4612   } else
4613     BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
4614 
4615   NamedDecl *ND = E->getMemberDecl();
4616   if (auto *Field = dyn_cast<FieldDecl>(ND)) {
4617     LValue LV = EmitLValueForField(BaseLV, Field);
4618     setObjCGCLValueClass(getContext(), E, LV);
4619     if (getLangOpts().OpenMP) {
4620       // If the member was explicitly marked as nontemporal, mark it as
4621       // nontemporal. If the base lvalue is marked as nontemporal, mark access
4622       // to children as nontemporal too.
4623       if ((IsWrappedCXXThis(BaseExpr) &&
4624            CGM.getOpenMPRuntime().isNontemporalDecl(Field)) ||
4625           BaseLV.isNontemporal())
4626         LV.setNontemporal(/*Value=*/true);
4627     }
4628     return LV;
4629   }
4630 
4631   if (const auto *FD = dyn_cast<FunctionDecl>(ND))
4632     return EmitFunctionDeclLValue(*this, E, FD);
4633 
4634   llvm_unreachable("Unhandled member declaration!");
4635 }
4636 
4637 /// Given that we are currently emitting a lambda, emit an l-value for
4638 /// one of its members.
4639 ///
4640 LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
4641                                                  llvm::Value *ThisValue) {
4642   bool HasExplicitObjectParameter = false;
4643   if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(CurCodeDecl)) {
4644     HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction();
4645     assert(MD->getParent()->isLambda());
4646     assert(MD->getParent() == Field->getParent());
4647   }
4648   LValue LambdaLV;
4649   if (HasExplicitObjectParameter) {
4650     const VarDecl *D = cast<CXXMethodDecl>(CurCodeDecl)->getParamDecl(0);
4651     auto It = LocalDeclMap.find(D);
4652     assert(It != LocalDeclMap.end() && "explicit parameter not loaded?");
4653     Address AddrOfExplicitObject = It->getSecond();
4654     if (D->getType()->isReferenceType())
4655       LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(),
4656                                            AlignmentSource::Decl);
4657     else
4658       LambdaLV = MakeNaturalAlignAddrLValue(AddrOfExplicitObject.getPointer(),
4659                                             D->getType().getNonReferenceType());
4660   } else {
4661     QualType LambdaTagType = getContext().getTagDeclType(Field->getParent());
4662     LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType);
4663   }
4664   return EmitLValueForField(LambdaLV, Field);
4665 }
4666 
4667 LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
4668   return EmitLValueForLambdaField(Field, CXXABIThisValue);
4669 }
4670 
4671 /// Get the field index in the debug info. The debug info structure/union
4672 /// will ignore the unnamed bitfields.
4673 unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
4674                                              unsigned FieldIndex) {
4675   unsigned I = 0, Skipped = 0;
4676 
4677   for (auto *F : Rec->getDefinition()->fields()) {
4678     if (I == FieldIndex)
4679       break;
4680     if (F->isUnnamedBitfield())
4681       Skipped++;
4682     I++;
4683   }
4684 
4685   return FieldIndex - Skipped;
4686 }
4687 
4688 /// Get the address of a zero-sized field within a record. The resulting
4689 /// address doesn't necessarily have the right type.
4690 static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
4691                                        const FieldDecl *Field) {
4692   CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
4693       CGF.getContext().getFieldOffset(Field));
4694   if (Offset.isZero())
4695     return Base;
4696   Base = Base.withElementType(CGF.Int8Ty);
4697   return CGF.Builder.CreateConstInBoundsByteGEP(Base, Offset);
4698 }
4699 
4700 /// Drill down to the storage of a field without walking into
4701 /// reference types.
4702 ///
4703 /// The resulting address doesn't necessarily have the right type.
4704 static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
4705                                       const FieldDecl *field) {
4706   if (field->isZeroSize(CGF.getContext()))
4707     return emitAddrOfZeroSizeField(CGF, base, field);
4708 
4709   const RecordDecl *rec = field->getParent();
4710 
4711   unsigned idx =
4712     CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4713 
4714   return CGF.Builder.CreateStructGEP(base, idx, field->getName());
4715 }
4716 
4717 static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
4718                                         Address addr, const FieldDecl *field) {
4719   const RecordDecl *rec = field->getParent();
4720   llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
4721       base.getType(), rec->getLocation());
4722 
4723   unsigned idx =
4724       CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4725 
4726   return CGF.Builder.CreatePreserveStructAccessIndex(
4727       addr, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo);
4728 }
4729 
4730 static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
4731   const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
4732   if (!RD)
4733     return false;
4734 
4735   if (RD->isDynamicClass())
4736     return true;
4737 
4738   for (const auto &Base : RD->bases())
4739     if (hasAnyVptr(Base.getType(), Context))
4740       return true;
4741 
4742   for (const FieldDecl *Field : RD->fields())
4743     if (hasAnyVptr(Field->getType(), Context))
4744       return true;
4745 
4746   return false;
4747 }
4748 
4749 LValue CodeGenFunction::EmitLValueForField(LValue base,
4750                                            const FieldDecl *field) {
4751   LValueBaseInfo BaseInfo = base.getBaseInfo();
4752 
4753   if (field->isBitField()) {
4754     const CGRecordLayout &RL =
4755         CGM.getTypes().getCGRecordLayout(field->getParent());
4756     const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
4757     const bool UseVolatile = isAAPCS(CGM.getTarget()) &&
4758                              CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
4759                              Info.VolatileStorageSize != 0 &&
4760                              field->getType()
4761                                  .withCVRQualifiers(base.getVRQualifiers())
4762                                  .isVolatileQualified();
4763     Address Addr = base.getAddress(*this);
4764     unsigned Idx = RL.getLLVMFieldNo(field);
4765     const RecordDecl *rec = field->getParent();
4766     if (hasBPFPreserveStaticOffset(rec))
4767       Addr = wrapWithBPFPreserveStaticOffset(*this, Addr);
4768     if (!UseVolatile) {
4769       if (!IsInPreservedAIRegion &&
4770           (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4771         if (Idx != 0)
4772           // For structs, we GEP to the field that the record layout suggests.
4773           Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
4774       } else {
4775         llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
4776             getContext().getRecordType(rec), rec->getLocation());
4777         Addr = Builder.CreatePreserveStructAccessIndex(
4778             Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()),
4779             DbgInfo);
4780       }
4781     }
4782     const unsigned SS =
4783         UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
4784     // Get the access type.
4785     llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS);
4786     Addr = Addr.withElementType(FieldIntTy);
4787     if (UseVolatile) {
4788       const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
4789       if (VolatileOffset)
4790         Addr = Builder.CreateConstInBoundsGEP(Addr, VolatileOffset);
4791     }
4792 
4793     QualType fieldType =
4794         field->getType().withCVRQualifiers(base.getVRQualifiers());
4795     // TODO: Support TBAA for bit fields.
4796     LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
4797     return LValue::MakeBitfield(Addr, Info, fieldType, FieldBaseInfo,
4798                                 TBAAAccessInfo());
4799   }
4800 
4801   // Fields of may-alias structures are may-alias themselves.
4802   // FIXME: this should get propagated down through anonymous structs
4803   // and unions.
4804   QualType FieldType = field->getType();
4805   const RecordDecl *rec = field->getParent();
4806   AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
4807   LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));
4808   TBAAAccessInfo FieldTBAAInfo;
4809   if (base.getTBAAInfo().isMayAlias() ||
4810           rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
4811     FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4812   } else if (rec->isUnion()) {
4813     // TODO: Support TBAA for unions.
4814     FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4815   } else {
4816     // If no base type been assigned for the base access, then try to generate
4817     // one for this base lvalue.
4818     FieldTBAAInfo = base.getTBAAInfo();
4819     if (!FieldTBAAInfo.BaseType) {
4820         FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(base.getType());
4821         assert(!FieldTBAAInfo.Offset &&
4822                "Nonzero offset for an access with no base type!");
4823     }
4824 
4825     // Adjust offset to be relative to the base type.
4826     const ASTRecordLayout &Layout =
4827         getContext().getASTRecordLayout(field->getParent());
4828     unsigned CharWidth = getContext().getCharWidth();
4829     if (FieldTBAAInfo.BaseType)
4830       FieldTBAAInfo.Offset +=
4831           Layout.getFieldOffset(field->getFieldIndex()) / CharWidth;
4832 
4833     // Update the final access type and size.
4834     FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(FieldType);
4835     FieldTBAAInfo.Size =
4836         getContext().getTypeSizeInChars(FieldType).getQuantity();
4837   }
4838 
4839   Address addr = base.getAddress(*this);
4840   if (hasBPFPreserveStaticOffset(rec))
4841     addr = wrapWithBPFPreserveStaticOffset(*this, addr);
4842   if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) {
4843     if (CGM.getCodeGenOpts().StrictVTablePointers &&
4844         ClassDef->isDynamicClass()) {
4845       // Getting to any field of dynamic object requires stripping dynamic
4846       // information provided by invariant.group.  This is because accessing
4847       // fields may leak the real address of dynamic object, which could result
4848       // in miscompilation when leaked pointer would be compared.
4849       auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
4850       addr = Address(stripped, addr.getElementType(), addr.getAlignment());
4851     }
4852   }
4853 
4854   unsigned RecordCVR = base.getVRQualifiers();
4855   if (rec->isUnion()) {
4856     // For unions, there is no pointer adjustment.
4857     if (CGM.getCodeGenOpts().StrictVTablePointers &&
4858         hasAnyVptr(FieldType, getContext()))
4859       // Because unions can easily skip invariant.barriers, we need to add
4860       // a barrier every time CXXRecord field with vptr is referenced.
4861       addr = Builder.CreateLaunderInvariantGroup(addr);
4862 
4863     if (IsInPreservedAIRegion ||
4864         (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4865       // Remember the original union field index
4866       llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),
4867           rec->getLocation());
4868       addr = Address(
4869           Builder.CreatePreserveUnionAccessIndex(
4870               addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
4871           addr.getElementType(), addr.getAlignment());
4872     }
4873 
4874     if (FieldType->isReferenceType())
4875       addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType));
4876   } else {
4877     if (!IsInPreservedAIRegion &&
4878         (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
4879       // For structs, we GEP to the field that the record layout suggests.
4880       addr = emitAddrOfFieldStorage(*this, addr, field);
4881     else
4882       // Remember the original struct field index
4883       addr = emitPreserveStructAccess(*this, base, addr, field);
4884   }
4885 
4886   // If this is a reference field, load the reference right now.
4887   if (FieldType->isReferenceType()) {
4888     LValue RefLVal =
4889         MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4890     if (RecordCVR & Qualifiers::Volatile)
4891       RefLVal.getQuals().addVolatile();
4892     addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo);
4893 
4894     // Qualifiers on the struct don't apply to the referencee.
4895     RecordCVR = 0;
4896     FieldType = FieldType->getPointeeType();
4897   }
4898 
4899   // Make sure that the address is pointing to the right type.  This is critical
4900   // for both unions and structs.
4901   addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType));
4902 
4903   if (field->hasAttr<AnnotateAttr>())
4904     addr = EmitFieldAnnotations(field, addr);
4905 
4906   LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4907   LV.getQuals().addCVRQualifiers(RecordCVR);
4908 
4909   // __weak attribute on a field is ignored.
4910   if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
4911     LV.getQuals().removeObjCGCAttr();
4912 
4913   return LV;
4914 }
4915 
4916 LValue
4917 CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
4918                                                   const FieldDecl *Field) {
4919   QualType FieldType = Field->getType();
4920 
4921   if (!FieldType->isReferenceType())
4922     return EmitLValueForField(Base, Field);
4923 
4924   Address V = emitAddrOfFieldStorage(*this, Base.getAddress(*this), Field);
4925 
4926   // Make sure that the address is pointing to the right type.
4927   llvm::Type *llvmType = ConvertTypeForMem(FieldType);
4928   V = V.withElementType(llvmType);
4929 
4930   // TODO: Generate TBAA information that describes this access as a structure
4931   // member access and not just an access to an object of the field's type. This
4932   // should be similar to what we do in EmitLValueForField().
4933   LValueBaseInfo BaseInfo = Base.getBaseInfo();
4934   AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
4935   LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(FieldAlignSource));
4936   return MakeAddrLValue(V, FieldType, FieldBaseInfo,
4937                         CGM.getTBAAInfoForSubobject(Base, FieldType));
4938 }
4939 
4940 LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
4941   if (E->isFileScope()) {
4942     ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
4943     return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl);
4944   }
4945   if (E->getType()->isVariablyModifiedType())
4946     // make sure to emit the VLA size.
4947     EmitVariablyModifiedType(E->getType());
4948 
4949   Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
4950   const Expr *InitExpr = E->getInitializer();
4951   LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl);
4952 
4953   EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
4954                    /*Init*/ true);
4955 
4956   // Block-scope compound literals are destroyed at the end of the enclosing
4957   // scope in C.
4958   if (!getLangOpts().CPlusPlus)
4959     if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
4960       pushLifetimeExtendedDestroy(getCleanupKind(DtorKind), DeclPtr,
4961                                   E->getType(), getDestroyer(DtorKind),
4962                                   DtorKind & EHCleanup);
4963 
4964   return Result;
4965 }
4966 
4967 LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
4968   if (!E->isGLValue())
4969     // Initializing an aggregate temporary in C++11: T{...}.
4970     return EmitAggExprToLValue(E);
4971 
4972   // An lvalue initializer list must be initializing a reference.
4973   assert(E->isTransparent() && "non-transparent glvalue init list");
4974   return EmitLValue(E->getInit(0));
4975 }
4976 
4977 /// Emit the operand of a glvalue conditional operator. This is either a glvalue
4978 /// or a (possibly-parenthesized) throw-expression. If this is a throw, no
4979 /// LValue is returned and the current block has been terminated.
4980 static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
4981                                                          const Expr *Operand) {
4982   if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Operand->IgnoreParens())) {
4983     CGF.EmitCXXThrowExpr(ThrowExpr, /*KeepInsertionPoint*/false);
4984     return std::nullopt;
4985   }
4986 
4987   return CGF.EmitLValue(Operand);
4988 }
4989 
4990 namespace {
4991 // Handle the case where the condition is a constant evaluatable simple integer,
4992 // which means we don't have to separately handle the true/false blocks.
4993 std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
4994     CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
4995   const Expr *condExpr = E->getCond();
4996   bool CondExprBool;
4997   if (CGF.ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
4998     const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
4999     if (!CondExprBool)
5000       std::swap(Live, Dead);
5001 
5002     if (!CGF.ContainsLabel(Dead)) {
5003       // If the true case is live, we need to track its region.
5004       if (CondExprBool)
5005         CGF.incrementProfileCounter(E);
5006       // If a throw expression we emit it and return an undefined lvalue
5007       // because it can't be used.
5008       if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Live->IgnoreParens())) {
5009         CGF.EmitCXXThrowExpr(ThrowExpr);
5010         llvm::Type *ElemTy = CGF.ConvertType(Dead->getType());
5011         llvm::Type *Ty = CGF.UnqualPtrTy;
5012         return CGF.MakeAddrLValue(
5013             Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()),
5014             Dead->getType());
5015       }
5016       return CGF.EmitLValue(Live);
5017     }
5018   }
5019   return std::nullopt;
5020 }
5021 struct ConditionalInfo {
5022   llvm::BasicBlock *lhsBlock, *rhsBlock;
5023   std::optional<LValue> LHS, RHS;
5024 };
5025 
5026 // Create and generate the 3 blocks for a conditional operator.
5027 // Leaves the 'current block' in the continuation basic block.
5028 template<typename FuncTy>
5029 ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
5030                                       const AbstractConditionalOperator *E,
5031                                       const FuncTy &BranchGenFunc) {
5032   ConditionalInfo Info{CGF.createBasicBlock("cond.true"),
5033                        CGF.createBasicBlock("cond.false"), std::nullopt,
5034                        std::nullopt};
5035   llvm::BasicBlock *endBlock = CGF.createBasicBlock("cond.end");
5036 
5037   CodeGenFunction::ConditionalEvaluation eval(CGF);
5038   CGF.EmitBranchOnBoolExpr(E->getCond(), Info.lhsBlock, Info.rhsBlock,
5039                            CGF.getProfileCount(E));
5040 
5041   // Any temporaries created here are conditional.
5042   CGF.EmitBlock(Info.lhsBlock);
5043   CGF.incrementProfileCounter(E);
5044   eval.begin(CGF);
5045   Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
5046   eval.end(CGF);
5047   Info.lhsBlock = CGF.Builder.GetInsertBlock();
5048 
5049   if (Info.LHS)
5050     CGF.Builder.CreateBr(endBlock);
5051 
5052   // Any temporaries created here are conditional.
5053   CGF.EmitBlock(Info.rhsBlock);
5054   eval.begin(CGF);
5055   Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
5056   eval.end(CGF);
5057   Info.rhsBlock = CGF.Builder.GetInsertBlock();
5058   CGF.EmitBlock(endBlock);
5059 
5060   return Info;
5061 }
5062 } // namespace
5063 
5064 void CodeGenFunction::EmitIgnoredConditionalOperator(
5065     const AbstractConditionalOperator *E) {
5066   if (!E->isGLValue()) {
5067     // ?: here should be an aggregate.
5068     assert(hasAggregateEvaluationKind(E->getType()) &&
5069            "Unexpected conditional operator!");
5070     return (void)EmitAggExprToLValue(E);
5071   }
5072 
5073   OpaqueValueMapping binding(*this, E);
5074   if (HandleConditionalOperatorLValueSimpleCase(*this, E))
5075     return;
5076 
5077   EmitConditionalBlocks(*this, E, [](CodeGenFunction &CGF, const Expr *E) {
5078     CGF.EmitIgnoredExpr(E);
5079     return LValue{};
5080   });
5081 }
5082 LValue CodeGenFunction::EmitConditionalOperatorLValue(
5083     const AbstractConditionalOperator *expr) {
5084   if (!expr->isGLValue()) {
5085     // ?: here should be an aggregate.
5086     assert(hasAggregateEvaluationKind(expr->getType()) &&
5087            "Unexpected conditional operator!");
5088     return EmitAggExprToLValue(expr);
5089   }
5090 
5091   OpaqueValueMapping binding(*this, expr);
5092   if (std::optional<LValue> Res =
5093           HandleConditionalOperatorLValueSimpleCase(*this, expr))
5094     return *Res;
5095 
5096   ConditionalInfo Info = EmitConditionalBlocks(
5097       *this, expr, [](CodeGenFunction &CGF, const Expr *E) {
5098         return EmitLValueOrThrowExpression(CGF, E);
5099       });
5100 
5101   if ((Info.LHS && !Info.LHS->isSimple()) ||
5102       (Info.RHS && !Info.RHS->isSimple()))
5103     return EmitUnsupportedLValue(expr, "conditional operator");
5104 
5105   if (Info.LHS && Info.RHS) {
5106     Address lhsAddr = Info.LHS->getAddress(*this);
5107     Address rhsAddr = Info.RHS->getAddress(*this);
5108     llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue");
5109     phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock);
5110     phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock);
5111     Address result(phi, lhsAddr.getElementType(),
5112                    std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
5113     AlignmentSource alignSource =
5114         std::max(Info.LHS->getBaseInfo().getAlignmentSource(),
5115                  Info.RHS->getBaseInfo().getAlignmentSource());
5116     TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
5117         Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo());
5118     return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),
5119                           TBAAInfo);
5120   } else {
5121     assert((Info.LHS || Info.RHS) &&
5122            "both operands of glvalue conditional are throw-expressions?");
5123     return Info.LHS ? *Info.LHS : *Info.RHS;
5124   }
5125 }
5126 
5127 /// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
5128 /// type. If the cast is to a reference, we can have the usual lvalue result,
5129 /// otherwise if a cast is needed by the code generator in an lvalue context,
5130 /// then it must mean that we need the address of an aggregate in order to
5131 /// access one of its members.  This can happen for all the reasons that casts
5132 /// are permitted with aggregate result, including noop aggregate casts, and
5133 /// cast from scalar to union.
5134 LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
5135   switch (E->getCastKind()) {
5136   case CK_ToVoid:
5137   case CK_BitCast:
5138   case CK_LValueToRValueBitCast:
5139   case CK_ArrayToPointerDecay:
5140   case CK_FunctionToPointerDecay:
5141   case CK_NullToMemberPointer:
5142   case CK_NullToPointer:
5143   case CK_IntegralToPointer:
5144   case CK_PointerToIntegral:
5145   case CK_PointerToBoolean:
5146   case CK_IntegralCast:
5147   case CK_BooleanToSignedIntegral:
5148   case CK_IntegralToBoolean:
5149   case CK_IntegralToFloating:
5150   case CK_FloatingToIntegral:
5151   case CK_FloatingToBoolean:
5152   case CK_FloatingCast:
5153   case CK_FloatingRealToComplex:
5154   case CK_FloatingComplexToReal:
5155   case CK_FloatingComplexToBoolean:
5156   case CK_FloatingComplexCast:
5157   case CK_FloatingComplexToIntegralComplex:
5158   case CK_IntegralRealToComplex:
5159   case CK_IntegralComplexToReal:
5160   case CK_IntegralComplexToBoolean:
5161   case CK_IntegralComplexCast:
5162   case CK_IntegralComplexToFloatingComplex:
5163   case CK_DerivedToBaseMemberPointer:
5164   case CK_BaseToDerivedMemberPointer:
5165   case CK_MemberPointerToBoolean:
5166   case CK_ReinterpretMemberPointer:
5167   case CK_AnyPointerToBlockPointerCast:
5168   case CK_ARCProduceObject:
5169   case CK_ARCConsumeObject:
5170   case CK_ARCReclaimReturnedObject:
5171   case CK_ARCExtendBlockObject:
5172   case CK_CopyAndAutoreleaseBlockObject:
5173   case CK_IntToOCLSampler:
5174   case CK_FloatingToFixedPoint:
5175   case CK_FixedPointToFloating:
5176   case CK_FixedPointCast:
5177   case CK_FixedPointToBoolean:
5178   case CK_FixedPointToIntegral:
5179   case CK_IntegralToFixedPoint:
5180   case CK_MatrixCast:
5181     return EmitUnsupportedLValue(E, "unexpected cast lvalue");
5182 
5183   case CK_Dependent:
5184     llvm_unreachable("dependent cast kind in IR gen!");
5185 
5186   case CK_BuiltinFnToFnPtr:
5187     llvm_unreachable("builtin functions are handled elsewhere");
5188 
5189   // These are never l-values; just use the aggregate emission code.
5190   case CK_NonAtomicToAtomic:
5191   case CK_AtomicToNonAtomic:
5192     return EmitAggExprToLValue(E);
5193 
5194   case CK_Dynamic: {
5195     LValue LV = EmitLValue(E->getSubExpr());
5196     Address V = LV.getAddress(*this);
5197     const auto *DCE = cast<CXXDynamicCastExpr>(E);
5198     return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType());
5199   }
5200 
5201   case CK_ConstructorConversion:
5202   case CK_UserDefinedConversion:
5203   case CK_CPointerToObjCPointerCast:
5204   case CK_BlockPointerToObjCPointerCast:
5205   case CK_LValueToRValue:
5206     return EmitLValue(E->getSubExpr());
5207 
5208   case CK_NoOp: {
5209     // CK_NoOp can model a qualification conversion, which can remove an array
5210     // bound and change the IR type.
5211     // FIXME: Once pointee types are removed from IR, remove this.
5212     LValue LV = EmitLValue(E->getSubExpr());
5213     // Propagate the volatile qualifer to LValue, if exist in E.
5214     if (E->changesVolatileQualification())
5215       LV.getQuals() = E->getType().getQualifiers();
5216     if (LV.isSimple()) {
5217       Address V = LV.getAddress(*this);
5218       if (V.isValid()) {
5219         llvm::Type *T = ConvertTypeForMem(E->getType());
5220         if (V.getElementType() != T)
5221           LV.setAddress(V.withElementType(T));
5222       }
5223     }
5224     return LV;
5225   }
5226 
5227   case CK_UncheckedDerivedToBase:
5228   case CK_DerivedToBase: {
5229     const auto *DerivedClassTy =
5230         E->getSubExpr()->getType()->castAs<RecordType>();
5231     auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
5232 
5233     LValue LV = EmitLValue(E->getSubExpr());
5234     Address This = LV.getAddress(*this);
5235 
5236     // Perform the derived-to-base conversion
5237     Address Base = GetAddressOfBaseClass(
5238         This, DerivedClassDecl, E->path_begin(), E->path_end(),
5239         /*NullCheckValue=*/false, E->getExprLoc());
5240 
5241     // TODO: Support accesses to members of base classes in TBAA. For now, we
5242     // conservatively pretend that the complete object is of the base class
5243     // type.
5244     return MakeAddrLValue(Base, E->getType(), LV.getBaseInfo(),
5245                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
5246   }
5247   case CK_ToUnion:
5248     return EmitAggExprToLValue(E);
5249   case CK_BaseToDerived: {
5250     const auto *DerivedClassTy = E->getType()->castAs<RecordType>();
5251     auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
5252 
5253     LValue LV = EmitLValue(E->getSubExpr());
5254 
5255     // Perform the base-to-derived conversion
5256     Address Derived = GetAddressOfDerivedClass(
5257         LV.getAddress(*this), DerivedClassDecl, E->path_begin(), E->path_end(),
5258         /*NullCheckValue=*/false);
5259 
5260     // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
5261     // performed and the object is not of the derived type.
5262     if (sanitizePerformTypeCheck())
5263       EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(),
5264                     Derived.getPointer(), E->getType());
5265 
5266     if (SanOpts.has(SanitizerKind::CFIDerivedCast))
5267       EmitVTablePtrCheckForCast(E->getType(), Derived,
5268                                 /*MayBeNull=*/false, CFITCK_DerivedCast,
5269                                 E->getBeginLoc());
5270 
5271     return MakeAddrLValue(Derived, E->getType(), LV.getBaseInfo(),
5272                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
5273   }
5274   case CK_LValueBitCast: {
5275     // This must be a reinterpret_cast (or c-style equivalent).
5276     const auto *CE = cast<ExplicitCastExpr>(E);
5277 
5278     CGM.EmitExplicitCastExprType(CE, this);
5279     LValue LV = EmitLValue(E->getSubExpr());
5280     Address V = LV.getAddress(*this).withElementType(
5281         ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType()));
5282 
5283     if (SanOpts.has(SanitizerKind::CFIUnrelatedCast))
5284       EmitVTablePtrCheckForCast(E->getType(), V,
5285                                 /*MayBeNull=*/false, CFITCK_UnrelatedCast,
5286                                 E->getBeginLoc());
5287 
5288     return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
5289                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
5290   }
5291   case CK_AddressSpaceConversion: {
5292     LValue LV = EmitLValue(E->getSubExpr());
5293     QualType DestTy = getContext().getPointerType(E->getType());
5294     llvm::Value *V = getTargetHooks().performAddrSpaceCast(
5295         *this, LV.getPointer(*this),
5296         E->getSubExpr()->getType().getAddressSpace(),
5297         E->getType().getAddressSpace(), ConvertType(DestTy));
5298     return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()),
5299                                   LV.getAddress(*this).getAlignment()),
5300                           E->getType(), LV.getBaseInfo(), LV.getTBAAInfo());
5301   }
5302   case CK_ObjCObjectLValueCast: {
5303     LValue LV = EmitLValue(E->getSubExpr());
5304     Address V = LV.getAddress(*this).withElementType(ConvertType(E->getType()));
5305     return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
5306                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
5307   }
5308   case CK_ZeroToOCLOpaqueType:
5309     llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
5310 
5311   case CK_VectorSplat: {
5312     // LValue results of vector splats are only supported in HLSL.
5313     if (!getLangOpts().HLSL)
5314       return EmitUnsupportedLValue(E, "unexpected cast lvalue");
5315     return EmitLValue(E->getSubExpr());
5316   }
5317   }
5318 
5319   llvm_unreachable("Unhandled lvalue cast kind?");
5320 }
5321 
5322 LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
5323   assert(OpaqueValueMappingData::shouldBindAsLValue(e));
5324   return getOrCreateOpaqueLValueMapping(e);
5325 }
5326 
5327 LValue
5328 CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
5329   assert(OpaqueValueMapping::shouldBindAsLValue(e));
5330 
5331   llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
5332       it = OpaqueLValues.find(e);
5333 
5334   if (it != OpaqueLValues.end())
5335     return it->second;
5336 
5337   assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
5338   return EmitLValue(e->getSourceExpr());
5339 }
5340 
5341 RValue
5342 CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
5343   assert(!OpaqueValueMapping::shouldBindAsLValue(e));
5344 
5345   llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
5346       it = OpaqueRValues.find(e);
5347 
5348   if (it != OpaqueRValues.end())
5349     return it->second;
5350 
5351   assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
5352   return EmitAnyExpr(e->getSourceExpr());
5353 }
5354 
5355 RValue CodeGenFunction::EmitRValueForField(LValue LV,
5356                                            const FieldDecl *FD,
5357                                            SourceLocation Loc) {
5358   QualType FT = FD->getType();
5359   LValue FieldLV = EmitLValueForField(LV, FD);
5360   switch (getEvaluationKind(FT)) {
5361   case TEK_Complex:
5362     return RValue::getComplex(EmitLoadOfComplex(FieldLV, Loc));
5363   case TEK_Aggregate:
5364     return FieldLV.asAggregateRValue(*this);
5365   case TEK_Scalar:
5366     // This routine is used to load fields one-by-one to perform a copy, so
5367     // don't load reference fields.
5368     if (FD->getType()->isReferenceType())
5369       return RValue::get(FieldLV.getPointer(*this));
5370     // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
5371     // primitive load.
5372     if (FieldLV.isBitField())
5373       return EmitLoadOfLValue(FieldLV, Loc);
5374     return RValue::get(EmitLoadOfScalar(FieldLV, Loc));
5375   }
5376   llvm_unreachable("bad evaluation kind");
5377 }
5378 
5379 //===--------------------------------------------------------------------===//
5380 //                             Expression Emission
5381 //===--------------------------------------------------------------------===//
5382 
5383 RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
5384                                      ReturnValueSlot ReturnValue) {
5385   // Builtins never have block type.
5386   if (E->getCallee()->getType()->isBlockPointerType())
5387     return EmitBlockCallExpr(E, ReturnValue);
5388 
5389   if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E))
5390     return EmitCXXMemberCallExpr(CE, ReturnValue);
5391 
5392   if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E))
5393     return EmitCUDAKernelCallExpr(CE, ReturnValue);
5394 
5395   // A CXXOperatorCallExpr is created even for explicit object methods, but
5396   // these should be treated like static function call.
5397   if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E))
5398     if (const auto *MD =
5399             dyn_cast_if_present<CXXMethodDecl>(CE->getCalleeDecl());
5400         MD && MD->isImplicitObjectMemberFunction())
5401       return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
5402 
5403   CGCallee callee = EmitCallee(E->getCallee());
5404 
5405   if (callee.isBuiltin()) {
5406     return EmitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(),
5407                            E, ReturnValue);
5408   }
5409 
5410   if (callee.isPseudoDestructor()) {
5411     return EmitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr());
5412   }
5413 
5414   return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue);
5415 }
5416 
5417 /// Emit a CallExpr without considering whether it might be a subclass.
5418 RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
5419                                            ReturnValueSlot ReturnValue) {
5420   CGCallee Callee = EmitCallee(E->getCallee());
5421   return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue);
5422 }
5423 
5424 // Detect the unusual situation where an inline version is shadowed by a
5425 // non-inline version. In that case we should pick the external one
5426 // everywhere. That's GCC behavior too.
5427 static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
5428   for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
5429     if (!PD->isInlineBuiltinDeclaration())
5430       return false;
5431   return true;
5432 }
5433 
5434 static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
5435   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
5436 
5437   if (auto builtinID = FD->getBuiltinID()) {
5438     std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
5439     std::string NoBuiltins = "no-builtins";
5440 
5441     StringRef Ident = CGF.CGM.getMangledName(GD);
5442     std::string FDInlineName = (Ident + ".inline").str();
5443 
5444     bool IsPredefinedLibFunction =
5445         CGF.getContext().BuiltinInfo.isPredefinedLibFunction(builtinID);
5446     bool HasAttributeNoBuiltin =
5447         CGF.CurFn->getAttributes().hasFnAttr(NoBuiltinFD) ||
5448         CGF.CurFn->getAttributes().hasFnAttr(NoBuiltins);
5449 
5450     // When directing calling an inline builtin, call it through it's mangled
5451     // name to make it clear it's not the actual builtin.
5452     if (CGF.CurFn->getName() != FDInlineName &&
5453         OnlyHasInlineBuiltinDeclaration(FD)) {
5454       llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
5455       llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr);
5456       llvm::Module *M = Fn->getParent();
5457       llvm::Function *Clone = M->getFunction(FDInlineName);
5458       if (!Clone) {
5459         Clone = llvm::Function::Create(Fn->getFunctionType(),
5460                                        llvm::GlobalValue::InternalLinkage,
5461                                        Fn->getAddressSpace(), FDInlineName, M);
5462         Clone->addFnAttr(llvm::Attribute::AlwaysInline);
5463       }
5464       return CGCallee::forDirect(Clone, GD);
5465     }
5466 
5467     // Replaceable builtins provide their own implementation of a builtin. If we
5468     // are in an inline builtin implementation, avoid trivial infinite
5469     // recursion. Honor __attribute__((no_builtin("foo"))) or
5470     // __attribute__((no_builtin)) on the current function unless foo is
5471     // not a predefined library function which means we must generate the
5472     // builtin no matter what.
5473     else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
5474       return CGCallee::forBuiltin(builtinID, FD);
5475   }
5476 
5477   llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
5478   if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
5479       FD->hasAttr<CUDAGlobalAttr>())
5480     CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
5481         cast<llvm::GlobalValue>(CalleePtr->stripPointerCasts()));
5482 
5483   return CGCallee::forDirect(CalleePtr, GD);
5484 }
5485 
5486 CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
5487   E = E->IgnoreParens();
5488 
5489   // Look through function-to-pointer decay.
5490   if (auto ICE = dyn_cast<ImplicitCastExpr>(E)) {
5491     if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
5492         ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
5493       return EmitCallee(ICE->getSubExpr());
5494     }
5495 
5496   // Resolve direct calls.
5497   } else if (auto DRE = dyn_cast<DeclRefExpr>(E)) {
5498     if (auto FD = dyn_cast<FunctionDecl>(DRE->getDecl())) {
5499       return EmitDirectCallee(*this, FD);
5500     }
5501   } else if (auto ME = dyn_cast<MemberExpr>(E)) {
5502     if (auto FD = dyn_cast<FunctionDecl>(ME->getMemberDecl())) {
5503       EmitIgnoredExpr(ME->getBase());
5504       return EmitDirectCallee(*this, FD);
5505     }
5506 
5507   // Look through template substitutions.
5508   } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {
5509     return EmitCallee(NTTP->getReplacement());
5510 
5511   // Treat pseudo-destructor calls differently.
5512   } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(E)) {
5513     return CGCallee::forPseudoDestructor(PDE);
5514   }
5515 
5516   // Otherwise, we have an indirect reference.
5517   llvm::Value *calleePtr;
5518   QualType functionType;
5519   if (auto ptrType = E->getType()->getAs<PointerType>()) {
5520     calleePtr = EmitScalarExpr(E);
5521     functionType = ptrType->getPointeeType();
5522   } else {
5523     functionType = E->getType();
5524     calleePtr = EmitLValue(E, KnownNonNull).getPointer(*this);
5525   }
5526   assert(functionType->isFunctionType());
5527 
5528   GlobalDecl GD;
5529   if (const auto *VD =
5530           dyn_cast_or_null<VarDecl>(E->getReferencedDeclOfCallee()))
5531     GD = GlobalDecl(VD);
5532 
5533   CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD);
5534   CGCallee callee(calleeInfo, calleePtr);
5535   return callee;
5536 }
5537 
5538 LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
5539   // Comma expressions just emit their LHS then their RHS as an l-value.
5540   if (E->getOpcode() == BO_Comma) {
5541     EmitIgnoredExpr(E->getLHS());
5542     EnsureInsertPoint();
5543     return EmitLValue(E->getRHS());
5544   }
5545 
5546   if (E->getOpcode() == BO_PtrMemD ||
5547       E->getOpcode() == BO_PtrMemI)
5548     return EmitPointerToDataMemberBinaryExpr(E);
5549 
5550   assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
5551 
5552   // Note that in all of these cases, __block variables need the RHS
5553   // evaluated first just in case the variable gets moved by the RHS.
5554 
5555   switch (getEvaluationKind(E->getType())) {
5556   case TEK_Scalar: {
5557     switch (E->getLHS()->getType().getObjCLifetime()) {
5558     case Qualifiers::OCL_Strong:
5559       return EmitARCStoreStrong(E, /*ignored*/ false).first;
5560 
5561     case Qualifiers::OCL_Autoreleasing:
5562       return EmitARCStoreAutoreleasing(E).first;
5563 
5564     // No reason to do any of these differently.
5565     case Qualifiers::OCL_None:
5566     case Qualifiers::OCL_ExplicitNone:
5567     case Qualifiers::OCL_Weak:
5568       break;
5569     }
5570 
5571     RValue RV = EmitAnyExpr(E->getRHS());
5572     LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);
5573     if (RV.isScalar())
5574       EmitNullabilityCheck(LV, RV.getScalarVal(), E->getExprLoc());
5575     EmitStoreThroughLValue(RV, LV);
5576     if (getLangOpts().OpenMP)
5577       CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
5578                                                                 E->getLHS());
5579     return LV;
5580   }
5581 
5582   case TEK_Complex:
5583     return EmitComplexAssignmentLValue(E);
5584 
5585   case TEK_Aggregate:
5586     return EmitAggExprToLValue(E);
5587   }
5588   llvm_unreachable("bad evaluation kind");
5589 }
5590 
5591 LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
5592   RValue RV = EmitCallExpr(E);
5593 
5594   if (!RV.isScalar())
5595     return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5596                           AlignmentSource::Decl);
5597 
5598   assert(E->getCallReturnType(getContext())->isReferenceType() &&
5599          "Can't have a scalar return unless the return type is a "
5600          "reference type!");
5601 
5602   return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
5603 }
5604 
5605 LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
5606   // FIXME: This shouldn't require another copy.
5607   return EmitAggExprToLValue(E);
5608 }
5609 
5610 LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
5611   assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
5612          && "binding l-value to type which needs a temporary");
5613   AggValueSlot Slot = CreateAggTemp(E->getType());
5614   EmitCXXConstructExpr(E, Slot);
5615   return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5616 }
5617 
5618 LValue
5619 CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
5620   return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType());
5621 }
5622 
5623 Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
5624   return CGM.GetAddrOfMSGuidDecl(E->getGuidDecl())
5625       .withElementType(ConvertType(E->getType()));
5626 }
5627 
5628 LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
5629   return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(),
5630                         AlignmentSource::Decl);
5631 }
5632 
5633 LValue
5634 CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
5635   AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
5636   Slot.setExternallyDestructed();
5637   EmitAggExpr(E->getSubExpr(), Slot);
5638   EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddress());
5639   return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5640 }
5641 
5642 LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
5643   RValue RV = EmitObjCMessageExpr(E);
5644 
5645   if (!RV.isScalar())
5646     return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5647                           AlignmentSource::Decl);
5648 
5649   assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
5650          "Can't have a scalar return unless the return type is a "
5651          "reference type!");
5652 
5653   return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
5654 }
5655 
5656 LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
5657   Address V =
5658     CGM.getObjCRuntime().GetAddrOfSelector(*this, E->getSelector());
5659   return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl);
5660 }
5661 
5662 llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
5663                                              const ObjCIvarDecl *Ivar) {
5664   return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);
5665 }
5666 
5667 llvm::Value *
5668 CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
5669                                              const ObjCIvarDecl *Ivar) {
5670   llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
5671   QualType PointerDiffType = getContext().getPointerDiffType();
5672   return Builder.CreateZExtOrTrunc(OffsetValue,
5673                                    getTypes().ConvertType(PointerDiffType));
5674 }
5675 
5676 LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
5677                                           llvm::Value *BaseValue,
5678                                           const ObjCIvarDecl *Ivar,
5679                                           unsigned CVRQualifiers) {
5680   return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,
5681                                                    Ivar, CVRQualifiers);
5682 }
5683 
5684 LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
5685   // FIXME: A lot of the code below could be shared with EmitMemberExpr.
5686   llvm::Value *BaseValue = nullptr;
5687   const Expr *BaseExpr = E->getBase();
5688   Qualifiers BaseQuals;
5689   QualType ObjectTy;
5690   if (E->isArrow()) {
5691     BaseValue = EmitScalarExpr(BaseExpr);
5692     ObjectTy = BaseExpr->getType()->getPointeeType();
5693     BaseQuals = ObjectTy.getQualifiers();
5694   } else {
5695     LValue BaseLV = EmitLValue(BaseExpr);
5696     BaseValue = BaseLV.getPointer(*this);
5697     ObjectTy = BaseExpr->getType();
5698     BaseQuals = ObjectTy.getQualifiers();
5699   }
5700 
5701   LValue LV =
5702     EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),
5703                       BaseQuals.getCVRQualifiers());
5704   setObjCGCLValueClass(getContext(), E, LV);
5705   return LV;
5706 }
5707 
5708 LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
5709   // Can only get l-value for message expression returning aggregate type
5710   RValue RV = EmitAnyExprToTemp(E);
5711   return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5712                         AlignmentSource::Decl);
5713 }
5714 
5715 RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee,
5716                                  const CallExpr *E, ReturnValueSlot ReturnValue,
5717                                  llvm::Value *Chain) {
5718   // Get the actual function type. The callee type will always be a pointer to
5719   // function type or a block pointer type.
5720   assert(CalleeType->isFunctionPointerType() &&
5721          "Call must have function pointer type!");
5722 
5723   const Decl *TargetDecl =
5724       OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
5725 
5726   assert((!isa_and_present<FunctionDecl>(TargetDecl) ||
5727           !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&
5728          "trying to emit a call to an immediate function");
5729 
5730   CalleeType = getContext().getCanonicalType(CalleeType);
5731 
5732   auto PointeeType = cast<PointerType>(CalleeType)->getPointeeType();
5733 
5734   CGCallee Callee = OrigCallee;
5735 
5736   if (SanOpts.has(SanitizerKind::Function) &&
5737       (!TargetDecl || !isa<FunctionDecl>(TargetDecl)) &&
5738       !isa<FunctionNoProtoType>(PointeeType)) {
5739     if (llvm::Constant *PrefixSig =
5740             CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
5741       SanitizerScope SanScope(this);
5742       auto *TypeHash = getUBSanFunctionTypeHash(PointeeType);
5743 
5744       llvm::Type *PrefixSigType = PrefixSig->getType();
5745       llvm::StructType *PrefixStructTy = llvm::StructType::get(
5746           CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true);
5747 
5748       llvm::Value *CalleePtr = Callee.getFunctionPointer();
5749 
5750       // On 32-bit Arm, the low bit of a function pointer indicates whether
5751       // it's using the Arm or Thumb instruction set. The actual first
5752       // instruction lives at the same address either way, so we must clear
5753       // that low bit before using the function address to find the prefix
5754       // structure.
5755       //
5756       // This applies to both Arm and Thumb target triples, because
5757       // either one could be used in an interworking context where it
5758       // might be passed function pointers of both types.
5759       llvm::Value *AlignedCalleePtr;
5760       if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {
5761         llvm::Value *CalleeAddress =
5762             Builder.CreatePtrToInt(CalleePtr, IntPtrTy);
5763         llvm::Value *Mask = llvm::ConstantInt::get(IntPtrTy, ~1);
5764         llvm::Value *AlignedCalleeAddress =
5765             Builder.CreateAnd(CalleeAddress, Mask);
5766         AlignedCalleePtr =
5767             Builder.CreateIntToPtr(AlignedCalleeAddress, CalleePtr->getType());
5768       } else {
5769         AlignedCalleePtr = CalleePtr;
5770       }
5771 
5772       llvm::Value *CalleePrefixStruct = AlignedCalleePtr;
5773       llvm::Value *CalleeSigPtr =
5774           Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, -1, 0);
5775       llvm::Value *CalleeSig =
5776           Builder.CreateAlignedLoad(PrefixSigType, CalleeSigPtr, getIntAlign());
5777       llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig);
5778 
5779       llvm::BasicBlock *Cont = createBasicBlock("cont");
5780       llvm::BasicBlock *TypeCheck = createBasicBlock("typecheck");
5781       Builder.CreateCondBr(CalleeSigMatch, TypeCheck, Cont);
5782 
5783       EmitBlock(TypeCheck);
5784       llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(
5785           Int32Ty,
5786           Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, -1, 1),
5787           getPointerAlign());
5788       llvm::Value *CalleeTypeHashMatch =
5789           Builder.CreateICmpEQ(CalleeTypeHash, TypeHash);
5790       llvm::Constant *StaticData[] = {EmitCheckSourceLocation(E->getBeginLoc()),
5791                                       EmitCheckTypeDescriptor(CalleeType)};
5792       EmitCheck(std::make_pair(CalleeTypeHashMatch, SanitizerKind::Function),
5793                 SanitizerHandler::FunctionTypeMismatch, StaticData,
5794                 {CalleePtr});
5795 
5796       Builder.CreateBr(Cont);
5797       EmitBlock(Cont);
5798     }
5799   }
5800 
5801   const auto *FnType = cast<FunctionType>(PointeeType);
5802 
5803   // If we are checking indirect calls and this call is indirect, check that the
5804   // function pointer is a member of the bit set for the function type.
5805   if (SanOpts.has(SanitizerKind::CFIICall) &&
5806       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5807     SanitizerScope SanScope(this);
5808     EmitSanitizerStatReport(llvm::SanStat_CFI_ICall);
5809 
5810     llvm::Metadata *MD;
5811     if (CGM.getCodeGenOpts().SanitizeCfiICallGeneralizePointers)
5812       MD = CGM.CreateMetadataIdentifierGeneralized(QualType(FnType, 0));
5813     else
5814       MD = CGM.CreateMetadataIdentifierForType(QualType(FnType, 0));
5815 
5816     llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
5817 
5818     llvm::Value *CalleePtr = Callee.getFunctionPointer();
5819     llvm::Value *TypeTest = Builder.CreateCall(
5820         CGM.getIntrinsic(llvm::Intrinsic::type_test), {CalleePtr, TypeId});
5821 
5822     auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
5823     llvm::Constant *StaticData[] = {
5824         llvm::ConstantInt::get(Int8Ty, CFITCK_ICall),
5825         EmitCheckSourceLocation(E->getBeginLoc()),
5826         EmitCheckTypeDescriptor(QualType(FnType, 0)),
5827     };
5828     if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
5829       EmitCfiSlowPathCheck(SanitizerKind::CFIICall, TypeTest, CrossDsoTypeId,
5830                            CalleePtr, StaticData);
5831     } else {
5832       EmitCheck(std::make_pair(TypeTest, SanitizerKind::CFIICall),
5833                 SanitizerHandler::CFICheckFail, StaticData,
5834                 {CalleePtr, llvm::UndefValue::get(IntPtrTy)});
5835     }
5836   }
5837 
5838   CallArgList Args;
5839   if (Chain)
5840     Args.add(RValue::get(Chain), CGM.getContext().VoidPtrTy);
5841 
5842   // C++17 requires that we evaluate arguments to a call using assignment syntax
5843   // right-to-left, and that we evaluate arguments to certain other operators
5844   // left-to-right. Note that we allow this to override the order dictated by
5845   // the calling convention on the MS ABI, which means that parameter
5846   // destruction order is not necessarily reverse construction order.
5847   // FIXME: Revisit this based on C++ committee response to unimplementability.
5848   EvaluationOrder Order = EvaluationOrder::Default;
5849   bool StaticOperator = false;
5850   if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) {
5851     if (OCE->isAssignmentOp())
5852       Order = EvaluationOrder::ForceRightToLeft;
5853     else {
5854       switch (OCE->getOperator()) {
5855       case OO_LessLess:
5856       case OO_GreaterGreater:
5857       case OO_AmpAmp:
5858       case OO_PipePipe:
5859       case OO_Comma:
5860       case OO_ArrowStar:
5861         Order = EvaluationOrder::ForceLeftToRight;
5862         break;
5863       default:
5864         break;
5865       }
5866     }
5867 
5868     if (const auto *MD =
5869             dyn_cast_if_present<CXXMethodDecl>(OCE->getCalleeDecl());
5870         MD && MD->isStatic())
5871       StaticOperator = true;
5872   }
5873 
5874   auto Arguments = E->arguments();
5875   if (StaticOperator) {
5876     // If we're calling a static operator, we need to emit the object argument
5877     // and ignore it.
5878     EmitIgnoredExpr(E->getArg(0));
5879     Arguments = drop_begin(Arguments, 1);
5880   }
5881   EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), Arguments,
5882                E->getDirectCallee(), /*ParamsToSkip=*/0, Order);
5883 
5884   const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
5885       Args, FnType, /*ChainCall=*/Chain);
5886 
5887   // C99 6.5.2.2p6:
5888   //   If the expression that denotes the called function has a type
5889   //   that does not include a prototype, [the default argument
5890   //   promotions are performed]. If the number of arguments does not
5891   //   equal the number of parameters, the behavior is undefined. If
5892   //   the function is defined with a type that includes a prototype,
5893   //   and either the prototype ends with an ellipsis (, ...) or the
5894   //   types of the arguments after promotion are not compatible with
5895   //   the types of the parameters, the behavior is undefined. If the
5896   //   function is defined with a type that does not include a
5897   //   prototype, and the types of the arguments after promotion are
5898   //   not compatible with those of the parameters after promotion,
5899   //   the behavior is undefined [except in some trivial cases].
5900   // That is, in the general case, we should assume that a call
5901   // through an unprototyped function type works like a *non-variadic*
5902   // call.  The way we make this work is to cast to the exact type
5903   // of the promoted arguments.
5904   //
5905   // Chain calls use this same code path to add the invisible chain parameter
5906   // to the function type.
5907   if (isa<FunctionNoProtoType>(FnType) || Chain) {
5908     llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
5909     int AS = Callee.getFunctionPointer()->getType()->getPointerAddressSpace();
5910     CalleeTy = CalleeTy->getPointerTo(AS);
5911 
5912     llvm::Value *CalleePtr = Callee.getFunctionPointer();
5913     CalleePtr = Builder.CreateBitCast(CalleePtr, CalleeTy, "callee.knr.cast");
5914     Callee.setFunctionPointer(CalleePtr);
5915   }
5916 
5917   // HIP function pointer contains kernel handle when it is used in triple
5918   // chevron. The kernel stub needs to be loaded from kernel handle and used
5919   // as callee.
5920   if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
5921       isa<CUDAKernelCallExpr>(E) &&
5922       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5923     llvm::Value *Handle = Callee.getFunctionPointer();
5924     auto *Stub = Builder.CreateLoad(
5925         Address(Handle, Handle->getType(), CGM.getPointerAlign()));
5926     Callee.setFunctionPointer(Stub);
5927   }
5928   llvm::CallBase *CallOrInvoke = nullptr;
5929   RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &CallOrInvoke,
5930                          E == MustTailCall, E->getExprLoc());
5931 
5932   // Generate function declaration DISuprogram in order to be used
5933   // in debug info about call sites.
5934   if (CGDebugInfo *DI = getDebugInfo()) {
5935     if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
5936       FunctionArgList Args;
5937       QualType ResTy = BuildFunctionArgList(CalleeDecl, Args);
5938       DI->EmitFuncDeclForCallSite(CallOrInvoke,
5939                                   DI->getFunctionType(CalleeDecl, ResTy, Args),
5940                                   CalleeDecl);
5941     }
5942   }
5943 
5944   return Call;
5945 }
5946 
5947 LValue CodeGenFunction::
5948 EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
5949   Address BaseAddr = Address::invalid();
5950   if (E->getOpcode() == BO_PtrMemI) {
5951     BaseAddr = EmitPointerWithAlignment(E->getLHS());
5952   } else {
5953     BaseAddr = EmitLValue(E->getLHS()).getAddress(*this);
5954   }
5955 
5956   llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
5957   const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
5958 
5959   LValueBaseInfo BaseInfo;
5960   TBAAAccessInfo TBAAInfo;
5961   Address MemberAddr =
5962     EmitCXXMemberDataPointerAddress(E, BaseAddr, OffsetV, MPT, &BaseInfo,
5963                                     &TBAAInfo);
5964 
5965   return MakeAddrLValue(MemberAddr, MPT->getPointeeType(), BaseInfo, TBAAInfo);
5966 }
5967 
5968 /// Given the address of a temporary variable, produce an r-value of
5969 /// its type.
5970 RValue CodeGenFunction::convertTempToRValue(Address addr,
5971                                             QualType type,
5972                                             SourceLocation loc) {
5973   LValue lvalue = MakeAddrLValue(addr, type, AlignmentSource::Decl);
5974   switch (getEvaluationKind(type)) {
5975   case TEK_Complex:
5976     return RValue::getComplex(EmitLoadOfComplex(lvalue, loc));
5977   case TEK_Aggregate:
5978     return lvalue.asAggregateRValue(*this);
5979   case TEK_Scalar:
5980     return RValue::get(EmitLoadOfScalar(lvalue, loc));
5981   }
5982   llvm_unreachable("bad evaluation kind");
5983 }
5984 
5985 void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
5986   assert(Val->getType()->isFPOrFPVectorTy());
5987   if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
5988     return;
5989 
5990   llvm::MDBuilder MDHelper(getLLVMContext());
5991   llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
5992 
5993   cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node);
5994 }
5995 
5996 void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {
5997   llvm::Type *EltTy = Val->getType()->getScalarType();
5998   if (!EltTy->isFloatTy())
5999     return;
6000 
6001   if ((getLangOpts().OpenCL &&
6002        !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
6003       (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
6004        !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
6005     // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 3ulp
6006     //
6007     // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
6008     // build option allows an application to specify that single precision
6009     // floating-point divide (x/y and 1/x) and sqrt used in the program
6010     // source are correctly rounded.
6011     //
6012     // TODO: CUDA has a prec-sqrt flag
6013     SetFPAccuracy(Val, 3.0f);
6014   }
6015 }
6016 
6017 void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {
6018   llvm::Type *EltTy = Val->getType()->getScalarType();
6019   if (!EltTy->isFloatTy())
6020     return;
6021 
6022   if ((getLangOpts().OpenCL &&
6023        !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
6024       (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
6025        !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
6026     // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5ulp
6027     //
6028     // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
6029     // build option allows an application to specify that single precision
6030     // floating-point divide (x/y and 1/x) and sqrt used in the program
6031     // source are correctly rounded.
6032     //
6033     // TODO: CUDA has a prec-div flag
6034     SetFPAccuracy(Val, 2.5f);
6035   }
6036 }
6037 
6038 namespace {
6039   struct LValueOrRValue {
6040     LValue LV;
6041     RValue RV;
6042   };
6043 }
6044 
6045 static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
6046                                            const PseudoObjectExpr *E,
6047                                            bool forLValue,
6048                                            AggValueSlot slot) {
6049   SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
6050 
6051   // Find the result expression, if any.
6052   const Expr *resultExpr = E->getResultExpr();
6053   LValueOrRValue result;
6054 
6055   for (PseudoObjectExpr::const_semantics_iterator
6056          i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
6057     const Expr *semantic = *i;
6058 
6059     // If this semantic expression is an opaque value, bind it
6060     // to the result of its source expression.
6061     if (const auto *ov = dyn_cast<OpaqueValueExpr>(semantic)) {
6062       // Skip unique OVEs.
6063       if (ov->isUnique()) {
6064         assert(ov != resultExpr &&
6065                "A unique OVE cannot be used as the result expression");
6066         continue;
6067       }
6068 
6069       // If this is the result expression, we may need to evaluate
6070       // directly into the slot.
6071       typedef CodeGenFunction::OpaqueValueMappingData OVMA;
6072       OVMA opaqueData;
6073       if (ov == resultExpr && ov->isPRValue() && !forLValue &&
6074           CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) {
6075         CGF.EmitAggExpr(ov->getSourceExpr(), slot);
6076         LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(),
6077                                        AlignmentSource::Decl);
6078         opaqueData = OVMA::bind(CGF, ov, LV);
6079         result.RV = slot.asRValue();
6080 
6081       // Otherwise, emit as normal.
6082       } else {
6083         opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr());
6084 
6085         // If this is the result, also evaluate the result now.
6086         if (ov == resultExpr) {
6087           if (forLValue)
6088             result.LV = CGF.EmitLValue(ov);
6089           else
6090             result.RV = CGF.EmitAnyExpr(ov, slot);
6091         }
6092       }
6093 
6094       opaques.push_back(opaqueData);
6095 
6096     // Otherwise, if the expression is the result, evaluate it
6097     // and remember the result.
6098     } else if (semantic == resultExpr) {
6099       if (forLValue)
6100         result.LV = CGF.EmitLValue(semantic);
6101       else
6102         result.RV = CGF.EmitAnyExpr(semantic, slot);
6103 
6104     // Otherwise, evaluate the expression in an ignored context.
6105     } else {
6106       CGF.EmitIgnoredExpr(semantic);
6107     }
6108   }
6109 
6110   // Unbind all the opaques now.
6111   for (unsigned i = 0, e = opaques.size(); i != e; ++i)
6112     opaques[i].unbind(CGF);
6113 
6114   return result;
6115 }
6116 
6117 RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
6118                                                AggValueSlot slot) {
6119   return emitPseudoObjectExpr(*this, E, false, slot).RV;
6120 }
6121 
6122 LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
6123   return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;
6124 }
6125