xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp (revision 5fb307d29b364982acbde82cbf77db3cae486f8c)
1 //===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Expr nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCUDARuntime.h"
14 #include "CGCXXABI.h"
15 #include "CGCall.h"
16 #include "CGCleanup.h"
17 #include "CGDebugInfo.h"
18 #include "CGObjCRuntime.h"
19 #include "CGOpenMPRuntime.h"
20 #include "CGRecordLayout.h"
21 #include "CodeGenFunction.h"
22 #include "CodeGenModule.h"
23 #include "ConstantEmitter.h"
24 #include "TargetInfo.h"
25 #include "clang/AST/ASTContext.h"
26 #include "clang/AST/Attr.h"
27 #include "clang/AST/DeclObjC.h"
28 #include "clang/AST/NSAPI.h"
29 #include "clang/Basic/Builtins.h"
30 #include "clang/Basic/CodeGenOptions.h"
31 #include "clang/Basic/SourceManager.h"
32 #include "llvm/ADT/Hashing.h"
33 #include "llvm/ADT/StringExtras.h"
34 #include "llvm/IR/DataLayout.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/IR/IntrinsicsWebAssembly.h"
37 #include "llvm/IR/LLVMContext.h"
38 #include "llvm/IR/MDBuilder.h"
39 #include "llvm/IR/MatrixBuilder.h"
40 #include "llvm/Passes/OptimizationLevel.h"
41 #include "llvm/Support/ConvertUTF.h"
42 #include "llvm/Support/MathExtras.h"
43 #include "llvm/Support/Path.h"
44 #include "llvm/Support/SaveAndRestore.h"
45 #include "llvm/Support/xxhash.h"
46 #include "llvm/Transforms/Utils/SanitizerStats.h"
47 
48 #include <optional>
49 #include <string>
50 
51 using namespace clang;
52 using namespace CodeGen;
53 
54 //===--------------------------------------------------------------------===//
55 //                        Miscellaneous Helper Methods
56 //===--------------------------------------------------------------------===//
57 
58 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
59 /// block.
60 Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
61                                                      CharUnits Align,
62                                                      const Twine &Name,
63                                                      llvm::Value *ArraySize) {
64   auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
65   Alloca->setAlignment(Align.getAsAlign());
66   return Address(Alloca, Ty, Align, KnownNonNull);
67 }
68 
69 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
70 /// block. The alloca is casted to default address space if necessary.
71 Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
72                                           const Twine &Name,
73                                           llvm::Value *ArraySize,
74                                           Address *AllocaAddr) {
75   auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
76   if (AllocaAddr)
77     *AllocaAddr = Alloca;
78   llvm::Value *V = Alloca.getPointer();
79   // Alloca always returns a pointer in alloca address space, which may
80   // be different from the type defined by the language. For example,
81   // in C++ the auto variables are in the default address space. Therefore
82   // cast alloca to the default address space when necessary.
83   if (getASTAllocaAddressSpace() != LangAS::Default) {
84     auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default);
85     llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
86     // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
87     // otherwise alloca is inserted at the current insertion point of the
88     // builder.
89     if (!ArraySize)
90       Builder.SetInsertPoint(getPostAllocaInsertPoint());
91     V = getTargetHooks().performAddrSpaceCast(
92         *this, V, getASTAllocaAddressSpace(), LangAS::Default,
93         Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
94   }
95 
96   return Address(V, Ty, Align, KnownNonNull);
97 }
98 
99 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
100 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
101 /// insertion point of the builder.
102 llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
103                                                     const Twine &Name,
104                                                     llvm::Value *ArraySize) {
105   if (ArraySize)
106     return Builder.CreateAlloca(Ty, ArraySize, Name);
107   return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
108                               ArraySize, Name, AllocaInsertPt);
109 }
110 
111 /// CreateDefaultAlignTempAlloca - This creates an alloca with the
112 /// default alignment of the corresponding LLVM type, which is *not*
113 /// guaranteed to be related in any way to the expected alignment of
114 /// an AST type that might have been lowered to Ty.
115 Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
116                                                       const Twine &Name) {
117   CharUnits Align =
118       CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty));
119   return CreateTempAlloca(Ty, Align, Name);
120 }
121 
122 Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
123   CharUnits Align = getContext().getTypeAlignInChars(Ty);
124   return CreateTempAlloca(ConvertType(Ty), Align, Name);
125 }
126 
127 Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
128                                        Address *Alloca) {
129   // FIXME: Should we prefer the preferred type alignment here?
130   return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);
131 }
132 
133 Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
134                                        const Twine &Name, Address *Alloca) {
135   Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,
136                                     /*ArraySize=*/nullptr, Alloca);
137 
138   if (Ty->isConstantMatrixType()) {
139     auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
140     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
141                                                 ArrayTy->getNumElements());
142 
143     Result = Address(
144         Builder.CreateBitCast(Result.getPointer(), VectorTy->getPointerTo()),
145         VectorTy, Result.getAlignment(), KnownNonNull);
146   }
147   return Result;
148 }
149 
150 Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align,
151                                                   const Twine &Name) {
152   return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);
153 }
154 
155 Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
156                                                   const Twine &Name) {
157   return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),
158                                   Name);
159 }
160 
161 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
162 /// expression and compare the result against zero, returning an Int1Ty value.
163 llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
164   PGO.setCurrentStmt(E);
165   if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
166     llvm::Value *MemPtr = EmitScalarExpr(E);
167     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
168   }
169 
170   QualType BoolTy = getContext().BoolTy;
171   SourceLocation Loc = E->getExprLoc();
172   CGFPOptionsRAII FPOptsRAII(*this, E);
173   if (!E->getType()->isAnyComplexType())
174     return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc);
175 
176   return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy,
177                                        Loc);
178 }
179 
180 /// EmitIgnoredExpr - Emit code to compute the specified expression,
181 /// ignoring the result.
182 void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
183   if (E->isPRValue())
184     return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true);
185 
186   // if this is a bitfield-resulting conditional operator, we can special case
187   // emit this. The normal 'EmitLValue' version of this is particularly
188   // difficult to codegen for, since creating a single "LValue" for two
189   // different sized arguments here is not particularly doable.
190   if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
191           E->IgnoreParenNoopCasts(getContext()))) {
192     if (CondOp->getObjectKind() == OK_BitField)
193       return EmitIgnoredConditionalOperator(CondOp);
194   }
195 
196   // Just emit it as an l-value and drop the result.
197   EmitLValue(E);
198 }
199 
200 /// EmitAnyExpr - Emit code to compute the specified expression which
201 /// can have any type.  The result is returned as an RValue struct.
202 /// If this is an aggregate expression, AggSlot indicates where the
203 /// result should be returned.
204 RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
205                                     AggValueSlot aggSlot,
206                                     bool ignoreResult) {
207   switch (getEvaluationKind(E->getType())) {
208   case TEK_Scalar:
209     return RValue::get(EmitScalarExpr(E, ignoreResult));
210   case TEK_Complex:
211     return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));
212   case TEK_Aggregate:
213     if (!ignoreResult && aggSlot.isIgnored())
214       aggSlot = CreateAggTemp(E->getType(), "agg-temp");
215     EmitAggExpr(E, aggSlot);
216     return aggSlot.asRValue();
217   }
218   llvm_unreachable("bad evaluation kind");
219 }
220 
221 /// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
222 /// always be accessible even if no aggregate location is provided.
223 RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
224   AggValueSlot AggSlot = AggValueSlot::ignored();
225 
226   if (hasAggregateEvaluationKind(E->getType()))
227     AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
228   return EmitAnyExpr(E, AggSlot);
229 }
230 
231 /// EmitAnyExprToMem - Evaluate an expression into a given memory
232 /// location.
233 void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
234                                        Address Location,
235                                        Qualifiers Quals,
236                                        bool IsInit) {
237   // FIXME: This function should take an LValue as an argument.
238   switch (getEvaluationKind(E->getType())) {
239   case TEK_Complex:
240     EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()),
241                               /*isInit*/ false);
242     return;
243 
244   case TEK_Aggregate: {
245     EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
246                                          AggValueSlot::IsDestructed_t(IsInit),
247                                          AggValueSlot::DoesNotNeedGCBarriers,
248                                          AggValueSlot::IsAliased_t(!IsInit),
249                                          AggValueSlot::MayOverlap));
250     return;
251   }
252 
253   case TEK_Scalar: {
254     RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
255     LValue LV = MakeAddrLValue(Location, E->getType());
256     EmitStoreThroughLValue(RV, LV);
257     return;
258   }
259   }
260   llvm_unreachable("bad evaluation kind");
261 }
262 
263 static void
264 pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
265                      const Expr *E, Address ReferenceTemporary) {
266   // Objective-C++ ARC:
267   //   If we are binding a reference to a temporary that has ownership, we
268   //   need to perform retain/release operations on the temporary.
269   //
270   // FIXME: This should be looking at E, not M.
271   if (auto Lifetime = M->getType().getObjCLifetime()) {
272     switch (Lifetime) {
273     case Qualifiers::OCL_None:
274     case Qualifiers::OCL_ExplicitNone:
275       // Carry on to normal cleanup handling.
276       break;
277 
278     case Qualifiers::OCL_Autoreleasing:
279       // Nothing to do; cleaned up by an autorelease pool.
280       return;
281 
282     case Qualifiers::OCL_Strong:
283     case Qualifiers::OCL_Weak:
284       switch (StorageDuration Duration = M->getStorageDuration()) {
285       case SD_Static:
286         // Note: we intentionally do not register a cleanup to release
287         // the object on program termination.
288         return;
289 
290       case SD_Thread:
291         // FIXME: We should probably register a cleanup in this case.
292         return;
293 
294       case SD_Automatic:
295       case SD_FullExpression:
296         CodeGenFunction::Destroyer *Destroy;
297         CleanupKind CleanupKind;
298         if (Lifetime == Qualifiers::OCL_Strong) {
299           const ValueDecl *VD = M->getExtendingDecl();
300           bool Precise =
301               VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>();
302           CleanupKind = CGF.getARCCleanupKind();
303           Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
304                             : &CodeGenFunction::destroyARCStrongImprecise;
305         } else {
306           // __weak objects always get EH cleanups; otherwise, exceptions
307           // could cause really nasty crashes instead of mere leaks.
308           CleanupKind = NormalAndEHCleanup;
309           Destroy = &CodeGenFunction::destroyARCWeak;
310         }
311         if (Duration == SD_FullExpression)
312           CGF.pushDestroy(CleanupKind, ReferenceTemporary,
313                           M->getType(), *Destroy,
314                           CleanupKind & EHCleanup);
315         else
316           CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,
317                                           M->getType(),
318                                           *Destroy, CleanupKind & EHCleanup);
319         return;
320 
321       case SD_Dynamic:
322         llvm_unreachable("temporary cannot have dynamic storage duration");
323       }
324       llvm_unreachable("unknown storage duration");
325     }
326   }
327 
328   CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
329   if (const RecordType *RT =
330           E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
331     // Get the destructor for the reference temporary.
332     auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
333     if (!ClassDecl->hasTrivialDestructor())
334       ReferenceTemporaryDtor = ClassDecl->getDestructor();
335   }
336 
337   if (!ReferenceTemporaryDtor)
338     return;
339 
340   // Call the destructor for the temporary.
341   switch (M->getStorageDuration()) {
342   case SD_Static:
343   case SD_Thread: {
344     llvm::FunctionCallee CleanupFn;
345     llvm::Constant *CleanupArg;
346     if (E->getType()->isArrayType()) {
347       CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
348           ReferenceTemporary, E->getType(),
349           CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions,
350           dyn_cast_or_null<VarDecl>(M->getExtendingDecl()));
351       CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);
352     } else {
353       CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
354           GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
355       CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
356     }
357     CGF.CGM.getCXXABI().registerGlobalDtor(
358         CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
359     break;
360   }
361 
362   case SD_FullExpression:
363     CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(),
364                     CodeGenFunction::destroyCXXObject,
365                     CGF.getLangOpts().Exceptions);
366     break;
367 
368   case SD_Automatic:
369     CGF.pushLifetimeExtendedDestroy(NormalAndEHCleanup,
370                                     ReferenceTemporary, E->getType(),
371                                     CodeGenFunction::destroyCXXObject,
372                                     CGF.getLangOpts().Exceptions);
373     break;
374 
375   case SD_Dynamic:
376     llvm_unreachable("temporary cannot have dynamic storage duration");
377   }
378 }
379 
380 static Address createReferenceTemporary(CodeGenFunction &CGF,
381                                         const MaterializeTemporaryExpr *M,
382                                         const Expr *Inner,
383                                         Address *Alloca = nullptr) {
384   auto &TCG = CGF.getTargetHooks();
385   switch (M->getStorageDuration()) {
386   case SD_FullExpression:
387   case SD_Automatic: {
388     // If we have a constant temporary array or record try to promote it into a
389     // constant global under the same rules a normal constant would've been
390     // promoted. This is easier on the optimizer and generally emits fewer
391     // instructions.
392     QualType Ty = Inner->getType();
393     if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
394         (Ty->isArrayType() || Ty->isRecordType()) &&
395         CGF.CGM.isTypeConstant(Ty, true, false))
396       if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) {
397         auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
398         auto *GV = new llvm::GlobalVariable(
399             CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
400             llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
401             llvm::GlobalValue::NotThreadLocal,
402             CGF.getContext().getTargetAddressSpace(AS));
403         CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
404         GV->setAlignment(alignment.getAsAlign());
405         llvm::Constant *C = GV;
406         if (AS != LangAS::Default)
407           C = TCG.performAddrSpaceCast(
408               CGF.CGM, GV, AS, LangAS::Default,
409               GV->getValueType()->getPointerTo(
410                   CGF.getContext().getTargetAddressSpace(LangAS::Default)));
411         // FIXME: Should we put the new global into a COMDAT?
412         return Address(C, GV->getValueType(), alignment);
413       }
414     return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
415   }
416   case SD_Thread:
417   case SD_Static:
418     return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);
419 
420   case SD_Dynamic:
421     llvm_unreachable("temporary can't have dynamic storage duration");
422   }
423   llvm_unreachable("unknown storage duration");
424 }
425 
426 /// Helper method to check if the underlying ABI is AAPCS
427 static bool isAAPCS(const TargetInfo &TargetInfo) {
428   return TargetInfo.getABI().startswith("aapcs");
429 }
430 
431 LValue CodeGenFunction::
432 EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
433   const Expr *E = M->getSubExpr();
434 
435   assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
436           !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
437          "Reference should never be pseudo-strong!");
438 
439   // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
440   // as that will cause the lifetime adjustment to be lost for ARC
441   auto ownership = M->getType().getObjCLifetime();
442   if (ownership != Qualifiers::OCL_None &&
443       ownership != Qualifiers::OCL_ExplicitNone) {
444     Address Object = createReferenceTemporary(*this, M, E);
445     if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
446       llvm::Type *Ty = ConvertTypeForMem(E->getType());
447       Object = Address(llvm::ConstantExpr::getBitCast(
448                            Var, Ty->getPointerTo(Object.getAddressSpace())),
449                        Ty, Object.getAlignment());
450 
451       // createReferenceTemporary will promote the temporary to a global with a
452       // constant initializer if it can.  It can only do this to a value of
453       // ARC-manageable type if the value is global and therefore "immune" to
454       // ref-counting operations.  Therefore we have no need to emit either a
455       // dynamic initialization or a cleanup and we can just return the address
456       // of the temporary.
457       if (Var->hasInitializer())
458         return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
459 
460       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
461     }
462     LValue RefTempDst = MakeAddrLValue(Object, M->getType(),
463                                        AlignmentSource::Decl);
464 
465     switch (getEvaluationKind(E->getType())) {
466     default: llvm_unreachable("expected scalar or aggregate expression");
467     case TEK_Scalar:
468       EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);
469       break;
470     case TEK_Aggregate: {
471       EmitAggExpr(E, AggValueSlot::forAddr(Object,
472                                            E->getType().getQualifiers(),
473                                            AggValueSlot::IsDestructed,
474                                            AggValueSlot::DoesNotNeedGCBarriers,
475                                            AggValueSlot::IsNotAliased,
476                                            AggValueSlot::DoesNotOverlap));
477       break;
478     }
479     }
480 
481     pushTemporaryCleanup(*this, M, E, Object);
482     return RefTempDst;
483   }
484 
485   SmallVector<const Expr *, 2> CommaLHSs;
486   SmallVector<SubobjectAdjustment, 2> Adjustments;
487   E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);
488 
489   for (const auto &Ignored : CommaLHSs)
490     EmitIgnoredExpr(Ignored);
491 
492   if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) {
493     if (opaque->getType()->isRecordType()) {
494       assert(Adjustments.empty());
495       return EmitOpaqueValueLValue(opaque);
496     }
497   }
498 
499   // Create and initialize the reference temporary.
500   Address Alloca = Address::invalid();
501   Address Object = createReferenceTemporary(*this, M, E, &Alloca);
502   if (auto *Var = dyn_cast<llvm::GlobalVariable>(
503           Object.getPointer()->stripPointerCasts())) {
504     llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
505     Object = Address(llvm::ConstantExpr::getBitCast(
506                          cast<llvm::Constant>(Object.getPointer()),
507                          TemporaryType->getPointerTo()),
508                      TemporaryType,
509                      Object.getAlignment());
510     // If the temporary is a global and has a constant initializer or is a
511     // constant temporary that we promoted to a global, we may have already
512     // initialized it.
513     if (!Var->hasInitializer()) {
514       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
515       EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
516     }
517   } else {
518     switch (M->getStorageDuration()) {
519     case SD_Automatic:
520       if (auto *Size = EmitLifetimeStart(
521               CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
522               Alloca.getPointer())) {
523         pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker,
524                                                   Alloca, Size);
525       }
526       break;
527 
528     case SD_FullExpression: {
529       if (!ShouldEmitLifetimeMarkers)
530         break;
531 
532       // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
533       // marker. Instead, start the lifetime of a conditional temporary earlier
534       // so that it's unconditional. Don't do this with sanitizers which need
535       // more precise lifetime marks. However when inside an "await.suspend"
536       // block, we should always avoid conditional cleanup because it creates
537       // boolean marker that lives across await_suspend, which can destroy coro
538       // frame.
539       ConditionalEvaluation *OldConditional = nullptr;
540       CGBuilderTy::InsertPoint OldIP;
541       if (isInConditionalBranch() && !E->getType().isDestructedType() &&
542           ((!SanOpts.has(SanitizerKind::HWAddress) &&
543             !SanOpts.has(SanitizerKind::Memory) &&
544             !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||
545            inSuspendBlock())) {
546         OldConditional = OutermostConditional;
547         OutermostConditional = nullptr;
548 
549         OldIP = Builder.saveIP();
550         llvm::BasicBlock *Block = OldConditional->getStartingBlock();
551         Builder.restoreIP(CGBuilderTy::InsertPoint(
552             Block, llvm::BasicBlock::iterator(Block->back())));
553       }
554 
555       if (auto *Size = EmitLifetimeStart(
556               CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
557               Alloca.getPointer())) {
558         pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca,
559                                              Size);
560       }
561 
562       if (OldConditional) {
563         OutermostConditional = OldConditional;
564         Builder.restoreIP(OldIP);
565       }
566       break;
567     }
568 
569     default:
570       break;
571     }
572     EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
573   }
574   pushTemporaryCleanup(*this, M, E, Object);
575 
576   // Perform derived-to-base casts and/or field accesses, to get from the
577   // temporary object we created (and, potentially, for which we extended
578   // the lifetime) to the subobject we're binding the reference to.
579   for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) {
580     switch (Adjustment.Kind) {
581     case SubobjectAdjustment::DerivedToBaseAdjustment:
582       Object =
583           GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass,
584                                 Adjustment.DerivedToBase.BasePath->path_begin(),
585                                 Adjustment.DerivedToBase.BasePath->path_end(),
586                                 /*NullCheckValue=*/ false, E->getExprLoc());
587       break;
588 
589     case SubobjectAdjustment::FieldAdjustment: {
590       LValue LV = MakeAddrLValue(Object, E->getType(), AlignmentSource::Decl);
591       LV = EmitLValueForField(LV, Adjustment.Field);
592       assert(LV.isSimple() &&
593              "materialized temporary field is not a simple lvalue");
594       Object = LV.getAddress(*this);
595       break;
596     }
597 
598     case SubobjectAdjustment::MemberPointerAdjustment: {
599       llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS);
600       Object = EmitCXXMemberDataPointerAddress(E, Object, Ptr,
601                                                Adjustment.Ptr.MPT);
602       break;
603     }
604     }
605   }
606 
607   return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
608 }
609 
610 RValue
611 CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
612   // Emit the expression as an lvalue.
613   LValue LV = EmitLValue(E);
614   assert(LV.isSimple());
615   llvm::Value *Value = LV.getPointer(*this);
616 
617   if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
618     // C++11 [dcl.ref]p5 (as amended by core issue 453):
619     //   If a glvalue to which a reference is directly bound designates neither
620     //   an existing object or function of an appropriate type nor a region of
621     //   storage of suitable size and alignment to contain an object of the
622     //   reference's type, the behavior is undefined.
623     QualType Ty = E->getType();
624     EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty);
625   }
626 
627   return RValue::get(Value);
628 }
629 
630 
631 /// getAccessedFieldNo - Given an encoded value and a result number, return the
632 /// input field number being accessed.
633 unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
634                                              const llvm::Constant *Elts) {
635   return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
636       ->getZExtValue();
637 }
638 
639 /// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
640 static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
641                                     llvm::Value *High) {
642   llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
643   llvm::Value *K47 = Builder.getInt64(47);
644   llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
645   llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
646   llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
647   llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
648   return Builder.CreateMul(B1, KMul);
649 }
650 
651 bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
652   return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
653          TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
654 }
655 
656 bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
657   CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
658   return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
659          (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
660           TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
661           TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
662 }
663 
664 bool CodeGenFunction::sanitizePerformTypeCheck() const {
665   return SanOpts.has(SanitizerKind::Null) ||
666          SanOpts.has(SanitizerKind::Alignment) ||
667          SanOpts.has(SanitizerKind::ObjectSize) ||
668          SanOpts.has(SanitizerKind::Vptr);
669 }
670 
671 void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
672                                     llvm::Value *Ptr, QualType Ty,
673                                     CharUnits Alignment,
674                                     SanitizerSet SkippedChecks,
675                                     llvm::Value *ArraySize) {
676   if (!sanitizePerformTypeCheck())
677     return;
678 
679   // Don't check pointers outside the default address space. The null check
680   // isn't correct, the object-size check isn't supported by LLVM, and we can't
681   // communicate the addresses to the runtime handler for the vptr check.
682   if (Ptr->getType()->getPointerAddressSpace())
683     return;
684 
685   // Don't check pointers to volatile data. The behavior here is implementation-
686   // defined.
687   if (Ty.isVolatileQualified())
688     return;
689 
690   SanitizerScope SanScope(this);
691 
692   SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks;
693   llvm::BasicBlock *Done = nullptr;
694 
695   // Quickly determine whether we have a pointer to an alloca. It's possible
696   // to skip null checks, and some alignment checks, for these pointers. This
697   // can reduce compile-time significantly.
698   auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
699 
700   llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());
701   llvm::Value *IsNonNull = nullptr;
702   bool IsGuaranteedNonNull =
703       SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca;
704   bool AllowNullPointers = isNullPointerAllowed(TCK);
705   if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
706       !IsGuaranteedNonNull) {
707     // The glvalue must not be an empty glvalue.
708     IsNonNull = Builder.CreateIsNotNull(Ptr);
709 
710     // The IR builder can constant-fold the null check if the pointer points to
711     // a constant.
712     IsGuaranteedNonNull = IsNonNull == True;
713 
714     // Skip the null check if the pointer is known to be non-null.
715     if (!IsGuaranteedNonNull) {
716       if (AllowNullPointers) {
717         // When performing pointer casts, it's OK if the value is null.
718         // Skip the remaining checks in that case.
719         Done = createBasicBlock("null");
720         llvm::BasicBlock *Rest = createBasicBlock("not.null");
721         Builder.CreateCondBr(IsNonNull, Rest, Done);
722         EmitBlock(Rest);
723       } else {
724         Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null));
725       }
726     }
727   }
728 
729   if (SanOpts.has(SanitizerKind::ObjectSize) &&
730       !SkippedChecks.has(SanitizerKind::ObjectSize) &&
731       !Ty->isIncompleteType()) {
732     uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
733     llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);
734     if (ArraySize)
735       Size = Builder.CreateMul(Size, ArraySize);
736 
737     // Degenerate case: new X[0] does not need an objectsize check.
738     llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
739     if (!ConstantSize || !ConstantSize->isNullValue()) {
740       // The glvalue must refer to a large enough storage region.
741       // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
742       //        to check this.
743       // FIXME: Get object address space
744       llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
745       llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
746       llvm::Value *Min = Builder.getFalse();
747       llvm::Value *NullIsUnknown = Builder.getFalse();
748       llvm::Value *Dynamic = Builder.getFalse();
749       llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy);
750       llvm::Value *LargeEnough = Builder.CreateICmpUGE(
751           Builder.CreateCall(F, {CastAddr, Min, NullIsUnknown, Dynamic}), Size);
752       Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
753     }
754   }
755 
756   llvm::MaybeAlign AlignVal;
757   llvm::Value *PtrAsInt = nullptr;
758 
759   if (SanOpts.has(SanitizerKind::Alignment) &&
760       !SkippedChecks.has(SanitizerKind::Alignment)) {
761     AlignVal = Alignment.getAsMaybeAlign();
762     if (!Ty->isIncompleteType() && !AlignVal)
763       AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
764                                              /*ForPointeeType=*/true)
765                      .getAsMaybeAlign();
766 
767     // The glvalue must be suitably aligned.
768     if (AlignVal && *AlignVal > llvm::Align(1) &&
769         (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
770       PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
771       llvm::Value *Align = Builder.CreateAnd(
772           PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
773       llvm::Value *Aligned =
774           Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
775       if (Aligned != True)
776         Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment));
777     }
778   }
779 
780   if (Checks.size() > 0) {
781     llvm::Constant *StaticData[] = {
782         EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty),
783         llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
784         llvm::ConstantInt::get(Int8Ty, TCK)};
785     EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
786               PtrAsInt ? PtrAsInt : Ptr);
787   }
788 
789   // If possible, check that the vptr indicates that there is a subobject of
790   // type Ty at offset zero within this object.
791   //
792   // C++11 [basic.life]p5,6:
793   //   [For storage which does not refer to an object within its lifetime]
794   //   The program has undefined behavior if:
795   //    -- the [pointer or glvalue] is used to access a non-static data member
796   //       or call a non-static member function
797   if (SanOpts.has(SanitizerKind::Vptr) &&
798       !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
799     // Ensure that the pointer is non-null before loading it. If there is no
800     // compile-time guarantee, reuse the run-time null check or emit a new one.
801     if (!IsGuaranteedNonNull) {
802       if (!IsNonNull)
803         IsNonNull = Builder.CreateIsNotNull(Ptr);
804       if (!Done)
805         Done = createBasicBlock("vptr.null");
806       llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null");
807       Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);
808       EmitBlock(VptrNotNull);
809     }
810 
811     // Compute a hash of the mangled name of the type.
812     //
813     // FIXME: This is not guaranteed to be deterministic! Move to a
814     //        fingerprinting mechanism once LLVM provides one. For the time
815     //        being the implementation happens to be deterministic.
816     SmallString<64> MangledName;
817     llvm::raw_svector_ostream Out(MangledName);
818     CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(),
819                                                      Out);
820 
821     // Contained in NoSanitizeList based on the mangled type.
822     if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr,
823                                                            Out.str())) {
824       llvm::hash_code TypeHash = hash_value(Out.str());
825 
826       // Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
827       llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
828       llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
829       Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), IntPtrTy,
830                        getPointerAlign());
831       llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
832       llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
833 
834       llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
835       Hash = Builder.CreateTrunc(Hash, IntPtrTy);
836 
837       // Look the hash up in our cache.
838       const int CacheSize = 128;
839       llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);
840       llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,
841                                                      "__ubsan_vptr_type_cache");
842       llvm::Value *Slot = Builder.CreateAnd(Hash,
843                                             llvm::ConstantInt::get(IntPtrTy,
844                                                                    CacheSize-1));
845       llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
846       llvm::Value *CacheVal = Builder.CreateAlignedLoad(
847           IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices),
848           getPointerAlign());
849 
850       // If the hash isn't in the cache, call a runtime handler to perform the
851       // hard work of checking whether the vptr is for an object of the right
852       // type. This will either fill in the cache and return, or produce a
853       // diagnostic.
854       llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash);
855       llvm::Constant *StaticData[] = {
856         EmitCheckSourceLocation(Loc),
857         EmitCheckTypeDescriptor(Ty),
858         CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()),
859         llvm::ConstantInt::get(Int8Ty, TCK)
860       };
861       llvm::Value *DynamicData[] = { Ptr, Hash };
862       EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
863                 SanitizerHandler::DynamicTypeCacheMiss, StaticData,
864                 DynamicData);
865     }
866   }
867 
868   if (Done) {
869     Builder.CreateBr(Done);
870     EmitBlock(Done);
871   }
872 }
873 
874 llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
875                                                    QualType EltTy) {
876   ASTContext &C = getContext();
877   uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity();
878   if (!EltSize)
879     return nullptr;
880 
881   auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts());
882   if (!ArrayDeclRef)
883     return nullptr;
884 
885   auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());
886   if (!ParamDecl)
887     return nullptr;
888 
889   auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
890   if (!POSAttr)
891     return nullptr;
892 
893   // Don't load the size if it's a lower bound.
894   int POSType = POSAttr->getType();
895   if (POSType != 0 && POSType != 1)
896     return nullptr;
897 
898   // Find the implicit size parameter.
899   auto PassedSizeIt = SizeArguments.find(ParamDecl);
900   if (PassedSizeIt == SizeArguments.end())
901     return nullptr;
902 
903   const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
904   assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
905   Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
906   llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false,
907                                               C.getSizeType(), E->getExprLoc());
908   llvm::Value *SizeOfElement =
909       llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);
910   return Builder.CreateUDiv(SizeInBytes, SizeOfElement);
911 }
912 
913 /// If Base is known to point to the start of an array, return the length of
914 /// that array. Return 0 if the length cannot be determined.
915 static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
916                                           const Expr *Base,
917                                           QualType &IndexedType,
918                                           LangOptions::StrictFlexArraysLevelKind
919                                           StrictFlexArraysLevel) {
920   // For the vector indexing extension, the bound is the number of elements.
921   if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
922     IndexedType = Base->getType();
923     return CGF.Builder.getInt32(VT->getNumElements());
924   }
925 
926   Base = Base->IgnoreParens();
927 
928   if (const auto *CE = dyn_cast<CastExpr>(Base)) {
929     if (CE->getCastKind() == CK_ArrayToPointerDecay &&
930         !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(),
931                                                      StrictFlexArraysLevel)) {
932       IndexedType = CE->getSubExpr()->getType();
933       const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
934       if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
935         return CGF.Builder.getInt(CAT->getSize());
936       else if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
937         return CGF.getVLASize(VAT).NumElts;
938       // Ignore pass_object_size here. It's not applicable on decayed pointers.
939     }
940   }
941 
942   QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
943   if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {
944     IndexedType = Base->getType();
945     return POS;
946   }
947 
948   return nullptr;
949 }
950 
951 void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
952                                       llvm::Value *Index, QualType IndexType,
953                                       bool Accessed) {
954   assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
955          "should not be called unless adding bounds checks");
956   SanitizerScope SanScope(this);
957 
958   const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
959     getLangOpts().getStrictFlexArraysLevel();
960 
961   QualType IndexedType;
962   llvm::Value *Bound =
963       getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel);
964   if (!Bound)
965     return;
966 
967   bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
968   llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
969   llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);
970 
971   llvm::Constant *StaticData[] = {
972     EmitCheckSourceLocation(E->getExprLoc()),
973     EmitCheckTypeDescriptor(IndexedType),
974     EmitCheckTypeDescriptor(IndexType)
975   };
976   llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal)
977                                 : Builder.CreateICmpULE(IndexVal, BoundVal);
978   EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds),
979             SanitizerHandler::OutOfBounds, StaticData, Index);
980 }
981 
982 
983 CodeGenFunction::ComplexPairTy CodeGenFunction::
984 EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
985                          bool isInc, bool isPre) {
986   ComplexPairTy InVal = EmitLoadOfComplex(LV, E->getExprLoc());
987 
988   llvm::Value *NextVal;
989   if (isa<llvm::IntegerType>(InVal.first->getType())) {
990     uint64_t AmountVal = isInc ? 1 : -1;
991     NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
992 
993     // Add the inc/dec to the real part.
994     NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
995   } else {
996     QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
997     llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
998     if (!isInc)
999       FVal.changeSign();
1000     NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
1001 
1002     // Add the inc/dec to the real part.
1003     NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1004   }
1005 
1006   ComplexPairTy IncVal(NextVal, InVal.second);
1007 
1008   // Store the updated result through the lvalue.
1009   EmitStoreOfComplex(IncVal, LV, /*init*/ false);
1010   if (getLangOpts().OpenMP)
1011     CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
1012                                                               E->getSubExpr());
1013 
1014   // If this is a postinc, return the value read from memory, otherwise use the
1015   // updated value.
1016   return isPre ? IncVal : InVal;
1017 }
1018 
1019 void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1020                                              CodeGenFunction *CGF) {
1021   // Bind VLAs in the cast type.
1022   if (CGF && E->getType()->isVariablyModifiedType())
1023     CGF->EmitVariablyModifiedType(E->getType());
1024 
1025   if (CGDebugInfo *DI = getModuleDebugInfo())
1026     DI->EmitExplicitCastType(E->getType());
1027 }
1028 
1029 //===----------------------------------------------------------------------===//
1030 //                         LValue Expression Emission
1031 //===----------------------------------------------------------------------===//
1032 
1033 static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
1034                                         TBAAAccessInfo *TBAAInfo,
1035                                         KnownNonNull_t IsKnownNonNull,
1036                                         CodeGenFunction &CGF) {
1037   // We allow this with ObjC object pointers because of fragile ABIs.
1038   assert(E->getType()->isPointerType() ||
1039          E->getType()->isObjCObjectPointerType());
1040   E = E->IgnoreParens();
1041 
1042   // Casts:
1043   if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
1044     if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
1045       CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
1046 
1047     switch (CE->getCastKind()) {
1048     // Non-converting casts (but not C's implicit conversion from void*).
1049     case CK_BitCast:
1050     case CK_NoOp:
1051     case CK_AddressSpaceConversion:
1052       if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1053         if (PtrTy->getPointeeType()->isVoidType())
1054           break;
1055 
1056         LValueBaseInfo InnerBaseInfo;
1057         TBAAAccessInfo InnerTBAAInfo;
1058         Address Addr = CGF.EmitPointerWithAlignment(
1059             CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull);
1060         if (BaseInfo) *BaseInfo = InnerBaseInfo;
1061         if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1062 
1063         if (isa<ExplicitCastExpr>(CE)) {
1064           LValueBaseInfo TargetTypeBaseInfo;
1065           TBAAAccessInfo TargetTypeTBAAInfo;
1066           CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(
1067               E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);
1068           if (TBAAInfo)
1069             *TBAAInfo =
1070                 CGF.CGM.mergeTBAAInfoForCast(*TBAAInfo, TargetTypeTBAAInfo);
1071           // If the source l-value is opaque, honor the alignment of the
1072           // casted-to type.
1073           if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1074             if (BaseInfo)
1075               BaseInfo->mergeForCast(TargetTypeBaseInfo);
1076             Addr = Address(Addr.getPointer(), Addr.getElementType(), Align,
1077                            IsKnownNonNull);
1078           }
1079         }
1080 
1081         if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast) &&
1082             CE->getCastKind() == CK_BitCast) {
1083           if (auto PT = E->getType()->getAs<PointerType>())
1084             CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr,
1085                                           /*MayBeNull=*/true,
1086                                           CodeGenFunction::CFITCK_UnrelatedCast,
1087                                           CE->getBeginLoc());
1088         }
1089 
1090         llvm::Type *ElemTy =
1091             CGF.ConvertTypeForMem(E->getType()->getPointeeType());
1092         Addr = Addr.withElementType(ElemTy);
1093         if (CE->getCastKind() == CK_AddressSpaceConversion)
1094           Addr = CGF.Builder.CreateAddrSpaceCast(Addr,
1095                                                  CGF.ConvertType(E->getType()));
1096         return Addr;
1097       }
1098       break;
1099 
1100     // Array-to-pointer decay.
1101     case CK_ArrayToPointerDecay:
1102       return CGF.EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo);
1103 
1104     // Derived-to-base conversions.
1105     case CK_UncheckedDerivedToBase:
1106     case CK_DerivedToBase: {
1107       // TODO: Support accesses to members of base classes in TBAA. For now, we
1108       // conservatively pretend that the complete object is of the base class
1109       // type.
1110       if (TBAAInfo)
1111         *TBAAInfo = CGF.CGM.getTBAAAccessInfo(E->getType());
1112       Address Addr = CGF.EmitPointerWithAlignment(
1113           CE->getSubExpr(), BaseInfo, nullptr,
1114           (KnownNonNull_t)(IsKnownNonNull ||
1115                            CE->getCastKind() == CK_UncheckedDerivedToBase));
1116       auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1117       return CGF.GetAddressOfBaseClass(
1118           Addr, Derived, CE->path_begin(), CE->path_end(),
1119           CGF.ShouldNullCheckClassCastValue(CE), CE->getExprLoc());
1120     }
1121 
1122     // TODO: Is there any reason to treat base-to-derived conversions
1123     // specially?
1124     default:
1125       break;
1126     }
1127   }
1128 
1129   // Unary &.
1130   if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
1131     if (UO->getOpcode() == UO_AddrOf) {
1132       LValue LV = CGF.EmitLValue(UO->getSubExpr(), IsKnownNonNull);
1133       if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1134       if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1135       return LV.getAddress(CGF);
1136     }
1137   }
1138 
1139   // std::addressof and variants.
1140   if (auto *Call = dyn_cast<CallExpr>(E)) {
1141     switch (Call->getBuiltinCallee()) {
1142     default:
1143       break;
1144     case Builtin::BIaddressof:
1145     case Builtin::BI__addressof:
1146     case Builtin::BI__builtin_addressof: {
1147       LValue LV = CGF.EmitLValue(Call->getArg(0), IsKnownNonNull);
1148       if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1149       if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1150       return LV.getAddress(CGF);
1151     }
1152     }
1153   }
1154 
1155   // TODO: conditional operators, comma.
1156 
1157   // Otherwise, use the alignment of the type.
1158   CharUnits Align =
1159       CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
1160   llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType());
1161   return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull);
1162 }
1163 
1164 /// EmitPointerWithAlignment - Given an expression of pointer type, try to
1165 /// derive a more accurate bound on the alignment of the pointer.
1166 Address CodeGenFunction::EmitPointerWithAlignment(
1167     const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,
1168     KnownNonNull_t IsKnownNonNull) {
1169   Address Addr =
1170       ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, *this);
1171   if (IsKnownNonNull && !Addr.isKnownNonNull())
1172     Addr.setKnownNonNull();
1173   return Addr;
1174 }
1175 
1176 llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1177   llvm::Value *V = RV.getScalarVal();
1178   if (auto MPT = T->getAs<MemberPointerType>())
1179     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, V, MPT);
1180   return Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
1181 }
1182 
1183 RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1184   if (Ty->isVoidType())
1185     return RValue::get(nullptr);
1186 
1187   switch (getEvaluationKind(Ty)) {
1188   case TEK_Complex: {
1189     llvm::Type *EltTy =
1190       ConvertType(Ty->castAs<ComplexType>()->getElementType());
1191     llvm::Value *U = llvm::UndefValue::get(EltTy);
1192     return RValue::getComplex(std::make_pair(U, U));
1193   }
1194 
1195   // If this is a use of an undefined aggregate type, the aggregate must have an
1196   // identifiable address.  Just because the contents of the value are undefined
1197   // doesn't mean that the address can't be taken and compared.
1198   case TEK_Aggregate: {
1199     Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
1200     return RValue::getAggregate(DestPtr);
1201   }
1202 
1203   case TEK_Scalar:
1204     return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
1205   }
1206   llvm_unreachable("bad evaluation kind");
1207 }
1208 
1209 RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1210                                               const char *Name) {
1211   ErrorUnsupported(E, Name);
1212   return GetUndefRValue(E->getType());
1213 }
1214 
1215 LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1216                                               const char *Name) {
1217   ErrorUnsupported(E, Name);
1218   llvm::Type *ElTy = ConvertType(E->getType());
1219   llvm::Type *Ty = llvm::PointerType::getUnqual(ElTy);
1220   return MakeAddrLValue(
1221       Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType());
1222 }
1223 
1224 bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1225   const Expr *Base = Obj;
1226   while (!isa<CXXThisExpr>(Base)) {
1227     // The result of a dynamic_cast can be null.
1228     if (isa<CXXDynamicCastExpr>(Base))
1229       return false;
1230 
1231     if (const auto *CE = dyn_cast<CastExpr>(Base)) {
1232       Base = CE->getSubExpr();
1233     } else if (const auto *PE = dyn_cast<ParenExpr>(Base)) {
1234       Base = PE->getSubExpr();
1235     } else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) {
1236       if (UO->getOpcode() == UO_Extension)
1237         Base = UO->getSubExpr();
1238       else
1239         return false;
1240     } else {
1241       return false;
1242     }
1243   }
1244   return true;
1245 }
1246 
1247 LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1248   LValue LV;
1249   if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E))
1250     LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true);
1251   else
1252     LV = EmitLValue(E);
1253   if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) {
1254     SanitizerSet SkippedChecks;
1255     if (const auto *ME = dyn_cast<MemberExpr>(E)) {
1256       bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase());
1257       if (IsBaseCXXThis)
1258         SkippedChecks.set(SanitizerKind::Alignment, true);
1259       if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
1260         SkippedChecks.set(SanitizerKind::Null, true);
1261     }
1262     EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(),
1263                   LV.getAlignment(), SkippedChecks);
1264   }
1265   return LV;
1266 }
1267 
1268 /// EmitLValue - Emit code to compute a designator that specifies the location
1269 /// of the expression.
1270 ///
1271 /// This can return one of two things: a simple address or a bitfield reference.
1272 /// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1273 /// an LLVM pointer type.
1274 ///
1275 /// If this returns a bitfield reference, nothing about the pointee type of the
1276 /// LLVM value is known: For example, it may not be a pointer to an integer.
1277 ///
1278 /// If this returns a normal address, and if the lvalue's C type is fixed size,
1279 /// this method guarantees that the returned pointer type will point to an LLVM
1280 /// type of the same size of the lvalue's type.  If the lvalue has a variable
1281 /// length type, this is not possible.
1282 ///
1283 LValue CodeGenFunction::EmitLValue(const Expr *E,
1284                                    KnownNonNull_t IsKnownNonNull) {
1285   LValue LV = EmitLValueHelper(E, IsKnownNonNull);
1286   if (IsKnownNonNull && !LV.isKnownNonNull())
1287     LV.setKnownNonNull();
1288   return LV;
1289 }
1290 
1291 LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
1292                                          KnownNonNull_t IsKnownNonNull) {
1293   ApplyDebugLocation DL(*this, E);
1294   switch (E->getStmtClass()) {
1295   default: return EmitUnsupportedLValue(E, "l-value expression");
1296 
1297   case Expr::ObjCPropertyRefExprClass:
1298     llvm_unreachable("cannot emit a property reference directly");
1299 
1300   case Expr::ObjCSelectorExprClass:
1301     return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
1302   case Expr::ObjCIsaExprClass:
1303     return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
1304   case Expr::BinaryOperatorClass:
1305     return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
1306   case Expr::CompoundAssignOperatorClass: {
1307     QualType Ty = E->getType();
1308     if (const AtomicType *AT = Ty->getAs<AtomicType>())
1309       Ty = AT->getValueType();
1310     if (!Ty->isAnyComplexType())
1311       return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1312     return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1313   }
1314   case Expr::CallExprClass:
1315   case Expr::CXXMemberCallExprClass:
1316   case Expr::CXXOperatorCallExprClass:
1317   case Expr::UserDefinedLiteralClass:
1318     return EmitCallExprLValue(cast<CallExpr>(E));
1319   case Expr::CXXRewrittenBinaryOperatorClass:
1320     return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm(),
1321                       IsKnownNonNull);
1322   case Expr::VAArgExprClass:
1323     return EmitVAArgExprLValue(cast<VAArgExpr>(E));
1324   case Expr::DeclRefExprClass:
1325     return EmitDeclRefLValue(cast<DeclRefExpr>(E));
1326   case Expr::ConstantExprClass: {
1327     const ConstantExpr *CE = cast<ConstantExpr>(E);
1328     if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
1329       QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit())
1330                              ->getCallReturnType(getContext())
1331                              ->getPointeeType();
1332       return MakeNaturalAlignAddrLValue(Result, RetType);
1333     }
1334     return EmitLValue(cast<ConstantExpr>(E)->getSubExpr(), IsKnownNonNull);
1335   }
1336   case Expr::ParenExprClass:
1337     return EmitLValue(cast<ParenExpr>(E)->getSubExpr(), IsKnownNonNull);
1338   case Expr::GenericSelectionExprClass:
1339     return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr(),
1340                       IsKnownNonNull);
1341   case Expr::PredefinedExprClass:
1342     return EmitPredefinedLValue(cast<PredefinedExpr>(E));
1343   case Expr::StringLiteralClass:
1344     return EmitStringLiteralLValue(cast<StringLiteral>(E));
1345   case Expr::ObjCEncodeExprClass:
1346     return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
1347   case Expr::PseudoObjectExprClass:
1348     return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
1349   case Expr::InitListExprClass:
1350     return EmitInitListLValue(cast<InitListExpr>(E));
1351   case Expr::CXXTemporaryObjectExprClass:
1352   case Expr::CXXConstructExprClass:
1353     return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
1354   case Expr::CXXBindTemporaryExprClass:
1355     return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
1356   case Expr::CXXUuidofExprClass:
1357     return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));
1358   case Expr::LambdaExprClass:
1359     return EmitAggExprToLValue(E);
1360 
1361   case Expr::ExprWithCleanupsClass: {
1362     const auto *cleanups = cast<ExprWithCleanups>(E);
1363     RunCleanupsScope Scope(*this);
1364     LValue LV = EmitLValue(cleanups->getSubExpr(), IsKnownNonNull);
1365     if (LV.isSimple()) {
1366       // Defend against branches out of gnu statement expressions surrounded by
1367       // cleanups.
1368       Address Addr = LV.getAddress(*this);
1369       llvm::Value *V = Addr.getPointer();
1370       Scope.ForceCleanup({&V});
1371       return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()),
1372                               LV.getType(), getContext(), LV.getBaseInfo(),
1373                               LV.getTBAAInfo());
1374     }
1375     // FIXME: Is it possible to create an ExprWithCleanups that produces a
1376     // bitfield lvalue or some other non-simple lvalue?
1377     return LV;
1378   }
1379 
1380   case Expr::CXXDefaultArgExprClass: {
1381     auto *DAE = cast<CXXDefaultArgExpr>(E);
1382     CXXDefaultArgExprScope Scope(*this, DAE);
1383     return EmitLValue(DAE->getExpr(), IsKnownNonNull);
1384   }
1385   case Expr::CXXDefaultInitExprClass: {
1386     auto *DIE = cast<CXXDefaultInitExpr>(E);
1387     CXXDefaultInitExprScope Scope(*this, DIE);
1388     return EmitLValue(DIE->getExpr(), IsKnownNonNull);
1389   }
1390   case Expr::CXXTypeidExprClass:
1391     return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
1392 
1393   case Expr::ObjCMessageExprClass:
1394     return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
1395   case Expr::ObjCIvarRefExprClass:
1396     return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
1397   case Expr::StmtExprClass:
1398     return EmitStmtExprLValue(cast<StmtExpr>(E));
1399   case Expr::UnaryOperatorClass:
1400     return EmitUnaryOpLValue(cast<UnaryOperator>(E));
1401   case Expr::ArraySubscriptExprClass:
1402     return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
1403   case Expr::MatrixSubscriptExprClass:
1404     return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E));
1405   case Expr::OMPArraySectionExprClass:
1406     return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E));
1407   case Expr::ExtVectorElementExprClass:
1408     return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
1409   case Expr::CXXThisExprClass:
1410     return MakeAddrLValue(LoadCXXThisAddress(), E->getType());
1411   case Expr::MemberExprClass:
1412     return EmitMemberExpr(cast<MemberExpr>(E));
1413   case Expr::CompoundLiteralExprClass:
1414     return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
1415   case Expr::ConditionalOperatorClass:
1416     return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
1417   case Expr::BinaryConditionalOperatorClass:
1418     return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
1419   case Expr::ChooseExprClass:
1420     return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(), IsKnownNonNull);
1421   case Expr::OpaqueValueExprClass:
1422     return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
1423   case Expr::SubstNonTypeTemplateParmExprClass:
1424     return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(),
1425                       IsKnownNonNull);
1426   case Expr::ImplicitCastExprClass:
1427   case Expr::CStyleCastExprClass:
1428   case Expr::CXXFunctionalCastExprClass:
1429   case Expr::CXXStaticCastExprClass:
1430   case Expr::CXXDynamicCastExprClass:
1431   case Expr::CXXReinterpretCastExprClass:
1432   case Expr::CXXConstCastExprClass:
1433   case Expr::CXXAddrspaceCastExprClass:
1434   case Expr::ObjCBridgedCastExprClass:
1435     return EmitCastLValue(cast<CastExpr>(E));
1436 
1437   case Expr::MaterializeTemporaryExprClass:
1438     return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
1439 
1440   case Expr::CoawaitExprClass:
1441     return EmitCoawaitLValue(cast<CoawaitExpr>(E));
1442   case Expr::CoyieldExprClass:
1443     return EmitCoyieldLValue(cast<CoyieldExpr>(E));
1444   }
1445 }
1446 
1447 /// Given an object of the given canonical type, can we safely copy a
1448 /// value out of it based on its initializer?
1449 static bool isConstantEmittableObjectType(QualType type) {
1450   assert(type.isCanonical());
1451   assert(!type->isReferenceType());
1452 
1453   // Must be const-qualified but non-volatile.
1454   Qualifiers qs = type.getLocalQualifiers();
1455   if (!qs.hasConst() || qs.hasVolatile()) return false;
1456 
1457   // Otherwise, all object types satisfy this except C++ classes with
1458   // mutable subobjects or non-trivial copy/destroy behavior.
1459   if (const auto *RT = dyn_cast<RecordType>(type))
1460     if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1461       if (RD->hasMutableFields() || !RD->isTrivial())
1462         return false;
1463 
1464   return true;
1465 }
1466 
1467 /// Can we constant-emit a load of a reference to a variable of the
1468 /// given type?  This is different from predicates like
1469 /// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1470 /// in situations that don't necessarily satisfy the language's rules
1471 /// for this (e.g. C++'s ODR-use rules).  For example, we want to able
1472 /// to do this with const float variables even if those variables
1473 /// aren't marked 'constexpr'.
1474 enum ConstantEmissionKind {
1475   CEK_None,
1476   CEK_AsReferenceOnly,
1477   CEK_AsValueOrReference,
1478   CEK_AsValueOnly
1479 };
1480 static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1481   type = type.getCanonicalType();
1482   if (const auto *ref = dyn_cast<ReferenceType>(type)) {
1483     if (isConstantEmittableObjectType(ref->getPointeeType()))
1484       return CEK_AsValueOrReference;
1485     return CEK_AsReferenceOnly;
1486   }
1487   if (isConstantEmittableObjectType(type))
1488     return CEK_AsValueOnly;
1489   return CEK_None;
1490 }
1491 
1492 /// Try to emit a reference to the given value without producing it as
1493 /// an l-value.  This is just an optimization, but it avoids us needing
1494 /// to emit global copies of variables if they're named without triggering
1495 /// a formal use in a context where we can't emit a direct reference to them,
1496 /// for instance if a block or lambda or a member of a local class uses a
1497 /// const int variable or constexpr variable from an enclosing function.
1498 CodeGenFunction::ConstantEmission
1499 CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
1500   ValueDecl *value = refExpr->getDecl();
1501 
1502   // The value needs to be an enum constant or a constant variable.
1503   ConstantEmissionKind CEK;
1504   if (isa<ParmVarDecl>(value)) {
1505     CEK = CEK_None;
1506   } else if (auto *var = dyn_cast<VarDecl>(value)) {
1507     CEK = checkVarTypeForConstantEmission(var->getType());
1508   } else if (isa<EnumConstantDecl>(value)) {
1509     CEK = CEK_AsValueOnly;
1510   } else {
1511     CEK = CEK_None;
1512   }
1513   if (CEK == CEK_None) return ConstantEmission();
1514 
1515   Expr::EvalResult result;
1516   bool resultIsReference;
1517   QualType resultType;
1518 
1519   // It's best to evaluate all the way as an r-value if that's permitted.
1520   if (CEK != CEK_AsReferenceOnly &&
1521       refExpr->EvaluateAsRValue(result, getContext())) {
1522     resultIsReference = false;
1523     resultType = refExpr->getType();
1524 
1525   // Otherwise, try to evaluate as an l-value.
1526   } else if (CEK != CEK_AsValueOnly &&
1527              refExpr->EvaluateAsLValue(result, getContext())) {
1528     resultIsReference = true;
1529     resultType = value->getType();
1530 
1531   // Failure.
1532   } else {
1533     return ConstantEmission();
1534   }
1535 
1536   // In any case, if the initializer has side-effects, abandon ship.
1537   if (result.HasSideEffects)
1538     return ConstantEmission();
1539 
1540   // In CUDA/HIP device compilation, a lambda may capture a reference variable
1541   // referencing a global host variable by copy. In this case the lambda should
1542   // make a copy of the value of the global host variable. The DRE of the
1543   // captured reference variable cannot be emitted as load from the host
1544   // global variable as compile time constant, since the host variable is not
1545   // accessible on device. The DRE of the captured reference variable has to be
1546   // loaded from captures.
1547   if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
1548       refExpr->refersToEnclosingVariableOrCapture()) {
1549     auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
1550     if (MD && MD->getParent()->isLambda() &&
1551         MD->getOverloadedOperator() == OO_Call) {
1552       const APValue::LValueBase &base = result.Val.getLValueBase();
1553       if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
1554         if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
1555           if (!VD->hasAttr<CUDADeviceAttr>()) {
1556             return ConstantEmission();
1557           }
1558         }
1559       }
1560     }
1561   }
1562 
1563   // Emit as a constant.
1564   auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(),
1565                                                result.Val, resultType);
1566 
1567   // Make sure we emit a debug reference to the global variable.
1568   // This should probably fire even for
1569   if (isa<VarDecl>(value)) {
1570     if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
1571       EmitDeclRefExprDbgValue(refExpr, result.Val);
1572   } else {
1573     assert(isa<EnumConstantDecl>(value));
1574     EmitDeclRefExprDbgValue(refExpr, result.Val);
1575   }
1576 
1577   // If we emitted a reference constant, we need to dereference that.
1578   if (resultIsReference)
1579     return ConstantEmission::forReference(C);
1580 
1581   return ConstantEmission::forValue(C);
1582 }
1583 
1584 static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
1585                                                         const MemberExpr *ME) {
1586   if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) {
1587     // Try to emit static variable member expressions as DREs.
1588     return DeclRefExpr::Create(
1589         CGF.getContext(), NestedNameSpecifierLoc(), SourceLocation(), VD,
1590         /*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(),
1591         ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse());
1592   }
1593   return nullptr;
1594 }
1595 
1596 CodeGenFunction::ConstantEmission
1597 CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
1598   if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, ME))
1599     return tryEmitAsConstant(DRE);
1600   return ConstantEmission();
1601 }
1602 
1603 llvm::Value *CodeGenFunction::emitScalarConstant(
1604     const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
1605   assert(Constant && "not a constant");
1606   if (Constant.isReference())
1607     return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E),
1608                             E->getExprLoc())
1609         .getScalarVal();
1610   return Constant.getValue();
1611 }
1612 
1613 llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
1614                                                SourceLocation Loc) {
1615   return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(),
1616                           lvalue.getType(), Loc, lvalue.getBaseInfo(),
1617                           lvalue.getTBAAInfo(), lvalue.isNontemporal());
1618 }
1619 
1620 static bool hasBooleanRepresentation(QualType Ty) {
1621   if (Ty->isBooleanType())
1622     return true;
1623 
1624   if (const EnumType *ET = Ty->getAs<EnumType>())
1625     return ET->getDecl()->getIntegerType()->isBooleanType();
1626 
1627   if (const AtomicType *AT = Ty->getAs<AtomicType>())
1628     return hasBooleanRepresentation(AT->getValueType());
1629 
1630   return false;
1631 }
1632 
1633 static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
1634                             llvm::APInt &Min, llvm::APInt &End,
1635                             bool StrictEnums, bool IsBool) {
1636   const EnumType *ET = Ty->getAs<EnumType>();
1637   bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
1638                                 ET && !ET->getDecl()->isFixed();
1639   if (!IsBool && !IsRegularCPlusPlusEnum)
1640     return false;
1641 
1642   if (IsBool) {
1643     Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);
1644     End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);
1645   } else {
1646     const EnumDecl *ED = ET->getDecl();
1647     ED->getValueRange(End, Min);
1648   }
1649   return true;
1650 }
1651 
1652 llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
1653   llvm::APInt Min, End;
1654   if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums,
1655                        hasBooleanRepresentation(Ty)))
1656     return nullptr;
1657 
1658   llvm::MDBuilder MDHelper(getLLVMContext());
1659   return MDHelper.createRange(Min, End);
1660 }
1661 
1662 bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
1663                                            SourceLocation Loc) {
1664   bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool);
1665   bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum);
1666   if (!HasBoolCheck && !HasEnumCheck)
1667     return false;
1668 
1669   bool IsBool = hasBooleanRepresentation(Ty) ||
1670                 NSAPI(CGM.getContext()).isObjCBOOLType(Ty);
1671   bool NeedsBoolCheck = HasBoolCheck && IsBool;
1672   bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>();
1673   if (!NeedsBoolCheck && !NeedsEnumCheck)
1674     return false;
1675 
1676   // Single-bit booleans don't need to be checked. Special-case this to avoid
1677   // a bit width mismatch when handling bitfield values. This is handled by
1678   // EmitFromMemory for the non-bitfield case.
1679   if (IsBool &&
1680       cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1)
1681     return false;
1682 
1683   llvm::APInt Min, End;
1684   if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
1685     return true;
1686 
1687   auto &Ctx = getLLVMContext();
1688   SanitizerScope SanScope(this);
1689   llvm::Value *Check;
1690   --End;
1691   if (!Min) {
1692     Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End));
1693   } else {
1694     llvm::Value *Upper =
1695         Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End));
1696     llvm::Value *Lower =
1697         Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min));
1698     Check = Builder.CreateAnd(Upper, Lower);
1699   }
1700   llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
1701                                   EmitCheckTypeDescriptor(Ty)};
1702   SanitizerMask Kind =
1703       NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1704   EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
1705             StaticArgs, EmitCheckValue(Value));
1706   return true;
1707 }
1708 
1709 llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
1710                                                QualType Ty,
1711                                                SourceLocation Loc,
1712                                                LValueBaseInfo BaseInfo,
1713                                                TBAAAccessInfo TBAAInfo,
1714                                                bool isNontemporal) {
1715   if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1716     if (GV->isThreadLocal())
1717       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
1718                               NotKnownNonNull);
1719 
1720   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1721     // Boolean vectors use `iN` as storage type.
1722     if (ClangVecTy->isExtVectorBoolType()) {
1723       llvm::Type *ValTy = ConvertType(Ty);
1724       unsigned ValNumElems =
1725           cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1726       // Load the `iP` storage object (P is the padded vector size).
1727       auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits");
1728       const auto *RawIntTy = RawIntV->getType();
1729       assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
1730       // Bitcast iP --> <P x i1>.
1731       auto *PaddedVecTy = llvm::FixedVectorType::get(
1732           Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1733       llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy);
1734       // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1735       V = emitBoolVecConversion(V, ValNumElems, "extractvec");
1736 
1737       return EmitFromMemory(V, Ty);
1738     }
1739 
1740     // Handle vectors of size 3 like size 4 for better performance.
1741     const llvm::Type *EltTy = Addr.getElementType();
1742     const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
1743 
1744     if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) {
1745 
1746       llvm::VectorType *vec4Ty =
1747           llvm::FixedVectorType::get(VTy->getElementType(), 4);
1748       Address Cast = Addr.withElementType(vec4Ty);
1749       // Now load value.
1750       llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
1751 
1752       // Shuffle vector to get vec3.
1753       V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec");
1754       return EmitFromMemory(V, Ty);
1755     }
1756   }
1757 
1758   // Atomic operations have to be done on integral types.
1759   LValue AtomicLValue =
1760       LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1761   if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) {
1762     return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();
1763   }
1764 
1765   llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
1766   if (isNontemporal) {
1767     llvm::MDNode *Node = llvm::MDNode::get(
1768         Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1769     Load->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);
1770   }
1771 
1772   CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
1773 
1774   if (EmitScalarRangeCheck(Load, Ty, Loc)) {
1775     // In order to prevent the optimizer from throwing away the check, don't
1776     // attach range metadata to the load.
1777   } else if (CGM.getCodeGenOpts().OptimizationLevel > 0)
1778     if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
1779       Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
1780       Load->setMetadata(llvm::LLVMContext::MD_noundef,
1781                         llvm::MDNode::get(getLLVMContext(), std::nullopt));
1782     }
1783 
1784   return EmitFromMemory(Load, Ty);
1785 }
1786 
1787 llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
1788   // Bool has a different representation in memory than in registers.
1789   if (hasBooleanRepresentation(Ty)) {
1790     // This should really always be an i1, but sometimes it's already
1791     // an i8, and it's awkward to track those cases down.
1792     if (Value->getType()->isIntegerTy(1))
1793       return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool");
1794     assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1795            "wrong value rep of bool");
1796   }
1797 
1798   return Value;
1799 }
1800 
1801 llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
1802   // Bool has a different representation in memory than in registers.
1803   if (hasBooleanRepresentation(Ty)) {
1804     assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1805            "wrong value rep of bool");
1806     return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
1807   }
1808   if (Ty->isExtVectorBoolType()) {
1809     const auto *RawIntTy = Value->getType();
1810     // Bitcast iP --> <P x i1>.
1811     auto *PaddedVecTy = llvm::FixedVectorType::get(
1812         Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1813     auto *V = Builder.CreateBitCast(Value, PaddedVecTy);
1814     // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1815     llvm::Type *ValTy = ConvertType(Ty);
1816     unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1817     return emitBoolVecConversion(V, ValNumElems, "extractvec");
1818   }
1819 
1820   return Value;
1821 }
1822 
1823 // Convert the pointer of \p Addr to a pointer to a vector (the value type of
1824 // MatrixType), if it points to a array (the memory type of MatrixType).
1825 static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF,
1826                                          bool IsVector = true) {
1827   auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
1828   if (ArrayTy && IsVector) {
1829     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
1830                                                 ArrayTy->getNumElements());
1831 
1832     return Addr.withElementType(VectorTy);
1833   }
1834   auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType());
1835   if (VectorTy && !IsVector) {
1836     auto *ArrayTy = llvm::ArrayType::get(
1837         VectorTy->getElementType(),
1838         cast<llvm::FixedVectorType>(VectorTy)->getNumElements());
1839 
1840     return Addr.withElementType(ArrayTy);
1841   }
1842 
1843   return Addr;
1844 }
1845 
1846 // Emit a store of a matrix LValue. This may require casting the original
1847 // pointer to memory address (ArrayType) to a pointer to the value type
1848 // (VectorType).
1849 static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
1850                                     bool isInit, CodeGenFunction &CGF) {
1851   Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF,
1852                                            value->getType()->isVectorTy());
1853   CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(),
1854                         lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit,
1855                         lvalue.isNontemporal());
1856 }
1857 
1858 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
1859                                         bool Volatile, QualType Ty,
1860                                         LValueBaseInfo BaseInfo,
1861                                         TBAAAccessInfo TBAAInfo,
1862                                         bool isInit, bool isNontemporal) {
1863   if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1864     if (GV->isThreadLocal())
1865       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
1866                               NotKnownNonNull);
1867 
1868   llvm::Type *SrcTy = Value->getType();
1869   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1870     auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
1871     if (VecTy && ClangVecTy->isExtVectorBoolType()) {
1872       auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType());
1873       // Expand to the memory bit width.
1874       unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
1875       // <N x i1> --> <P x i1>.
1876       Value = emitBoolVecConversion(Value, MemNumElems, "insertvec");
1877       // <P x i1> --> iP.
1878       Value = Builder.CreateBitCast(Value, MemIntTy);
1879     } else if (!CGM.getCodeGenOpts().PreserveVec3Type) {
1880       // Handle vec3 special.
1881       if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
1882         // Our source is a vec3, do a shuffle vector to make it a vec4.
1883         Value = Builder.CreateShuffleVector(Value, ArrayRef<int>{0, 1, 2, -1},
1884                                             "extractVec");
1885         SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4);
1886       }
1887       if (Addr.getElementType() != SrcTy) {
1888         Addr = Addr.withElementType(SrcTy);
1889       }
1890     }
1891   }
1892 
1893   Value = EmitToMemory(Value, Ty);
1894 
1895   LValue AtomicLValue =
1896       LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1897   if (Ty->isAtomicType() ||
1898       (!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) {
1899     EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit);
1900     return;
1901   }
1902 
1903   llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
1904   if (isNontemporal) {
1905     llvm::MDNode *Node =
1906         llvm::MDNode::get(Store->getContext(),
1907                           llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1908     Store->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);
1909   }
1910 
1911   CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
1912 }
1913 
1914 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
1915                                         bool isInit) {
1916   if (lvalue.getType()->isConstantMatrixType()) {
1917     EmitStoreOfMatrixScalar(value, lvalue, isInit, *this);
1918     return;
1919   }
1920 
1921   EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(),
1922                     lvalue.getType(), lvalue.getBaseInfo(),
1923                     lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal());
1924 }
1925 
1926 // Emit a load of a LValue of matrix type. This may require casting the pointer
1927 // to memory address (ArrayType) to a pointer to the value type (VectorType).
1928 static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
1929                                      CodeGenFunction &CGF) {
1930   assert(LV.getType()->isConstantMatrixType());
1931   Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF);
1932   LV.setAddress(Addr);
1933   return RValue::get(CGF.EmitLoadOfScalar(LV, Loc));
1934 }
1935 
1936 /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
1937 /// method emits the address of the lvalue, then loads the result as an rvalue,
1938 /// returning the rvalue.
1939 RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
1940   if (LV.isObjCWeak()) {
1941     // load of a __weak object.
1942     Address AddrWeakObj = LV.getAddress(*this);
1943     return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
1944                                                              AddrWeakObj));
1945   }
1946   if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
1947     // In MRC mode, we do a load+autorelease.
1948     if (!getLangOpts().ObjCAutoRefCount) {
1949       return RValue::get(EmitARCLoadWeak(LV.getAddress(*this)));
1950     }
1951 
1952     // In ARC mode, we load retained and then consume the value.
1953     llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this));
1954     Object = EmitObjCConsumeObject(LV.getType(), Object);
1955     return RValue::get(Object);
1956   }
1957 
1958   if (LV.isSimple()) {
1959     assert(!LV.getType()->isFunctionType());
1960 
1961     if (LV.getType()->isConstantMatrixType())
1962       return EmitLoadOfMatrixLValue(LV, Loc, *this);
1963 
1964     // Everything needs a load.
1965     return RValue::get(EmitLoadOfScalar(LV, Loc));
1966   }
1967 
1968   if (LV.isVectorElt()) {
1969     llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(),
1970                                               LV.isVolatileQualified());
1971     return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
1972                                                     "vecext"));
1973   }
1974 
1975   // If this is a reference to a subset of the elements of a vector, either
1976   // shuffle the input or extract/insert them as appropriate.
1977   if (LV.isExtVectorElt()) {
1978     return EmitLoadOfExtVectorElementLValue(LV);
1979   }
1980 
1981   // Global Register variables always invoke intrinsics
1982   if (LV.isGlobalReg())
1983     return EmitLoadOfGlobalRegLValue(LV);
1984 
1985   if (LV.isMatrixElt()) {
1986     llvm::Value *Idx = LV.getMatrixIdx();
1987     if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
1988       const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();
1989       llvm::MatrixBuilder MB(Builder);
1990       MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
1991     }
1992     llvm::LoadInst *Load =
1993         Builder.CreateLoad(LV.getMatrixAddress(), LV.isVolatileQualified());
1994     return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext"));
1995   }
1996 
1997   assert(LV.isBitField() && "Unknown LValue type!");
1998   return EmitLoadOfBitfieldLValue(LV, Loc);
1999 }
2000 
2001 RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2002                                                  SourceLocation Loc) {
2003   const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2004 
2005   // Get the output type.
2006   llvm::Type *ResLTy = ConvertType(LV.getType());
2007 
2008   Address Ptr = LV.getBitFieldAddress();
2009   llvm::Value *Val =
2010       Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load");
2011 
2012   bool UseVolatile = LV.isVolatileQualified() &&
2013                      Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2014   const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2015   const unsigned StorageSize =
2016       UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2017   if (Info.IsSigned) {
2018     assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2019     unsigned HighBits = StorageSize - Offset - Info.Size;
2020     if (HighBits)
2021       Val = Builder.CreateShl(Val, HighBits, "bf.shl");
2022     if (Offset + HighBits)
2023       Val = Builder.CreateAShr(Val, Offset + HighBits, "bf.ashr");
2024   } else {
2025     if (Offset)
2026       Val = Builder.CreateLShr(Val, Offset, "bf.lshr");
2027     if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2028       Val = Builder.CreateAnd(
2029           Val, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), "bf.clear");
2030   }
2031   Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast");
2032   EmitScalarRangeCheck(Val, LV.getType(), Loc);
2033   return RValue::get(Val);
2034 }
2035 
2036 // If this is a reference to a subset of the elements of a vector, create an
2037 // appropriate shufflevector.
2038 RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2039   llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(),
2040                                         LV.isVolatileQualified());
2041 
2042   const llvm::Constant *Elts = LV.getExtVectorElts();
2043 
2044   // If the result of the expression is a non-vector type, we must be extracting
2045   // a single element.  Just codegen as an extractelement.
2046   const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2047   if (!ExprVT) {
2048     unsigned InIdx = getAccessedFieldNo(0, Elts);
2049     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2050     return RValue::get(Builder.CreateExtractElement(Vec, Elt));
2051   }
2052 
2053   // Always use shuffle vector to try to retain the original program structure
2054   unsigned NumResultElts = ExprVT->getNumElements();
2055 
2056   SmallVector<int, 4> Mask;
2057   for (unsigned i = 0; i != NumResultElts; ++i)
2058     Mask.push_back(getAccessedFieldNo(i, Elts));
2059 
2060   Vec = Builder.CreateShuffleVector(Vec, Mask);
2061   return RValue::get(Vec);
2062 }
2063 
2064 /// Generates lvalue for partial ext_vector access.
2065 Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2066   Address VectorAddress = LV.getExtVectorAddress();
2067   QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2068   llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT);
2069 
2070   Address CastToPointerElement = VectorAddress.withElementType(VectorElementTy);
2071 
2072   const llvm::Constant *Elts = LV.getExtVectorElts();
2073   unsigned ix = getAccessedFieldNo(0, Elts);
2074 
2075   Address VectorBasePtrPlusIx =
2076     Builder.CreateConstInBoundsGEP(CastToPointerElement, ix,
2077                                    "vector.elt");
2078 
2079   return VectorBasePtrPlusIx;
2080 }
2081 
2082 /// Load of global gamed gegisters are always calls to intrinsics.
2083 RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2084   assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2085          "Bad type for register variable");
2086   llvm::MDNode *RegName = cast<llvm::MDNode>(
2087       cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata());
2088 
2089   // We accept integer and pointer types only
2090   llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType());
2091   llvm::Type *Ty = OrigTy;
2092   if (OrigTy->isPointerTy())
2093     Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2094   llvm::Type *Types[] = { Ty };
2095 
2096   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
2097   llvm::Value *Call = Builder.CreateCall(
2098       F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));
2099   if (OrigTy->isPointerTy())
2100     Call = Builder.CreateIntToPtr(Call, OrigTy);
2101   return RValue::get(Call);
2102 }
2103 
2104 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
2105 /// lvalue, where both are guaranteed to the have the same type, and that type
2106 /// is 'Ty'.
2107 void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2108                                              bool isInit) {
2109   if (!Dst.isSimple()) {
2110     if (Dst.isVectorElt()) {
2111       // Read/modify/write the vector, inserting the new element.
2112       llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
2113                                             Dst.isVolatileQualified());
2114       auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
2115       if (IRStoreTy) {
2116         auto *IRVecTy = llvm::FixedVectorType::get(
2117             Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
2118         Vec = Builder.CreateBitCast(Vec, IRVecTy);
2119         // iN --> <N x i1>.
2120       }
2121       Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
2122                                         Dst.getVectorIdx(), "vecins");
2123       if (IRStoreTy) {
2124         // <N x i1> --> <iN>.
2125         Vec = Builder.CreateBitCast(Vec, IRStoreTy);
2126       }
2127       Builder.CreateStore(Vec, Dst.getVectorAddress(),
2128                           Dst.isVolatileQualified());
2129       return;
2130     }
2131 
2132     // If this is an update of extended vector elements, insert them as
2133     // appropriate.
2134     if (Dst.isExtVectorElt())
2135       return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2136 
2137     if (Dst.isGlobalReg())
2138       return EmitStoreThroughGlobalRegLValue(Src, Dst);
2139 
2140     if (Dst.isMatrixElt()) {
2141       llvm::Value *Idx = Dst.getMatrixIdx();
2142       if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2143         const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2144         llvm::MatrixBuilder MB(Builder);
2145         MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2146       }
2147       llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress());
2148       llvm::Value *Vec =
2149           Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins");
2150       Builder.CreateStore(Vec, Dst.getMatrixAddress(),
2151                           Dst.isVolatileQualified());
2152       return;
2153     }
2154 
2155     assert(Dst.isBitField() && "Unknown LValue type");
2156     return EmitStoreThroughBitfieldLValue(Src, Dst);
2157   }
2158 
2159   // There's special magic for assigning into an ARC-qualified l-value.
2160   if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2161     switch (Lifetime) {
2162     case Qualifiers::OCL_None:
2163       llvm_unreachable("present but none");
2164 
2165     case Qualifiers::OCL_ExplicitNone:
2166       // nothing special
2167       break;
2168 
2169     case Qualifiers::OCL_Strong:
2170       if (isInit) {
2171         Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal()));
2172         break;
2173       }
2174       EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
2175       return;
2176 
2177     case Qualifiers::OCL_Weak:
2178       if (isInit)
2179         // Initialize and then skip the primitive store.
2180         EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal());
2181       else
2182         EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(),
2183                          /*ignore*/ true);
2184       return;
2185 
2186     case Qualifiers::OCL_Autoreleasing:
2187       Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),
2188                                                      Src.getScalarVal()));
2189       // fall into the normal path
2190       break;
2191     }
2192   }
2193 
2194   if (Dst.isObjCWeak() && !Dst.isNonGC()) {
2195     // load of a __weak object.
2196     Address LvalueDst = Dst.getAddress(*this);
2197     llvm::Value *src = Src.getScalarVal();
2198      CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
2199     return;
2200   }
2201 
2202   if (Dst.isObjCStrong() && !Dst.isNonGC()) {
2203     // load of a __strong object.
2204     Address LvalueDst = Dst.getAddress(*this);
2205     llvm::Value *src = Src.getScalarVal();
2206     if (Dst.isObjCIvar()) {
2207       assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
2208       llvm::Type *ResultType = IntPtrTy;
2209       Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp());
2210       llvm::Value *RHS = dst.getPointer();
2211       RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
2212       llvm::Value *LHS =
2213         Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
2214                                "sub.ptr.lhs.cast");
2215       llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
2216       CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
2217                                               BytesBetween);
2218     } else if (Dst.isGlobalObjCRef()) {
2219       CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
2220                                                 Dst.isThreadLocalRef());
2221     }
2222     else
2223       CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
2224     return;
2225   }
2226 
2227   assert(Src.isScalar() && "Can't emit an agg store with this method");
2228   EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
2229 }
2230 
2231 void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
2232                                                      llvm::Value **Result) {
2233   const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
2234   llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
2235   Address Ptr = Dst.getBitFieldAddress();
2236 
2237   // Get the source value, truncated to the width of the bit-field.
2238   llvm::Value *SrcVal = Src.getScalarVal();
2239 
2240   // Cast the source to the storage type and shift it into place.
2241   SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(),
2242                                  /*isSigned=*/false);
2243   llvm::Value *MaskedVal = SrcVal;
2244 
2245   const bool UseVolatile =
2246       CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
2247       Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2248   const unsigned StorageSize =
2249       UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2250   const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2251   // See if there are other bits in the bitfield's storage we'll need to load
2252   // and mask together with source before storing.
2253   if (StorageSize != Info.Size) {
2254     assert(StorageSize > Info.Size && "Invalid bitfield size.");
2255     llvm::Value *Val =
2256         Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load");
2257 
2258     // Mask the source value as needed.
2259     if (!hasBooleanRepresentation(Dst.getType()))
2260       SrcVal = Builder.CreateAnd(
2261           SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.Size),
2262           "bf.value");
2263     MaskedVal = SrcVal;
2264     if (Offset)
2265       SrcVal = Builder.CreateShl(SrcVal, Offset, "bf.shl");
2266 
2267     // Mask out the original value.
2268     Val = Builder.CreateAnd(
2269         Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.Size),
2270         "bf.clear");
2271 
2272     // Or together the unchanged values and the source value.
2273     SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set");
2274   } else {
2275     assert(Offset == 0);
2276     // According to the AACPS:
2277     // When a volatile bit-field is written, and its container does not overlap
2278     // with any non-bit-field member, its container must be read exactly once
2279     // and written exactly once using the access width appropriate to the type
2280     // of the container. The two accesses are not atomic.
2281     if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) &&
2282         CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
2283       Builder.CreateLoad(Ptr, true, "bf.load");
2284   }
2285 
2286   // Write the new value back out.
2287   Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified());
2288 
2289   // Return the new value of the bit-field, if requested.
2290   if (Result) {
2291     llvm::Value *ResultVal = MaskedVal;
2292 
2293     // Sign extend the value if needed.
2294     if (Info.IsSigned) {
2295       assert(Info.Size <= StorageSize);
2296       unsigned HighBits = StorageSize - Info.Size;
2297       if (HighBits) {
2298         ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl");
2299         ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr");
2300       }
2301     }
2302 
2303     ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned,
2304                                       "bf.result.cast");
2305     *Result = EmitFromMemory(ResultVal, Dst.getType());
2306   }
2307 }
2308 
2309 void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
2310                                                                LValue Dst) {
2311   // This access turns into a read/modify/write of the vector.  Load the input
2312   // value now.
2313   llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(),
2314                                         Dst.isVolatileQualified());
2315   const llvm::Constant *Elts = Dst.getExtVectorElts();
2316 
2317   llvm::Value *SrcVal = Src.getScalarVal();
2318 
2319   if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
2320     unsigned NumSrcElts = VTy->getNumElements();
2321     unsigned NumDstElts =
2322         cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
2323     if (NumDstElts == NumSrcElts) {
2324       // Use shuffle vector is the src and destination are the same number of
2325       // elements and restore the vector mask since it is on the side it will be
2326       // stored.
2327       SmallVector<int, 4> Mask(NumDstElts);
2328       for (unsigned i = 0; i != NumSrcElts; ++i)
2329         Mask[getAccessedFieldNo(i, Elts)] = i;
2330 
2331       Vec = Builder.CreateShuffleVector(SrcVal, Mask);
2332     } else if (NumDstElts > NumSrcElts) {
2333       // Extended the source vector to the same length and then shuffle it
2334       // into the destination.
2335       // FIXME: since we're shuffling with undef, can we just use the indices
2336       //        into that?  This could be simpler.
2337       SmallVector<int, 4> ExtMask;
2338       for (unsigned i = 0; i != NumSrcElts; ++i)
2339         ExtMask.push_back(i);
2340       ExtMask.resize(NumDstElts, -1);
2341       llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask);
2342       // build identity
2343       SmallVector<int, 4> Mask;
2344       for (unsigned i = 0; i != NumDstElts; ++i)
2345         Mask.push_back(i);
2346 
2347       // When the vector size is odd and .odd or .hi is used, the last element
2348       // of the Elts constant array will be one past the size of the vector.
2349       // Ignore the last element here, if it is greater than the mask size.
2350       if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size())
2351         NumSrcElts--;
2352 
2353       // modify when what gets shuffled in
2354       for (unsigned i = 0; i != NumSrcElts; ++i)
2355         Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts;
2356       Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);
2357     } else {
2358       // We should never shorten the vector
2359       llvm_unreachable("unexpected shorten vector length");
2360     }
2361   } else {
2362     // If the Src is a scalar (not a vector) it must be updating one element.
2363     unsigned InIdx = getAccessedFieldNo(0, Elts);
2364     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2365     Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
2366   }
2367 
2368   Builder.CreateStore(Vec, Dst.getExtVectorAddress(),
2369                       Dst.isVolatileQualified());
2370 }
2371 
2372 /// Store of global named registers are always calls to intrinsics.
2373 void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
2374   assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
2375          "Bad type for register variable");
2376   llvm::MDNode *RegName = cast<llvm::MDNode>(
2377       cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata());
2378   assert(RegName && "Register LValue is not metadata");
2379 
2380   // We accept integer and pointer types only
2381   llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType());
2382   llvm::Type *Ty = OrigTy;
2383   if (OrigTy->isPointerTy())
2384     Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2385   llvm::Type *Types[] = { Ty };
2386 
2387   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
2388   llvm::Value *Value = Src.getScalarVal();
2389   if (OrigTy->isPointerTy())
2390     Value = Builder.CreatePtrToInt(Value, Ty);
2391   Builder.CreateCall(
2392       F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value});
2393 }
2394 
2395 // setObjCGCLValueClass - sets class of the lvalue for the purpose of
2396 // generating write-barries API. It is currently a global, ivar,
2397 // or neither.
2398 static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
2399                                  LValue &LV,
2400                                  bool IsMemberAccess=false) {
2401   if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
2402     return;
2403 
2404   if (isa<ObjCIvarRefExpr>(E)) {
2405     QualType ExpTy = E->getType();
2406     if (IsMemberAccess && ExpTy->isPointerType()) {
2407       // If ivar is a structure pointer, assigning to field of
2408       // this struct follows gcc's behavior and makes it a non-ivar
2409       // writer-barrier conservatively.
2410       ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2411       if (ExpTy->isRecordType()) {
2412         LV.setObjCIvar(false);
2413         return;
2414       }
2415     }
2416     LV.setObjCIvar(true);
2417     auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E));
2418     LV.setBaseIvarExp(Exp->getBase());
2419     LV.setObjCArray(E->getType()->isArrayType());
2420     return;
2421   }
2422 
2423   if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) {
2424     if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
2425       if (VD->hasGlobalStorage()) {
2426         LV.setGlobalObjCRef(true);
2427         LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
2428       }
2429     }
2430     LV.setObjCArray(E->getType()->isArrayType());
2431     return;
2432   }
2433 
2434   if (const auto *Exp = dyn_cast<UnaryOperator>(E)) {
2435     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2436     return;
2437   }
2438 
2439   if (const auto *Exp = dyn_cast<ParenExpr>(E)) {
2440     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2441     if (LV.isObjCIvar()) {
2442       // If cast is to a structure pointer, follow gcc's behavior and make it
2443       // a non-ivar write-barrier.
2444       QualType ExpTy = E->getType();
2445       if (ExpTy->isPointerType())
2446         ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2447       if (ExpTy->isRecordType())
2448         LV.setObjCIvar(false);
2449     }
2450     return;
2451   }
2452 
2453   if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) {
2454     setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
2455     return;
2456   }
2457 
2458   if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) {
2459     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2460     return;
2461   }
2462 
2463   if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) {
2464     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2465     return;
2466   }
2467 
2468   if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
2469     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2470     return;
2471   }
2472 
2473   if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
2474     setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
2475     if (LV.isObjCIvar() && !LV.isObjCArray())
2476       // Using array syntax to assigning to what an ivar points to is not
2477       // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
2478       LV.setObjCIvar(false);
2479     else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
2480       // Using array syntax to assigning to what global points to is not
2481       // same as assigning to the global itself. {id *G;} G[i] = 0;
2482       LV.setGlobalObjCRef(false);
2483     return;
2484   }
2485 
2486   if (const auto *Exp = dyn_cast<MemberExpr>(E)) {
2487     setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
2488     // We don't know if member is an 'ivar', but this flag is looked at
2489     // only in the context of LV.isObjCIvar().
2490     LV.setObjCArray(E->getType()->isArrayType());
2491     return;
2492   }
2493 }
2494 
2495 static llvm::Value *
2496 EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
2497                                 llvm::Value *V, llvm::Type *IRType,
2498                                 StringRef Name = StringRef()) {
2499   unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
2500   return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
2501 }
2502 
2503 static LValue EmitThreadPrivateVarDeclLValue(
2504     CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
2505     llvm::Type *RealVarTy, SourceLocation Loc) {
2506   if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
2507     Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
2508         CGF, VD, Addr, Loc);
2509   else
2510     Addr =
2511         CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
2512 
2513   Addr = Addr.withElementType(RealVarTy);
2514   return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2515 }
2516 
2517 static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
2518                                            const VarDecl *VD, QualType T) {
2519   std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2520       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2521   // Return an invalid address if variable is MT_To (or MT_Enter starting with
2522   // OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link
2523   // and MT_To (or MT_Enter) with unified memory, return a valid address.
2524   if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2525                 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2526                !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
2527     return Address::invalid();
2528   assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2529           ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2530             *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2531            CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
2532          "Expected link clause OR to clause with unified memory enabled.");
2533   QualType PtrTy = CGF.getContext().getPointerType(VD->getType());
2534   Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
2535   return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>());
2536 }
2537 
2538 Address
2539 CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
2540                                      LValueBaseInfo *PointeeBaseInfo,
2541                                      TBAAAccessInfo *PointeeTBAAInfo) {
2542   llvm::LoadInst *Load =
2543       Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
2544   CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo());
2545 
2546   QualType PointeeType = RefLVal.getType()->getPointeeType();
2547   CharUnits Align = CGM.getNaturalTypeAlignment(
2548       PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
2549       /* forPointeeType= */ true);
2550   return Address(Load, ConvertTypeForMem(PointeeType), Align);
2551 }
2552 
2553 LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
2554   LValueBaseInfo PointeeBaseInfo;
2555   TBAAAccessInfo PointeeTBAAInfo;
2556   Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo,
2557                                             &PointeeTBAAInfo);
2558   return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(),
2559                         PointeeBaseInfo, PointeeTBAAInfo);
2560 }
2561 
2562 Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
2563                                            const PointerType *PtrTy,
2564                                            LValueBaseInfo *BaseInfo,
2565                                            TBAAAccessInfo *TBAAInfo) {
2566   llvm::Value *Addr = Builder.CreateLoad(Ptr);
2567   return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
2568                  CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
2569                                              TBAAInfo,
2570                                              /*forPointeeType=*/true));
2571 }
2572 
2573 LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
2574                                                 const PointerType *PtrTy) {
2575   LValueBaseInfo BaseInfo;
2576   TBAAAccessInfo TBAAInfo;
2577   Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo);
2578   return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
2579 }
2580 
2581 static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
2582                                       const Expr *E, const VarDecl *VD) {
2583   QualType T = E->getType();
2584 
2585   // If it's thread_local, emit a call to its wrapper function instead.
2586   if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
2587       CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
2588     return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T);
2589   // Check if the variable is marked as declare target with link clause in
2590   // device codegen.
2591   if (CGF.getLangOpts().OpenMPIsTargetDevice) {
2592     Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
2593     if (Addr.isValid())
2594       return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2595   }
2596 
2597   llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
2598 
2599   if (VD->getTLSKind() != VarDecl::TLS_None)
2600     V = CGF.Builder.CreateThreadLocalAddress(V);
2601 
2602   llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
2603   V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
2604   CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2605   Address Addr(V, RealVarTy, Alignment);
2606   // Emit reference to the private copy of the variable if it is an OpenMP
2607   // threadprivate variable.
2608   if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
2609       VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2610     return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
2611                                           E->getExprLoc());
2612   }
2613   LValue LV = VD->getType()->isReferenceType() ?
2614       CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2615                                     AlignmentSource::Decl) :
2616       CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2617   setObjCGCLValueClass(CGF.getContext(), E, LV);
2618   return LV;
2619 }
2620 
2621 static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM,
2622                                                GlobalDecl GD) {
2623   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2624   if (FD->hasAttr<WeakRefAttr>()) {
2625     ConstantAddress aliasee = CGM.GetWeakRefReference(FD);
2626     return aliasee.getPointer();
2627   }
2628 
2629   llvm::Constant *V = CGM.GetAddrOfFunction(GD);
2630   if (!FD->hasPrototype()) {
2631     if (const FunctionProtoType *Proto =
2632             FD->getType()->getAs<FunctionProtoType>()) {
2633       // Ugly case: for a K&R-style definition, the type of the definition
2634       // isn't the same as the type of a use.  Correct for this with a
2635       // bitcast.
2636       QualType NoProtoType =
2637           CGM.getContext().getFunctionNoProtoType(Proto->getReturnType());
2638       NoProtoType = CGM.getContext().getPointerType(NoProtoType);
2639       V = llvm::ConstantExpr::getBitCast(V,
2640                                       CGM.getTypes().ConvertType(NoProtoType));
2641     }
2642   }
2643   return V;
2644 }
2645 
2646 static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
2647                                      GlobalDecl GD) {
2648   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2649   llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD);
2650   CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
2651   return CGF.MakeAddrLValue(V, E->getType(), Alignment,
2652                             AlignmentSource::Decl);
2653 }
2654 
2655 static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
2656                                       llvm::Value *ThisValue) {
2657   QualType TagType = CGF.getContext().getTagDeclType(FD->getParent());
2658   LValue LV = CGF.MakeNaturalAlignAddrLValue(ThisValue, TagType);
2659   return CGF.EmitLValueForField(LV, FD);
2660 }
2661 
2662 /// Named Registers are named metadata pointing to the register name
2663 /// which will be read from/written to as an argument to the intrinsic
2664 /// @llvm.read/write_register.
2665 /// So far, only the name is being passed down, but other options such as
2666 /// register type, allocation type or even optimization options could be
2667 /// passed down via the metadata node.
2668 static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
2669   SmallString<64> Name("llvm.named.register.");
2670   AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
2671   assert(Asm->getLabel().size() < 64-Name.size() &&
2672       "Register name too big");
2673   Name.append(Asm->getLabel());
2674   llvm::NamedMDNode *M =
2675     CGM.getModule().getOrInsertNamedMetadata(Name);
2676   if (M->getNumOperands() == 0) {
2677     llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(),
2678                                               Asm->getLabel());
2679     llvm::Metadata *Ops[] = {Str};
2680     M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
2681   }
2682 
2683   CharUnits Alignment = CGM.getContext().getDeclAlign(VD);
2684 
2685   llvm::Value *Ptr =
2686     llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));
2687   return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType());
2688 }
2689 
2690 /// Determine whether we can emit a reference to \p VD from the current
2691 /// context, despite not necessarily having seen an odr-use of the variable in
2692 /// this context.
2693 static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
2694                                                const DeclRefExpr *E,
2695                                                const VarDecl *VD) {
2696   // For a variable declared in an enclosing scope, do not emit a spurious
2697   // reference even if we have a capture, as that will emit an unwarranted
2698   // reference to our capture state, and will likely generate worse code than
2699   // emitting a local copy.
2700   if (E->refersToEnclosingVariableOrCapture())
2701     return false;
2702 
2703   // For a local declaration declared in this function, we can always reference
2704   // it even if we don't have an odr-use.
2705   if (VD->hasLocalStorage()) {
2706     return VD->getDeclContext() ==
2707            dyn_cast_or_null<DeclContext>(CGF.CurCodeDecl);
2708   }
2709 
2710   // For a global declaration, we can emit a reference to it if we know
2711   // for sure that we are able to emit a definition of it.
2712   VD = VD->getDefinition(CGF.getContext());
2713   if (!VD)
2714     return false;
2715 
2716   // Don't emit a spurious reference if it might be to a variable that only
2717   // exists on a different device / target.
2718   // FIXME: This is unnecessarily broad. Check whether this would actually be a
2719   // cross-target reference.
2720   if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
2721       CGF.getLangOpts().OpenCL) {
2722     return false;
2723   }
2724 
2725   // We can emit a spurious reference only if the linkage implies that we'll
2726   // be emitting a non-interposable symbol that will be retained until link
2727   // time.
2728   switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {
2729   case llvm::GlobalValue::ExternalLinkage:
2730   case llvm::GlobalValue::LinkOnceODRLinkage:
2731   case llvm::GlobalValue::WeakODRLinkage:
2732   case llvm::GlobalValue::InternalLinkage:
2733   case llvm::GlobalValue::PrivateLinkage:
2734     return true;
2735   default:
2736     return false;
2737   }
2738 }
2739 
2740 LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
2741   const NamedDecl *ND = E->getDecl();
2742   QualType T = E->getType();
2743 
2744   assert(E->isNonOdrUse() != NOUR_Unevaluated &&
2745          "should not emit an unevaluated operand");
2746 
2747   if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2748     // Global Named registers access via intrinsics only
2749     if (VD->getStorageClass() == SC_Register &&
2750         VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
2751       return EmitGlobalNamedRegister(VD, CGM);
2752 
2753     // If this DeclRefExpr does not constitute an odr-use of the variable,
2754     // we're not permitted to emit a reference to it in general, and it might
2755     // not be captured if capture would be necessary for a use. Emit the
2756     // constant value directly instead.
2757     if (E->isNonOdrUse() == NOUR_Constant &&
2758         (VD->getType()->isReferenceType() ||
2759          !canEmitSpuriousReferenceToVariable(*this, E, VD))) {
2760       VD->getAnyInitializer(VD);
2761       llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
2762           E->getLocation(), *VD->evaluateValue(), VD->getType());
2763       assert(Val && "failed to emit constant expression");
2764 
2765       Address Addr = Address::invalid();
2766       if (!VD->getType()->isReferenceType()) {
2767         // Spill the constant value to a global.
2768         Addr = CGM.createUnnamedGlobalFrom(*VD, Val,
2769                                            getContext().getDeclAlign(VD));
2770         llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());
2771         auto *PTy = llvm::PointerType::get(
2772             VarTy, getTypes().getTargetAddressSpace(VD->getType()));
2773         Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
2774       } else {
2775         // Should we be using the alignment of the constant pointer we emitted?
2776         CharUnits Alignment =
2777             CGM.getNaturalTypeAlignment(E->getType(),
2778                                         /* BaseInfo= */ nullptr,
2779                                         /* TBAAInfo= */ nullptr,
2780                                         /* forPointeeType= */ true);
2781         Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
2782       }
2783       return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2784     }
2785 
2786     // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
2787 
2788     // Check for captured variables.
2789     if (E->refersToEnclosingVariableOrCapture()) {
2790       VD = VD->getCanonicalDecl();
2791       if (auto *FD = LambdaCaptureFields.lookup(VD))
2792         return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2793       if (CapturedStmtInfo) {
2794         auto I = LocalDeclMap.find(VD);
2795         if (I != LocalDeclMap.end()) {
2796           LValue CapLVal;
2797           if (VD->getType()->isReferenceType())
2798             CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(),
2799                                                 AlignmentSource::Decl);
2800           else
2801             CapLVal = MakeAddrLValue(I->second, T);
2802           // Mark lvalue as nontemporal if the variable is marked as nontemporal
2803           // in simd context.
2804           if (getLangOpts().OpenMP &&
2805               CGM.getOpenMPRuntime().isNontemporalDecl(VD))
2806             CapLVal.setNontemporal(/*Value=*/true);
2807           return CapLVal;
2808         }
2809         LValue CapLVal =
2810             EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
2811                                     CapturedStmtInfo->getContextValue());
2812         Address LValueAddress = CapLVal.getAddress(*this);
2813         CapLVal = MakeAddrLValue(
2814             Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
2815                     getContext().getDeclAlign(VD)),
2816             CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl),
2817             CapLVal.getTBAAInfo());
2818         // Mark lvalue as nontemporal if the variable is marked as nontemporal
2819         // in simd context.
2820         if (getLangOpts().OpenMP &&
2821             CGM.getOpenMPRuntime().isNontemporalDecl(VD))
2822           CapLVal.setNontemporal(/*Value=*/true);
2823         return CapLVal;
2824       }
2825 
2826       assert(isa<BlockDecl>(CurCodeDecl));
2827       Address addr = GetAddrOfBlockDecl(VD);
2828       return MakeAddrLValue(addr, T, AlignmentSource::Decl);
2829     }
2830   }
2831 
2832   // FIXME: We should be able to assert this for FunctionDecls as well!
2833   // FIXME: We should be able to assert this for all DeclRefExprs, not just
2834   // those with a valid source location.
2835   assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
2836           !E->getLocation().isValid()) &&
2837          "Should not use decl without marking it used!");
2838 
2839   if (ND->hasAttr<WeakRefAttr>()) {
2840     const auto *VD = cast<ValueDecl>(ND);
2841     ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
2842     return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl);
2843   }
2844 
2845   if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2846     // Check if this is a global variable.
2847     if (VD->hasLinkage() || VD->isStaticDataMember())
2848       return EmitGlobalVarDeclLValue(*this, E, VD);
2849 
2850     Address addr = Address::invalid();
2851 
2852     // The variable should generally be present in the local decl map.
2853     auto iter = LocalDeclMap.find(VD);
2854     if (iter != LocalDeclMap.end()) {
2855       addr = iter->second;
2856 
2857     // Otherwise, it might be static local we haven't emitted yet for
2858     // some reason; most likely, because it's in an outer function.
2859     } else if (VD->isStaticLocal()) {
2860       llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
2861           *VD, CGM.getLLVMLinkageVarDefinition(VD));
2862       addr = Address(
2863           var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
2864 
2865     // No other cases for now.
2866     } else {
2867       llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
2868     }
2869 
2870     // Handle threadlocal function locals.
2871     if (VD->getTLSKind() != VarDecl::TLS_None)
2872       addr = addr.withPointer(
2873           Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull);
2874 
2875     // Check for OpenMP threadprivate variables.
2876     if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
2877         VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2878       return EmitThreadPrivateVarDeclLValue(
2879           *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()),
2880           E->getExprLoc());
2881     }
2882 
2883     // Drill into block byref variables.
2884     bool isBlockByref = VD->isEscapingByref();
2885     if (isBlockByref) {
2886       addr = emitBlockByrefAddress(addr, VD);
2887     }
2888 
2889     // Drill into reference types.
2890     LValue LV = VD->getType()->isReferenceType() ?
2891         EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) :
2892         MakeAddrLValue(addr, T, AlignmentSource::Decl);
2893 
2894     bool isLocalStorage = VD->hasLocalStorage();
2895 
2896     bool NonGCable = isLocalStorage &&
2897                      !VD->getType()->isReferenceType() &&
2898                      !isBlockByref;
2899     if (NonGCable) {
2900       LV.getQuals().removeObjCGCAttr();
2901       LV.setNonGC(true);
2902     }
2903 
2904     bool isImpreciseLifetime =
2905       (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
2906     if (isImpreciseLifetime)
2907       LV.setARCPreciseLifetime(ARCImpreciseLifetime);
2908     setObjCGCLValueClass(getContext(), E, LV);
2909     return LV;
2910   }
2911 
2912   if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
2913     LValue LV = EmitFunctionDeclLValue(*this, E, FD);
2914 
2915     // Emit debuginfo for the function declaration if the target wants to.
2916     if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) {
2917       if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) {
2918         auto *Fn =
2919             cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts());
2920         if (!Fn->getSubprogram())
2921           DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn);
2922       }
2923     }
2924 
2925     return LV;
2926   }
2927 
2928   // FIXME: While we're emitting a binding from an enclosing scope, all other
2929   // DeclRefExprs we see should be implicitly treated as if they also refer to
2930   // an enclosing scope.
2931   if (const auto *BD = dyn_cast<BindingDecl>(ND)) {
2932     if (E->refersToEnclosingVariableOrCapture()) {
2933       auto *FD = LambdaCaptureFields.lookup(BD);
2934       return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2935     }
2936     return EmitLValue(BD->getBinding());
2937   }
2938 
2939   // We can form DeclRefExprs naming GUID declarations when reconstituting
2940   // non-type template parameters into expressions.
2941   if (const auto *GD = dyn_cast<MSGuidDecl>(ND))
2942     return MakeAddrLValue(CGM.GetAddrOfMSGuidDecl(GD), T,
2943                           AlignmentSource::Decl);
2944 
2945   if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND))
2946     return MakeAddrLValue(CGM.GetAddrOfTemplateParamObject(TPO), T,
2947                           AlignmentSource::Decl);
2948 
2949   llvm_unreachable("Unhandled DeclRefExpr");
2950 }
2951 
2952 LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
2953   // __extension__ doesn't affect lvalue-ness.
2954   if (E->getOpcode() == UO_Extension)
2955     return EmitLValue(E->getSubExpr());
2956 
2957   QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
2958   switch (E->getOpcode()) {
2959   default: llvm_unreachable("Unknown unary operator lvalue!");
2960   case UO_Deref: {
2961     QualType T = E->getSubExpr()->getType()->getPointeeType();
2962     assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
2963 
2964     LValueBaseInfo BaseInfo;
2965     TBAAAccessInfo TBAAInfo;
2966     Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &BaseInfo,
2967                                             &TBAAInfo);
2968     LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
2969     LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
2970 
2971     // We should not generate __weak write barrier on indirect reference
2972     // of a pointer to object; as in void foo (__weak id *param); *param = 0;
2973     // But, we continue to generate __strong write barrier on indirect write
2974     // into a pointer to object.
2975     if (getLangOpts().ObjC &&
2976         getLangOpts().getGC() != LangOptions::NonGC &&
2977         LV.isObjCWeak())
2978       LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
2979     return LV;
2980   }
2981   case UO_Real:
2982   case UO_Imag: {
2983     LValue LV = EmitLValue(E->getSubExpr());
2984     assert(LV.isSimple() && "real/imag on non-ordinary l-value");
2985 
2986     // __real is valid on scalars.  This is a faster way of testing that.
2987     // __imag can only produce an rvalue on scalars.
2988     if (E->getOpcode() == UO_Real &&
2989         !LV.getAddress(*this).getElementType()->isStructTy()) {
2990       assert(E->getSubExpr()->getType()->isArithmeticType());
2991       return LV;
2992     }
2993 
2994     QualType T = ExprTy->castAs<ComplexType>()->getElementType();
2995 
2996     Address Component =
2997         (E->getOpcode() == UO_Real
2998              ? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType())
2999              : emitAddrOfImagComponent(LV.getAddress(*this), LV.getType()));
3000     LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(),
3001                                    CGM.getTBAAInfoForSubobject(LV, T));
3002     ElemLV.getQuals().addQualifiers(LV.getQuals());
3003     return ElemLV;
3004   }
3005   case UO_PreInc:
3006   case UO_PreDec: {
3007     LValue LV = EmitLValue(E->getSubExpr());
3008     bool isInc = E->getOpcode() == UO_PreInc;
3009 
3010     if (E->getType()->isAnyComplexType())
3011       EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
3012     else
3013       EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
3014     return LV;
3015   }
3016   }
3017 }
3018 
3019 LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
3020   return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
3021                         E->getType(), AlignmentSource::Decl);
3022 }
3023 
3024 LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
3025   return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
3026                         E->getType(), AlignmentSource::Decl);
3027 }
3028 
3029 LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
3030   auto SL = E->getFunctionName();
3031   assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3032   StringRef FnName = CurFn->getName();
3033   if (FnName.startswith("\01"))
3034     FnName = FnName.substr(1);
3035   StringRef NameItems[] = {
3036       PredefinedExpr::getIdentKindName(E->getIdentKind()), FnName};
3037   std::string GVName = llvm::join(NameItems, NameItems + 2, ".");
3038   if (auto *BD = dyn_cast_or_null<BlockDecl>(CurCodeDecl)) {
3039     std::string Name = std::string(SL->getString());
3040     if (!Name.empty()) {
3041       unsigned Discriminator =
3042           CGM.getCXXABI().getMangleContext().getBlockId(BD, true);
3043       if (Discriminator)
3044         Name += "_" + Twine(Discriminator + 1).str();
3045       auto C = CGM.GetAddrOfConstantCString(Name, GVName.c_str());
3046       return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3047     } else {
3048       auto C =
3049           CGM.GetAddrOfConstantCString(std::string(FnName), GVName.c_str());
3050       return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3051     }
3052   }
3053   auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName);
3054   return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3055 }
3056 
3057 /// Emit a type description suitable for use by a runtime sanitizer library. The
3058 /// format of a type descriptor is
3059 ///
3060 /// \code
3061 ///   { i16 TypeKind, i16 TypeInfo }
3062 /// \endcode
3063 ///
3064 /// followed by an array of i8 containing the type name. TypeKind is 0 for an
3065 /// integer, 1 for a floating point value, and -1 for anything else.
3066 llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
3067   // Only emit each type's descriptor once.
3068   if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))
3069     return C;
3070 
3071   uint16_t TypeKind = -1;
3072   uint16_t TypeInfo = 0;
3073 
3074   if (T->isIntegerType()) {
3075     TypeKind = 0;
3076     TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
3077                (T->isSignedIntegerType() ? 1 : 0);
3078   } else if (T->isFloatingType()) {
3079     TypeKind = 1;
3080     TypeInfo = getContext().getTypeSize(T);
3081   }
3082 
3083   // Format the type name as if for a diagnostic, including quotes and
3084   // optionally an 'aka'.
3085   SmallString<32> Buffer;
3086   CGM.getDiags().ConvertArgToString(
3087       DiagnosticsEngine::ak_qualtype, (intptr_t)T.getAsOpaquePtr(), StringRef(),
3088       StringRef(), std::nullopt, Buffer, std::nullopt);
3089 
3090   llvm::Constant *Components[] = {
3091     Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
3092     llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
3093   };
3094   llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
3095 
3096   auto *GV = new llvm::GlobalVariable(
3097       CGM.getModule(), Descriptor->getType(),
3098       /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3099   GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3100   CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
3101 
3102   // Remember the descriptor for this type.
3103   CGM.setTypeDescriptorInMap(T, GV);
3104 
3105   return GV;
3106 }
3107 
3108 llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
3109   llvm::Type *TargetTy = IntPtrTy;
3110 
3111   if (V->getType() == TargetTy)
3112     return V;
3113 
3114   // Floating-point types which fit into intptr_t are bitcast to integers
3115   // and then passed directly (after zero-extension, if necessary).
3116   if (V->getType()->isFloatingPointTy()) {
3117     unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
3118     if (Bits <= TargetTy->getIntegerBitWidth())
3119       V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(),
3120                                                          Bits));
3121   }
3122 
3123   // Integers which fit in intptr_t are zero-extended and passed directly.
3124   if (V->getType()->isIntegerTy() &&
3125       V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3126     return Builder.CreateZExt(V, TargetTy);
3127 
3128   // Pointers are passed directly, everything else is passed by address.
3129   if (!V->getType()->isPointerTy()) {
3130     Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
3131     Builder.CreateStore(V, Ptr);
3132     V = Ptr.getPointer();
3133   }
3134   return Builder.CreatePtrToInt(V, TargetTy);
3135 }
3136 
3137 /// Emit a representation of a SourceLocation for passing to a handler
3138 /// in a sanitizer runtime library. The format for this data is:
3139 /// \code
3140 ///   struct SourceLocation {
3141 ///     const char *Filename;
3142 ///     int32_t Line, Column;
3143 ///   };
3144 /// \endcode
3145 /// For an invalid SourceLocation, the Filename pointer is null.
3146 llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
3147   llvm::Constant *Filename;
3148   int Line, Column;
3149 
3150   PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
3151   if (PLoc.isValid()) {
3152     StringRef FilenameString = PLoc.getFilename();
3153 
3154     int PathComponentsToStrip =
3155         CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
3156     if (PathComponentsToStrip < 0) {
3157       assert(PathComponentsToStrip != INT_MIN);
3158       int PathComponentsToKeep = -PathComponentsToStrip;
3159       auto I = llvm::sys::path::rbegin(FilenameString);
3160       auto E = llvm::sys::path::rend(FilenameString);
3161       while (I != E && --PathComponentsToKeep)
3162         ++I;
3163 
3164       FilenameString = FilenameString.substr(I - E);
3165     } else if (PathComponentsToStrip > 0) {
3166       auto I = llvm::sys::path::begin(FilenameString);
3167       auto E = llvm::sys::path::end(FilenameString);
3168       while (I != E && PathComponentsToStrip--)
3169         ++I;
3170 
3171       if (I != E)
3172         FilenameString =
3173             FilenameString.substr(I - llvm::sys::path::begin(FilenameString));
3174       else
3175         FilenameString = llvm::sys::path::filename(FilenameString);
3176     }
3177 
3178     auto FilenameGV =
3179         CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src");
3180     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
3181         cast<llvm::GlobalVariable>(
3182             FilenameGV.getPointer()->stripPointerCasts()));
3183     Filename = FilenameGV.getPointer();
3184     Line = PLoc.getLine();
3185     Column = PLoc.getColumn();
3186   } else {
3187     Filename = llvm::Constant::getNullValue(Int8PtrTy);
3188     Line = Column = 0;
3189   }
3190 
3191   llvm::Constant *Data[] = {Filename, Builder.getInt32(Line),
3192                             Builder.getInt32(Column)};
3193 
3194   return llvm::ConstantStruct::getAnon(Data);
3195 }
3196 
3197 namespace {
3198 /// Specify under what conditions this check can be recovered
3199 enum class CheckRecoverableKind {
3200   /// Always terminate program execution if this check fails.
3201   Unrecoverable,
3202   /// Check supports recovering, runtime has both fatal (noreturn) and
3203   /// non-fatal handlers for this check.
3204   Recoverable,
3205   /// Runtime conditionally aborts, always need to support recovery.
3206   AlwaysRecoverable
3207 };
3208 }
3209 
3210 static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) {
3211   assert(Kind.countPopulation() == 1);
3212   if (Kind == SanitizerKind::Vptr)
3213     return CheckRecoverableKind::AlwaysRecoverable;
3214   else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3215     return CheckRecoverableKind::Unrecoverable;
3216   else
3217     return CheckRecoverableKind::Recoverable;
3218 }
3219 
3220 namespace {
3221 struct SanitizerHandlerInfo {
3222   char const *const Name;
3223   unsigned Version;
3224 };
3225 }
3226 
3227 const SanitizerHandlerInfo SanitizerHandlers[] = {
3228 #define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3229     LIST_SANITIZER_CHECKS
3230 #undef SANITIZER_CHECK
3231 };
3232 
3233 static void emitCheckHandlerCall(CodeGenFunction &CGF,
3234                                  llvm::FunctionType *FnType,
3235                                  ArrayRef<llvm::Value *> FnArgs,
3236                                  SanitizerHandler CheckHandler,
3237                                  CheckRecoverableKind RecoverKind, bool IsFatal,
3238                                  llvm::BasicBlock *ContBB) {
3239   assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3240   std::optional<ApplyDebugLocation> DL;
3241   if (!CGF.Builder.getCurrentDebugLocation()) {
3242     // Ensure that the call has at least an artificial debug location.
3243     DL.emplace(CGF, SourceLocation());
3244   }
3245   bool NeedsAbortSuffix =
3246       IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3247   bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
3248   const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
3249   const StringRef CheckName = CheckInfo.Name;
3250   std::string FnName = "__ubsan_handle_" + CheckName.str();
3251   if (CheckInfo.Version && !MinimalRuntime)
3252     FnName += "_v" + llvm::utostr(CheckInfo.Version);
3253   if (MinimalRuntime)
3254     FnName += "_minimal";
3255   if (NeedsAbortSuffix)
3256     FnName += "_abort";
3257   bool MayReturn =
3258       !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3259 
3260   llvm::AttrBuilder B(CGF.getLLVMContext());
3261   if (!MayReturn) {
3262     B.addAttribute(llvm::Attribute::NoReturn)
3263         .addAttribute(llvm::Attribute::NoUnwind);
3264   }
3265   B.addUWTableAttr(llvm::UWTableKind::Default);
3266 
3267   llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
3268       FnType, FnName,
3269       llvm::AttributeList::get(CGF.getLLVMContext(),
3270                                llvm::AttributeList::FunctionIndex, B),
3271       /*Local=*/true);
3272   llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs);
3273   if (!MayReturn) {
3274     HandlerCall->setDoesNotReturn();
3275     CGF.Builder.CreateUnreachable();
3276   } else {
3277     CGF.Builder.CreateBr(ContBB);
3278   }
3279 }
3280 
3281 void CodeGenFunction::EmitCheck(
3282     ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3283     SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
3284     ArrayRef<llvm::Value *> DynamicArgs) {
3285   assert(IsSanitizerScope);
3286   assert(Checked.size() > 0);
3287   assert(CheckHandler >= 0 &&
3288          size_t(CheckHandler) < std::size(SanitizerHandlers));
3289   const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
3290 
3291   llvm::Value *FatalCond = nullptr;
3292   llvm::Value *RecoverableCond = nullptr;
3293   llvm::Value *TrapCond = nullptr;
3294   for (int i = 0, n = Checked.size(); i < n; ++i) {
3295     llvm::Value *Check = Checked[i].first;
3296     // -fsanitize-trap= overrides -fsanitize-recover=.
3297     llvm::Value *&Cond =
3298         CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second)
3299             ? TrapCond
3300             : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second)
3301                   ? RecoverableCond
3302                   : FatalCond;
3303     Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check;
3304   }
3305 
3306   if (TrapCond)
3307     EmitTrapCheck(TrapCond, CheckHandler);
3308   if (!FatalCond && !RecoverableCond)
3309     return;
3310 
3311   llvm::Value *JointCond;
3312   if (FatalCond && RecoverableCond)
3313     JointCond = Builder.CreateAnd(FatalCond, RecoverableCond);
3314   else
3315     JointCond = FatalCond ? FatalCond : RecoverableCond;
3316   assert(JointCond);
3317 
3318   CheckRecoverableKind RecoverKind = getRecoverableKind(Checked[0].second);
3319   assert(SanOpts.has(Checked[0].second));
3320 #ifndef NDEBUG
3321   for (int i = 1, n = Checked.size(); i < n; ++i) {
3322     assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
3323            "All recoverable kinds in a single check must be same!");
3324     assert(SanOpts.has(Checked[i].second));
3325   }
3326 #endif
3327 
3328   llvm::BasicBlock *Cont = createBasicBlock("cont");
3329   llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName);
3330   llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers);
3331   // Give hint that we very much don't expect to execute the handler
3332   // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
3333   llvm::MDBuilder MDHelper(getLLVMContext());
3334   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3335   Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);
3336   EmitBlock(Handlers);
3337 
3338   // Handler functions take an i8* pointing to the (handler-specific) static
3339   // information block, followed by a sequence of intptr_t arguments
3340   // representing operand values.
3341   SmallVector<llvm::Value *, 4> Args;
3342   SmallVector<llvm::Type *, 4> ArgTypes;
3343   if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
3344     Args.reserve(DynamicArgs.size() + 1);
3345     ArgTypes.reserve(DynamicArgs.size() + 1);
3346 
3347     // Emit handler arguments and create handler function type.
3348     if (!StaticArgs.empty()) {
3349       llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3350       auto *InfoPtr = new llvm::GlobalVariable(
3351           CGM.getModule(), Info->getType(), false,
3352           llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr,
3353           llvm::GlobalVariable::NotThreadLocal,
3354           CGM.getDataLayout().getDefaultGlobalsAddressSpace());
3355       InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3356       CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
3357       Args.push_back(InfoPtr);
3358       ArgTypes.push_back(Args.back()->getType());
3359     }
3360 
3361     for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3362       Args.push_back(EmitCheckValue(DynamicArgs[i]));
3363       ArgTypes.push_back(IntPtrTy);
3364     }
3365   }
3366 
3367   llvm::FunctionType *FnType =
3368     llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);
3369 
3370   if (!FatalCond || !RecoverableCond) {
3371     // Simple case: we need to generate a single handler call, either
3372     // fatal, or non-fatal.
3373     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,
3374                          (FatalCond != nullptr), Cont);
3375   } else {
3376     // Emit two handler calls: first one for set of unrecoverable checks,
3377     // another one for recoverable.
3378     llvm::BasicBlock *NonFatalHandlerBB =
3379         createBasicBlock("non_fatal." + CheckName);
3380     llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName);
3381     Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
3382     EmitBlock(FatalHandlerBB);
3383     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,
3384                          NonFatalHandlerBB);
3385     EmitBlock(NonFatalHandlerBB);
3386     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,
3387                          Cont);
3388   }
3389 
3390   EmitBlock(Cont);
3391 }
3392 
3393 void CodeGenFunction::EmitCfiSlowPathCheck(
3394     SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3395     llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) {
3396   llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");
3397 
3398   llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");
3399   llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);
3400 
3401   llvm::MDBuilder MDHelper(getLLVMContext());
3402   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3403   BI->setMetadata(llvm::LLVMContext::MD_prof, Node);
3404 
3405   EmitBlock(CheckBB);
3406 
3407   bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Kind);
3408 
3409   llvm::CallInst *CheckCall;
3410   llvm::FunctionCallee SlowPathFn;
3411   if (WithDiag) {
3412     llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3413     auto *InfoPtr =
3414         new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
3415                                  llvm::GlobalVariable::PrivateLinkage, Info);
3416     InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3417     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
3418 
3419     SlowPathFn = CGM.getModule().getOrInsertFunction(
3420         "__cfi_slowpath_diag",
3421         llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy},
3422                                 false));
3423     CheckCall = Builder.CreateCall(
3424         SlowPathFn, {TypeId, Ptr, Builder.CreateBitCast(InfoPtr, Int8PtrTy)});
3425   } else {
3426     SlowPathFn = CGM.getModule().getOrInsertFunction(
3427         "__cfi_slowpath",
3428         llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false));
3429     CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
3430   }
3431 
3432   CGM.setDSOLocal(
3433       cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));
3434   CheckCall->setDoesNotThrow();
3435 
3436   EmitBlock(Cont);
3437 }
3438 
3439 // Emit a stub for __cfi_check function so that the linker knows about this
3440 // symbol in LTO mode.
3441 void CodeGenFunction::EmitCfiCheckStub() {
3442   llvm::Module *M = &CGM.getModule();
3443   auto &Ctx = M->getContext();
3444   llvm::Function *F = llvm::Function::Create(
3445       llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false),
3446       llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M);
3447   CGM.setDSOLocal(F);
3448   llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F);
3449   // FIXME: consider emitting an intrinsic call like
3450   // call void @llvm.cfi_check(i64 %0, i8* %1, i8* %2)
3451   // which can be lowered in CrossDSOCFI pass to the actual contents of
3452   // __cfi_check. This would allow inlining of __cfi_check calls.
3453   llvm::CallInst::Create(
3454       llvm::Intrinsic::getDeclaration(M, llvm::Intrinsic::trap), "", BB);
3455   llvm::ReturnInst::Create(Ctx, nullptr, BB);
3456 }
3457 
3458 // This function is basically a switch over the CFI failure kind, which is
3459 // extracted from CFICheckFailData (1st function argument). Each case is either
3460 // llvm.trap or a call to one of the two runtime handlers, based on
3461 // -fsanitize-trap and -fsanitize-recover settings.  Default case (invalid
3462 // failure kind) traps, but this should really never happen.  CFICheckFailData
3463 // can be nullptr if the calling module has -fsanitize-trap behavior for this
3464 // check kind; in this case __cfi_check_fail traps as well.
3465 void CodeGenFunction::EmitCfiCheckFail() {
3466   SanitizerScope SanScope(this);
3467   FunctionArgList Args;
3468   ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy,
3469                             ImplicitParamDecl::Other);
3470   ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy,
3471                             ImplicitParamDecl::Other);
3472   Args.push_back(&ArgData);
3473   Args.push_back(&ArgAddr);
3474 
3475   const CGFunctionInfo &FI =
3476     CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, Args);
3477 
3478   llvm::Function *F = llvm::Function::Create(
3479       llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false),
3480       llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule());
3481 
3482   CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);
3483   CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
3484   F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3485 
3486   StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args,
3487                 SourceLocation());
3488 
3489   // This function is not affected by NoSanitizeList. This function does
3490   // not have a source location, but "src:*" would still apply. Revert any
3491   // changes to SanOpts made in StartFunction.
3492   SanOpts = CGM.getLangOpts().Sanitize;
3493 
3494   llvm::Value *Data =
3495       EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,
3496                        CGM.getContext().VoidPtrTy, ArgData.getLocation());
3497   llvm::Value *Addr =
3498       EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,
3499                        CGM.getContext().VoidPtrTy, ArgAddr.getLocation());
3500 
3501   // Data == nullptr means the calling module has trap behaviour for this check.
3502   llvm::Value *DataIsNotNullPtr =
3503       Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy));
3504   EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
3505 
3506   llvm::StructType *SourceLocationTy =
3507       llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty);
3508   llvm::StructType *CfiCheckFailDataTy =
3509       llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy);
3510 
3511   llvm::Value *V = Builder.CreateConstGEP2_32(
3512       CfiCheckFailDataTy,
3513       Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
3514       0);
3515 
3516   Address CheckKindAddr(V, Int8Ty, getIntAlign());
3517   llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
3518 
3519   llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3520       CGM.getLLVMContext(),
3521       llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
3522   llvm::Value *ValidVtable = Builder.CreateZExt(
3523       Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
3524                          {Addr, AllVtables}),
3525       IntPtrTy);
3526 
3527   const std::pair<int, SanitizerMask> CheckKinds[] = {
3528       {CFITCK_VCall, SanitizerKind::CFIVCall},
3529       {CFITCK_NVCall, SanitizerKind::CFINVCall},
3530       {CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast},
3531       {CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast},
3532       {CFITCK_ICall, SanitizerKind::CFIICall}};
3533 
3534   SmallVector<std::pair<llvm::Value *, SanitizerMask>, 5> Checks;
3535   for (auto CheckKindMaskPair : CheckKinds) {
3536     int Kind = CheckKindMaskPair.first;
3537     SanitizerMask Mask = CheckKindMaskPair.second;
3538     llvm::Value *Cond =
3539         Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind));
3540     if (CGM.getLangOpts().Sanitize.has(Mask))
3541       EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
3542                 {Data, Addr, ValidVtable});
3543     else
3544       EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail);
3545   }
3546 
3547   FinishFunction();
3548   // The only reference to this function will be created during LTO link.
3549   // Make sure it survives until then.
3550   CGM.addUsedGlobal(F);
3551 }
3552 
3553 void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
3554   if (SanOpts.has(SanitizerKind::Unreachable)) {
3555     SanitizerScope SanScope(this);
3556     EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
3557                              SanitizerKind::Unreachable),
3558               SanitizerHandler::BuiltinUnreachable,
3559               EmitCheckSourceLocation(Loc), std::nullopt);
3560   }
3561   Builder.CreateUnreachable();
3562 }
3563 
3564 void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
3565                                     SanitizerHandler CheckHandlerID) {
3566   llvm::BasicBlock *Cont = createBasicBlock("cont");
3567 
3568   // If we're optimizing, collapse all calls to trap down to just one per
3569   // check-type per function to save on code size.
3570   if (TrapBBs.size() <= CheckHandlerID)
3571     TrapBBs.resize(CheckHandlerID + 1);
3572   llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3573 
3574   if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB ||
3575       (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>())) {
3576     TrapBB = createBasicBlock("trap");
3577     Builder.CreateCondBr(Checked, Cont, TrapBB);
3578     EmitBlock(TrapBB);
3579 
3580     llvm::CallInst *TrapCall =
3581         Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),
3582                            llvm::ConstantInt::get(CGM.Int8Ty, CheckHandlerID));
3583 
3584     if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3585       auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3586                                     CGM.getCodeGenOpts().TrapFuncName);
3587       TrapCall->addFnAttr(A);
3588     }
3589     TrapCall->setDoesNotReturn();
3590     TrapCall->setDoesNotThrow();
3591     Builder.CreateUnreachable();
3592   } else {
3593     auto Call = TrapBB->begin();
3594     assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
3595 
3596     Call->applyMergedLocation(Call->getDebugLoc(),
3597                               Builder.getCurrentDebugLocation());
3598     Builder.CreateCondBr(Checked, Cont, TrapBB);
3599   }
3600 
3601   EmitBlock(Cont);
3602 }
3603 
3604 llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
3605   llvm::CallInst *TrapCall =
3606       Builder.CreateCall(CGM.getIntrinsic(IntrID));
3607 
3608   if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3609     auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3610                                   CGM.getCodeGenOpts().TrapFuncName);
3611     TrapCall->addFnAttr(A);
3612   }
3613 
3614   return TrapCall;
3615 }
3616 
3617 Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
3618                                                  LValueBaseInfo *BaseInfo,
3619                                                  TBAAAccessInfo *TBAAInfo) {
3620   assert(E->getType()->isArrayType() &&
3621          "Array to pointer decay must have array source type!");
3622 
3623   // Expressions of array type can't be bitfields or vector elements.
3624   LValue LV = EmitLValue(E);
3625   Address Addr = LV.getAddress(*this);
3626 
3627   // If the array type was an incomplete type, we need to make sure
3628   // the decay ends up being the right type.
3629   llvm::Type *NewTy = ConvertType(E->getType());
3630   Addr = Addr.withElementType(NewTy);
3631 
3632   // Note that VLA pointers are always decayed, so we don't need to do
3633   // anything here.
3634   if (!E->getType()->isVariableArrayType()) {
3635     assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3636            "Expected pointer to array");
3637     Addr = Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3638   }
3639 
3640   // The result of this decay conversion points to an array element within the
3641   // base lvalue. However, since TBAA currently does not support representing
3642   // accesses to elements of member arrays, we conservatively represent accesses
3643   // to the pointee object as if it had no any base lvalue specified.
3644   // TODO: Support TBAA for member arrays.
3645   QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
3646   if (BaseInfo) *BaseInfo = LV.getBaseInfo();
3647   if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType);
3648 
3649   return Addr.withElementType(ConvertTypeForMem(EltType));
3650 }
3651 
3652 /// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
3653 /// array to pointer, return the array subexpression.
3654 static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
3655   // If this isn't just an array->pointer decay, bail out.
3656   const auto *CE = dyn_cast<CastExpr>(E);
3657   if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
3658     return nullptr;
3659 
3660   // If this is a decay from variable width array, bail out.
3661   const Expr *SubExpr = CE->getSubExpr();
3662   if (SubExpr->getType()->isVariableArrayType())
3663     return nullptr;
3664 
3665   return SubExpr;
3666 }
3667 
3668 static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
3669                                           llvm::Type *elemType,
3670                                           llvm::Value *ptr,
3671                                           ArrayRef<llvm::Value*> indices,
3672                                           bool inbounds,
3673                                           bool signedIndices,
3674                                           SourceLocation loc,
3675                                     const llvm::Twine &name = "arrayidx") {
3676   if (inbounds) {
3677     return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices,
3678                                       CodeGenFunction::NotSubtraction, loc,
3679                                       name);
3680   } else {
3681     return CGF.Builder.CreateGEP(elemType, ptr, indices, name);
3682   }
3683 }
3684 
3685 static CharUnits getArrayElementAlign(CharUnits arrayAlign,
3686                                       llvm::Value *idx,
3687                                       CharUnits eltSize) {
3688   // If we have a constant index, we can use the exact offset of the
3689   // element we're accessing.
3690   if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
3691     CharUnits offset = constantIdx->getZExtValue() * eltSize;
3692     return arrayAlign.alignmentAtOffset(offset);
3693 
3694   // Otherwise, use the worst-case alignment for any element.
3695   } else {
3696     return arrayAlign.alignmentOfArrayElement(eltSize);
3697   }
3698 }
3699 
3700 static QualType getFixedSizeElementType(const ASTContext &ctx,
3701                                         const VariableArrayType *vla) {
3702   QualType eltType;
3703   do {
3704     eltType = vla->getElementType();
3705   } while ((vla = ctx.getAsVariableArrayType(eltType)));
3706   return eltType;
3707 }
3708 
3709 /// Given an array base, check whether its member access belongs to a record
3710 /// with preserve_access_index attribute or not.
3711 static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
3712   if (!ArrayBase || !CGF.getDebugInfo())
3713     return false;
3714 
3715   // Only support base as either a MemberExpr or DeclRefExpr.
3716   // DeclRefExpr to cover cases like:
3717   //    struct s { int a; int b[10]; };
3718   //    struct s *p;
3719   //    p[1].a
3720   // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
3721   // p->b[5] is a MemberExpr example.
3722   const Expr *E = ArrayBase->IgnoreImpCasts();
3723   if (const auto *ME = dyn_cast<MemberExpr>(E))
3724     return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3725 
3726   if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
3727     const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());
3728     if (!VarDef)
3729       return false;
3730 
3731     const auto *PtrT = VarDef->getType()->getAs<PointerType>();
3732     if (!PtrT)
3733       return false;
3734 
3735     const auto *PointeeT = PtrT->getPointeeType()
3736                              ->getUnqualifiedDesugaredType();
3737     if (const auto *RecT = dyn_cast<RecordType>(PointeeT))
3738       return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3739     return false;
3740   }
3741 
3742   return false;
3743 }
3744 
3745 static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
3746                                      ArrayRef<llvm::Value *> indices,
3747                                      QualType eltType, bool inbounds,
3748                                      bool signedIndices, SourceLocation loc,
3749                                      QualType *arrayType = nullptr,
3750                                      const Expr *Base = nullptr,
3751                                      const llvm::Twine &name = "arrayidx") {
3752   // All the indices except that last must be zero.
3753 #ifndef NDEBUG
3754   for (auto *idx : indices.drop_back())
3755     assert(isa<llvm::ConstantInt>(idx) &&
3756            cast<llvm::ConstantInt>(idx)->isZero());
3757 #endif
3758 
3759   // Determine the element size of the statically-sized base.  This is
3760   // the thing that the indices are expressed in terms of.
3761   if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) {
3762     eltType = getFixedSizeElementType(CGF.getContext(), vla);
3763   }
3764 
3765   // We can use that to compute the best alignment of the element.
3766   CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
3767   CharUnits eltAlign =
3768     getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
3769 
3770   llvm::Value *eltPtr;
3771   auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
3772   if (!LastIndex ||
3773       (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) {
3774     eltPtr = emitArraySubscriptGEP(
3775         CGF, addr.getElementType(), addr.getPointer(), indices, inbounds,
3776         signedIndices, loc, name);
3777   } else {
3778     // Remember the original array subscript for bpf target
3779     unsigned idx = LastIndex->getZExtValue();
3780     llvm::DIType *DbgInfo = nullptr;
3781     if (arrayType)
3782       DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);
3783     eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(),
3784                                                         addr.getPointer(),
3785                                                         indices.size() - 1,
3786                                                         idx, DbgInfo);
3787   }
3788 
3789   return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
3790 }
3791 
3792 LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
3793                                                bool Accessed) {
3794   // The index must always be an integer, which is not an aggregate.  Emit it
3795   // in lexical order (this complexity is, sadly, required by C++17).
3796   llvm::Value *IdxPre =
3797       (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr;
3798   bool SignedIndices = false;
3799   auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
3800     auto *Idx = IdxPre;
3801     if (E->getLHS() != E->getIdx()) {
3802       assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
3803       Idx = EmitScalarExpr(E->getIdx());
3804     }
3805 
3806     QualType IdxTy = E->getIdx()->getType();
3807     bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
3808     SignedIndices |= IdxSigned;
3809 
3810     if (SanOpts.has(SanitizerKind::ArrayBounds))
3811       EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed);
3812 
3813     // Extend or truncate the index type to 32 or 64-bits.
3814     if (Promote && Idx->getType() != IntPtrTy)
3815       Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
3816 
3817     return Idx;
3818   };
3819   IdxPre = nullptr;
3820 
3821   // If the base is a vector type, then we are forming a vector element lvalue
3822   // with this subscript.
3823   if (E->getBase()->getType()->isVectorType() &&
3824       !isa<ExtVectorElementExpr>(E->getBase())) {
3825     // Emit the vector as an lvalue to get its address.
3826     LValue LHS = EmitLValue(E->getBase());
3827     auto *Idx = EmitIdxAfterBase(/*Promote*/false);
3828     assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
3829     return LValue::MakeVectorElt(LHS.getAddress(*this), Idx,
3830                                  E->getBase()->getType(), LHS.getBaseInfo(),
3831                                  TBAAAccessInfo());
3832   }
3833 
3834   // All the other cases basically behave like simple offsetting.
3835 
3836   // Handle the extvector case we ignored above.
3837   if (isa<ExtVectorElementExpr>(E->getBase())) {
3838     LValue LV = EmitLValue(E->getBase());
3839     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3840     Address Addr = EmitExtVectorElementLValue(LV);
3841 
3842     QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
3843     Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true,
3844                                  SignedIndices, E->getExprLoc());
3845     return MakeAddrLValue(Addr, EltType, LV.getBaseInfo(),
3846                           CGM.getTBAAInfoForSubobject(LV, EltType));
3847   }
3848 
3849   LValueBaseInfo EltBaseInfo;
3850   TBAAAccessInfo EltTBAAInfo;
3851   Address Addr = Address::invalid();
3852   if (const VariableArrayType *vla =
3853            getContext().getAsVariableArrayType(E->getType())) {
3854     // The base must be a pointer, which is not an aggregate.  Emit
3855     // it.  It needs to be emitted first in case it's what captures
3856     // the VLA bounds.
3857     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3858     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3859 
3860     // The element count here is the total number of non-VLA elements.
3861     llvm::Value *numElements = getVLASize(vla).NumElts;
3862 
3863     // Effectively, the multiply by the VLA size is part of the GEP.
3864     // GEP indexes are signed, and scaling an index isn't permitted to
3865     // signed-overflow, so we use the same semantics for our explicit
3866     // multiply.  We suppress this if overflow is not undefined behavior.
3867     if (getLangOpts().isSignedOverflowDefined()) {
3868       Idx = Builder.CreateMul(Idx, numElements);
3869     } else {
3870       Idx = Builder.CreateNSWMul(Idx, numElements);
3871     }
3872 
3873     Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
3874                                  !getLangOpts().isSignedOverflowDefined(),
3875                                  SignedIndices, E->getExprLoc());
3876 
3877   } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
3878     // Indexing over an interface, as in "NSString *P; P[4];"
3879 
3880     // Emit the base pointer.
3881     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3882     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3883 
3884     CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);
3885     llvm::Value *InterfaceSizeVal =
3886         llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());
3887 
3888     llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal);
3889 
3890     // We don't necessarily build correct LLVM struct types for ObjC
3891     // interfaces, so we can't rely on GEP to do this scaling
3892     // correctly, so we need to cast to i8*.  FIXME: is this actually
3893     // true?  A lot of other things in the fragile ABI would break...
3894     llvm::Type *OrigBaseElemTy = Addr.getElementType();
3895 
3896     // Do the GEP.
3897     CharUnits EltAlign =
3898       getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
3899     llvm::Value *EltPtr =
3900         emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx,
3901                               false, SignedIndices, E->getExprLoc());
3902     Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
3903   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
3904     // If this is A[i] where A is an array, the frontend will have decayed the
3905     // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
3906     // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
3907     // "gep x, i" here.  Emit one "gep A, 0, i".
3908     assert(Array->getType()->isArrayType() &&
3909            "Array to pointer decay must have array source type!");
3910     LValue ArrayLV;
3911     // For simple multidimensional array indexing, set the 'accessed' flag for
3912     // better bounds-checking of the base expression.
3913     if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
3914       ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
3915     else
3916       ArrayLV = EmitLValue(Array);
3917     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3918 
3919     // Propagate the alignment from the array itself to the result.
3920     QualType arrayType = Array->getType();
3921     Addr = emitArraySubscriptGEP(
3922         *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
3923         E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices,
3924         E->getExprLoc(), &arrayType, E->getBase());
3925     EltBaseInfo = ArrayLV.getBaseInfo();
3926     EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
3927   } else {
3928     // The base must be a pointer; emit it with an estimate of its alignment.
3929     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3930     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3931     QualType ptrType = E->getBase()->getType();
3932     Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
3933                                  !getLangOpts().isSignedOverflowDefined(),
3934                                  SignedIndices, E->getExprLoc(), &ptrType,
3935                                  E->getBase());
3936   }
3937 
3938   LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
3939 
3940   if (getLangOpts().ObjC &&
3941       getLangOpts().getGC() != LangOptions::NonGC) {
3942     LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
3943     setObjCGCLValueClass(getContext(), E, LV);
3944   }
3945   return LV;
3946 }
3947 
3948 LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
3949   assert(
3950       !E->isIncomplete() &&
3951       "incomplete matrix subscript expressions should be rejected during Sema");
3952   LValue Base = EmitLValue(E->getBase());
3953   llvm::Value *RowIdx = EmitScalarExpr(E->getRowIdx());
3954   llvm::Value *ColIdx = EmitScalarExpr(E->getColumnIdx());
3955   llvm::Value *NumRows = Builder.getIntN(
3956       RowIdx->getType()->getScalarSizeInBits(),
3957       E->getBase()->getType()->castAs<ConstantMatrixType>()->getNumRows());
3958   llvm::Value *FinalIdx =
3959       Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx);
3960   return LValue::MakeMatrixElt(
3961       MaybeConvertMatrixAddress(Base.getAddress(*this), *this), FinalIdx,
3962       E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo());
3963 }
3964 
3965 static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
3966                                        LValueBaseInfo &BaseInfo,
3967                                        TBAAAccessInfo &TBAAInfo,
3968                                        QualType BaseTy, QualType ElTy,
3969                                        bool IsLowerBound) {
3970   LValue BaseLVal;
3971   if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) {
3972     BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound);
3973     if (BaseTy->isArrayType()) {
3974       Address Addr = BaseLVal.getAddress(CGF);
3975       BaseInfo = BaseLVal.getBaseInfo();
3976 
3977       // If the array type was an incomplete type, we need to make sure
3978       // the decay ends up being the right type.
3979       llvm::Type *NewTy = CGF.ConvertType(BaseTy);
3980       Addr = Addr.withElementType(NewTy);
3981 
3982       // Note that VLA pointers are always decayed, so we don't need to do
3983       // anything here.
3984       if (!BaseTy->isVariableArrayType()) {
3985         assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3986                "Expected pointer to array");
3987         Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3988       }
3989 
3990       return Addr.withElementType(CGF.ConvertTypeForMem(ElTy));
3991     }
3992     LValueBaseInfo TypeBaseInfo;
3993     TBAAAccessInfo TypeTBAAInfo;
3994     CharUnits Align =
3995         CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo);
3996     BaseInfo.mergeForCast(TypeBaseInfo);
3997     TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo);
3998     return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)),
3999                    CGF.ConvertTypeForMem(ElTy), Align);
4000   }
4001   return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
4002 }
4003 
4004 LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
4005                                                 bool IsLowerBound) {
4006   QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(E->getBase());
4007   QualType ResultExprTy;
4008   if (auto *AT = getContext().getAsArrayType(BaseTy))
4009     ResultExprTy = AT->getElementType();
4010   else
4011     ResultExprTy = BaseTy->getPointeeType();
4012   llvm::Value *Idx = nullptr;
4013   if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
4014     // Requesting lower bound or upper bound, but without provided length and
4015     // without ':' symbol for the default length -> length = 1.
4016     // Idx = LowerBound ?: 0;
4017     if (auto *LowerBound = E->getLowerBound()) {
4018       Idx = Builder.CreateIntCast(
4019           EmitScalarExpr(LowerBound), IntPtrTy,
4020           LowerBound->getType()->hasSignedIntegerRepresentation());
4021     } else
4022       Idx = llvm::ConstantInt::getNullValue(IntPtrTy);
4023   } else {
4024     // Try to emit length or lower bound as constant. If this is possible, 1
4025     // is subtracted from constant length or lower bound. Otherwise, emit LLVM
4026     // IR (LB + Len) - 1.
4027     auto &C = CGM.getContext();
4028     auto *Length = E->getLength();
4029     llvm::APSInt ConstLength;
4030     if (Length) {
4031       // Idx = LowerBound + Length - 1;
4032       if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) {
4033         ConstLength = CL->zextOrTrunc(PointerWidthInBits);
4034         Length = nullptr;
4035       }
4036       auto *LowerBound = E->getLowerBound();
4037       llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
4038       if (LowerBound) {
4039         if (std::optional<llvm::APSInt> LB =
4040                 LowerBound->getIntegerConstantExpr(C)) {
4041           ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits);
4042           LowerBound = nullptr;
4043         }
4044       }
4045       if (!Length)
4046         --ConstLength;
4047       else if (!LowerBound)
4048         --ConstLowerBound;
4049 
4050       if (Length || LowerBound) {
4051         auto *LowerBoundVal =
4052             LowerBound
4053                 ? Builder.CreateIntCast(
4054                       EmitScalarExpr(LowerBound), IntPtrTy,
4055                       LowerBound->getType()->hasSignedIntegerRepresentation())
4056                 : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound);
4057         auto *LengthVal =
4058             Length
4059                 ? Builder.CreateIntCast(
4060                       EmitScalarExpr(Length), IntPtrTy,
4061                       Length->getType()->hasSignedIntegerRepresentation())
4062                 : llvm::ConstantInt::get(IntPtrTy, ConstLength);
4063         Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",
4064                                 /*HasNUW=*/false,
4065                                 !getLangOpts().isSignedOverflowDefined());
4066         if (Length && LowerBound) {
4067           Idx = Builder.CreateSub(
4068               Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",
4069               /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4070         }
4071       } else
4072         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);
4073     } else {
4074       // Idx = ArraySize - 1;
4075       QualType ArrayTy = BaseTy->isPointerType()
4076                              ? E->getBase()->IgnoreParenImpCasts()->getType()
4077                              : BaseTy;
4078       if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) {
4079         Length = VAT->getSizeExpr();
4080         if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) {
4081           ConstLength = *L;
4082           Length = nullptr;
4083         }
4084       } else {
4085         auto *CAT = C.getAsConstantArrayType(ArrayTy);
4086         assert(CAT && "unexpected type for array initializer");
4087         ConstLength = CAT->getSize();
4088       }
4089       if (Length) {
4090         auto *LengthVal = Builder.CreateIntCast(
4091             EmitScalarExpr(Length), IntPtrTy,
4092             Length->getType()->hasSignedIntegerRepresentation());
4093         Idx = Builder.CreateSub(
4094             LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",
4095             /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4096       } else {
4097         ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
4098         --ConstLength;
4099         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength);
4100       }
4101     }
4102   }
4103   assert(Idx);
4104 
4105   Address EltPtr = Address::invalid();
4106   LValueBaseInfo BaseInfo;
4107   TBAAAccessInfo TBAAInfo;
4108   if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) {
4109     // The base must be a pointer, which is not an aggregate.  Emit
4110     // it.  It needs to be emitted first in case it's what captures
4111     // the VLA bounds.
4112     Address Base =
4113         emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo,
4114                                 BaseTy, VLA->getElementType(), IsLowerBound);
4115     // The element count here is the total number of non-VLA elements.
4116     llvm::Value *NumElements = getVLASize(VLA).NumElts;
4117 
4118     // Effectively, the multiply by the VLA size is part of the GEP.
4119     // GEP indexes are signed, and scaling an index isn't permitted to
4120     // signed-overflow, so we use the same semantics for our explicit
4121     // multiply.  We suppress this if overflow is not undefined behavior.
4122     if (getLangOpts().isSignedOverflowDefined())
4123       Idx = Builder.CreateMul(Idx, NumElements);
4124     else
4125       Idx = Builder.CreateNSWMul(Idx, NumElements);
4126     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),
4127                                    !getLangOpts().isSignedOverflowDefined(),
4128                                    /*signedIndices=*/false, E->getExprLoc());
4129   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
4130     // If this is A[i] where A is an array, the frontend will have decayed the
4131     // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
4132     // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4133     // "gep x, i" here.  Emit one "gep A, 0, i".
4134     assert(Array->getType()->isArrayType() &&
4135            "Array to pointer decay must have array source type!");
4136     LValue ArrayLV;
4137     // For simple multidimensional array indexing, set the 'accessed' flag for
4138     // better bounds-checking of the base expression.
4139     if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4140       ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
4141     else
4142       ArrayLV = EmitLValue(Array);
4143 
4144     // Propagate the alignment from the array itself to the result.
4145     EltPtr = emitArraySubscriptGEP(
4146         *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
4147         ResultExprTy, !getLangOpts().isSignedOverflowDefined(),
4148         /*signedIndices=*/false, E->getExprLoc());
4149     BaseInfo = ArrayLV.getBaseInfo();
4150     TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
4151   } else {
4152     Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo,
4153                                            TBAAInfo, BaseTy, ResultExprTy,
4154                                            IsLowerBound);
4155     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
4156                                    !getLangOpts().isSignedOverflowDefined(),
4157                                    /*signedIndices=*/false, E->getExprLoc());
4158   }
4159 
4160   return MakeAddrLValue(EltPtr, ResultExprTy, BaseInfo, TBAAInfo);
4161 }
4162 
4163 LValue CodeGenFunction::
4164 EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
4165   // Emit the base vector as an l-value.
4166   LValue Base;
4167 
4168   // ExtVectorElementExpr's base can either be a vector or pointer to vector.
4169   if (E->isArrow()) {
4170     // If it is a pointer to a vector, emit the address and form an lvalue with
4171     // it.
4172     LValueBaseInfo BaseInfo;
4173     TBAAAccessInfo TBAAInfo;
4174     Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo);
4175     const auto *PT = E->getBase()->getType()->castAs<PointerType>();
4176     Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo);
4177     Base.getQuals().removeObjCGCAttr();
4178   } else if (E->getBase()->isGLValue()) {
4179     // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
4180     // emit the base as an lvalue.
4181     assert(E->getBase()->getType()->isVectorType());
4182     Base = EmitLValue(E->getBase());
4183   } else {
4184     // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
4185     assert(E->getBase()->getType()->isVectorType() &&
4186            "Result must be a vector");
4187     llvm::Value *Vec = EmitScalarExpr(E->getBase());
4188 
4189     // Store the vector to memory (because LValue wants an address).
4190     Address VecMem = CreateMemTemp(E->getBase()->getType());
4191     Builder.CreateStore(Vec, VecMem);
4192     Base = MakeAddrLValue(VecMem, E->getBase()->getType(),
4193                           AlignmentSource::Decl);
4194   }
4195 
4196   QualType type =
4197     E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
4198 
4199   // Encode the element access list into a vector of unsigned indices.
4200   SmallVector<uint32_t, 4> Indices;
4201   E->getEncodedElementAccess(Indices);
4202 
4203   if (Base.isSimple()) {
4204     llvm::Constant *CV =
4205         llvm::ConstantDataVector::get(getLLVMContext(), Indices);
4206     return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type,
4207                                     Base.getBaseInfo(), TBAAAccessInfo());
4208   }
4209   assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
4210 
4211   llvm::Constant *BaseElts = Base.getExtVectorElts();
4212   SmallVector<llvm::Constant *, 4> CElts;
4213 
4214   for (unsigned i = 0, e = Indices.size(); i != e; ++i)
4215     CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
4216   llvm::Constant *CV = llvm::ConstantVector::get(CElts);
4217   return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,
4218                                   Base.getBaseInfo(), TBAAAccessInfo());
4219 }
4220 
4221 LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
4222   if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, E)) {
4223     EmitIgnoredExpr(E->getBase());
4224     return EmitDeclRefLValue(DRE);
4225   }
4226 
4227   Expr *BaseExpr = E->getBase();
4228   // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
4229   LValue BaseLV;
4230   if (E->isArrow()) {
4231     LValueBaseInfo BaseInfo;
4232     TBAAAccessInfo TBAAInfo;
4233     Address Addr = EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
4234     QualType PtrTy = BaseExpr->getType()->getPointeeType();
4235     SanitizerSet SkippedChecks;
4236     bool IsBaseCXXThis = IsWrappedCXXThis(BaseExpr);
4237     if (IsBaseCXXThis)
4238       SkippedChecks.set(SanitizerKind::Alignment, true);
4239     if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
4240       SkippedChecks.set(SanitizerKind::Null, true);
4241     EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy,
4242                   /*Alignment=*/CharUnits::Zero(), SkippedChecks);
4243     BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
4244   } else
4245     BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
4246 
4247   NamedDecl *ND = E->getMemberDecl();
4248   if (auto *Field = dyn_cast<FieldDecl>(ND)) {
4249     LValue LV = EmitLValueForField(BaseLV, Field);
4250     setObjCGCLValueClass(getContext(), E, LV);
4251     if (getLangOpts().OpenMP) {
4252       // If the member was explicitly marked as nontemporal, mark it as
4253       // nontemporal. If the base lvalue is marked as nontemporal, mark access
4254       // to children as nontemporal too.
4255       if ((IsWrappedCXXThis(BaseExpr) &&
4256            CGM.getOpenMPRuntime().isNontemporalDecl(Field)) ||
4257           BaseLV.isNontemporal())
4258         LV.setNontemporal(/*Value=*/true);
4259     }
4260     return LV;
4261   }
4262 
4263   if (const auto *FD = dyn_cast<FunctionDecl>(ND))
4264     return EmitFunctionDeclLValue(*this, E, FD);
4265 
4266   llvm_unreachable("Unhandled member declaration!");
4267 }
4268 
4269 /// Given that we are currently emitting a lambda, emit an l-value for
4270 /// one of its members.
4271 LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
4272   if (CurCodeDecl) {
4273     assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent()->isLambda());
4274     assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent() == Field->getParent());
4275   }
4276   QualType LambdaTagType =
4277     getContext().getTagDeclType(Field->getParent());
4278   LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue, LambdaTagType);
4279   return EmitLValueForField(LambdaLV, Field);
4280 }
4281 
4282 /// Get the field index in the debug info. The debug info structure/union
4283 /// will ignore the unnamed bitfields.
4284 unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
4285                                              unsigned FieldIndex) {
4286   unsigned I = 0, Skipped = 0;
4287 
4288   for (auto *F : Rec->getDefinition()->fields()) {
4289     if (I == FieldIndex)
4290       break;
4291     if (F->isUnnamedBitfield())
4292       Skipped++;
4293     I++;
4294   }
4295 
4296   return FieldIndex - Skipped;
4297 }
4298 
4299 /// Get the address of a zero-sized field within a record. The resulting
4300 /// address doesn't necessarily have the right type.
4301 static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
4302                                        const FieldDecl *Field) {
4303   CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
4304       CGF.getContext().getFieldOffset(Field));
4305   if (Offset.isZero())
4306     return Base;
4307   Base = Base.withElementType(CGF.Int8Ty);
4308   return CGF.Builder.CreateConstInBoundsByteGEP(Base, Offset);
4309 }
4310 
4311 /// Drill down to the storage of a field without walking into
4312 /// reference types.
4313 ///
4314 /// The resulting address doesn't necessarily have the right type.
4315 static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
4316                                       const FieldDecl *field) {
4317   if (field->isZeroSize(CGF.getContext()))
4318     return emitAddrOfZeroSizeField(CGF, base, field);
4319 
4320   const RecordDecl *rec = field->getParent();
4321 
4322   unsigned idx =
4323     CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4324 
4325   return CGF.Builder.CreateStructGEP(base, idx, field->getName());
4326 }
4327 
4328 static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
4329                                         Address addr, const FieldDecl *field) {
4330   const RecordDecl *rec = field->getParent();
4331   llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
4332       base.getType(), rec->getLocation());
4333 
4334   unsigned idx =
4335       CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4336 
4337   return CGF.Builder.CreatePreserveStructAccessIndex(
4338       addr, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo);
4339 }
4340 
4341 static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
4342   const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
4343   if (!RD)
4344     return false;
4345 
4346   if (RD->isDynamicClass())
4347     return true;
4348 
4349   for (const auto &Base : RD->bases())
4350     if (hasAnyVptr(Base.getType(), Context))
4351       return true;
4352 
4353   for (const FieldDecl *Field : RD->fields())
4354     if (hasAnyVptr(Field->getType(), Context))
4355       return true;
4356 
4357   return false;
4358 }
4359 
4360 LValue CodeGenFunction::EmitLValueForField(LValue base,
4361                                            const FieldDecl *field) {
4362   LValueBaseInfo BaseInfo = base.getBaseInfo();
4363 
4364   if (field->isBitField()) {
4365     const CGRecordLayout &RL =
4366         CGM.getTypes().getCGRecordLayout(field->getParent());
4367     const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
4368     const bool UseVolatile = isAAPCS(CGM.getTarget()) &&
4369                              CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
4370                              Info.VolatileStorageSize != 0 &&
4371                              field->getType()
4372                                  .withCVRQualifiers(base.getVRQualifiers())
4373                                  .isVolatileQualified();
4374     Address Addr = base.getAddress(*this);
4375     unsigned Idx = RL.getLLVMFieldNo(field);
4376     const RecordDecl *rec = field->getParent();
4377     if (!UseVolatile) {
4378       if (!IsInPreservedAIRegion &&
4379           (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4380         if (Idx != 0)
4381           // For structs, we GEP to the field that the record layout suggests.
4382           Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
4383       } else {
4384         llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
4385             getContext().getRecordType(rec), rec->getLocation());
4386         Addr = Builder.CreatePreserveStructAccessIndex(
4387             Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()),
4388             DbgInfo);
4389       }
4390     }
4391     const unsigned SS =
4392         UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
4393     // Get the access type.
4394     llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS);
4395     Addr = Addr.withElementType(FieldIntTy);
4396     if (UseVolatile) {
4397       const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
4398       if (VolatileOffset)
4399         Addr = Builder.CreateConstInBoundsGEP(Addr, VolatileOffset);
4400     }
4401 
4402     QualType fieldType =
4403         field->getType().withCVRQualifiers(base.getVRQualifiers());
4404     // TODO: Support TBAA for bit fields.
4405     LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
4406     return LValue::MakeBitfield(Addr, Info, fieldType, FieldBaseInfo,
4407                                 TBAAAccessInfo());
4408   }
4409 
4410   // Fields of may-alias structures are may-alias themselves.
4411   // FIXME: this should get propagated down through anonymous structs
4412   // and unions.
4413   QualType FieldType = field->getType();
4414   const RecordDecl *rec = field->getParent();
4415   AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
4416   LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));
4417   TBAAAccessInfo FieldTBAAInfo;
4418   if (base.getTBAAInfo().isMayAlias() ||
4419           rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
4420     FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4421   } else if (rec->isUnion()) {
4422     // TODO: Support TBAA for unions.
4423     FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4424   } else {
4425     // If no base type been assigned for the base access, then try to generate
4426     // one for this base lvalue.
4427     FieldTBAAInfo = base.getTBAAInfo();
4428     if (!FieldTBAAInfo.BaseType) {
4429         FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(base.getType());
4430         assert(!FieldTBAAInfo.Offset &&
4431                "Nonzero offset for an access with no base type!");
4432     }
4433 
4434     // Adjust offset to be relative to the base type.
4435     const ASTRecordLayout &Layout =
4436         getContext().getASTRecordLayout(field->getParent());
4437     unsigned CharWidth = getContext().getCharWidth();
4438     if (FieldTBAAInfo.BaseType)
4439       FieldTBAAInfo.Offset +=
4440           Layout.getFieldOffset(field->getFieldIndex()) / CharWidth;
4441 
4442     // Update the final access type and size.
4443     FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(FieldType);
4444     FieldTBAAInfo.Size =
4445         getContext().getTypeSizeInChars(FieldType).getQuantity();
4446   }
4447 
4448   Address addr = base.getAddress(*this);
4449   if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) {
4450     if (CGM.getCodeGenOpts().StrictVTablePointers &&
4451         ClassDef->isDynamicClass()) {
4452       // Getting to any field of dynamic object requires stripping dynamic
4453       // information provided by invariant.group.  This is because accessing
4454       // fields may leak the real address of dynamic object, which could result
4455       // in miscompilation when leaked pointer would be compared.
4456       auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
4457       addr = Address(stripped, addr.getElementType(), addr.getAlignment());
4458     }
4459   }
4460 
4461   unsigned RecordCVR = base.getVRQualifiers();
4462   if (rec->isUnion()) {
4463     // For unions, there is no pointer adjustment.
4464     if (CGM.getCodeGenOpts().StrictVTablePointers &&
4465         hasAnyVptr(FieldType, getContext()))
4466       // Because unions can easily skip invariant.barriers, we need to add
4467       // a barrier every time CXXRecord field with vptr is referenced.
4468       addr = Builder.CreateLaunderInvariantGroup(addr);
4469 
4470     if (IsInPreservedAIRegion ||
4471         (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4472       // Remember the original union field index
4473       llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),
4474           rec->getLocation());
4475       addr = Address(
4476           Builder.CreatePreserveUnionAccessIndex(
4477               addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
4478           addr.getElementType(), addr.getAlignment());
4479     }
4480 
4481     if (FieldType->isReferenceType())
4482       addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType));
4483   } else {
4484     if (!IsInPreservedAIRegion &&
4485         (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
4486       // For structs, we GEP to the field that the record layout suggests.
4487       addr = emitAddrOfFieldStorage(*this, addr, field);
4488     else
4489       // Remember the original struct field index
4490       addr = emitPreserveStructAccess(*this, base, addr, field);
4491   }
4492 
4493   // If this is a reference field, load the reference right now.
4494   if (FieldType->isReferenceType()) {
4495     LValue RefLVal =
4496         MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4497     if (RecordCVR & Qualifiers::Volatile)
4498       RefLVal.getQuals().addVolatile();
4499     addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo);
4500 
4501     // Qualifiers on the struct don't apply to the referencee.
4502     RecordCVR = 0;
4503     FieldType = FieldType->getPointeeType();
4504   }
4505 
4506   // Make sure that the address is pointing to the right type.  This is critical
4507   // for both unions and structs.
4508   addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType));
4509 
4510   if (field->hasAttr<AnnotateAttr>())
4511     addr = EmitFieldAnnotations(field, addr);
4512 
4513   LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4514   LV.getQuals().addCVRQualifiers(RecordCVR);
4515 
4516   // __weak attribute on a field is ignored.
4517   if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
4518     LV.getQuals().removeObjCGCAttr();
4519 
4520   return LV;
4521 }
4522 
4523 LValue
4524 CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
4525                                                   const FieldDecl *Field) {
4526   QualType FieldType = Field->getType();
4527 
4528   if (!FieldType->isReferenceType())
4529     return EmitLValueForField(Base, Field);
4530 
4531   Address V = emitAddrOfFieldStorage(*this, Base.getAddress(*this), Field);
4532 
4533   // Make sure that the address is pointing to the right type.
4534   llvm::Type *llvmType = ConvertTypeForMem(FieldType);
4535   V = V.withElementType(llvmType);
4536 
4537   // TODO: Generate TBAA information that describes this access as a structure
4538   // member access and not just an access to an object of the field's type. This
4539   // should be similar to what we do in EmitLValueForField().
4540   LValueBaseInfo BaseInfo = Base.getBaseInfo();
4541   AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
4542   LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(FieldAlignSource));
4543   return MakeAddrLValue(V, FieldType, FieldBaseInfo,
4544                         CGM.getTBAAInfoForSubobject(Base, FieldType));
4545 }
4546 
4547 LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
4548   if (E->isFileScope()) {
4549     ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
4550     return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl);
4551   }
4552   if (E->getType()->isVariablyModifiedType())
4553     // make sure to emit the VLA size.
4554     EmitVariablyModifiedType(E->getType());
4555 
4556   Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
4557   const Expr *InitExpr = E->getInitializer();
4558   LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl);
4559 
4560   EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
4561                    /*Init*/ true);
4562 
4563   // Block-scope compound literals are destroyed at the end of the enclosing
4564   // scope in C.
4565   if (!getLangOpts().CPlusPlus)
4566     if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
4567       pushLifetimeExtendedDestroy(getCleanupKind(DtorKind), DeclPtr,
4568                                   E->getType(), getDestroyer(DtorKind),
4569                                   DtorKind & EHCleanup);
4570 
4571   return Result;
4572 }
4573 
4574 LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
4575   if (!E->isGLValue())
4576     // Initializing an aggregate temporary in C++11: T{...}.
4577     return EmitAggExprToLValue(E);
4578 
4579   // An lvalue initializer list must be initializing a reference.
4580   assert(E->isTransparent() && "non-transparent glvalue init list");
4581   return EmitLValue(E->getInit(0));
4582 }
4583 
4584 /// Emit the operand of a glvalue conditional operator. This is either a glvalue
4585 /// or a (possibly-parenthesized) throw-expression. If this is a throw, no
4586 /// LValue is returned and the current block has been terminated.
4587 static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
4588                                                          const Expr *Operand) {
4589   if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Operand->IgnoreParens())) {
4590     CGF.EmitCXXThrowExpr(ThrowExpr, /*KeepInsertionPoint*/false);
4591     return std::nullopt;
4592   }
4593 
4594   return CGF.EmitLValue(Operand);
4595 }
4596 
4597 namespace {
4598 // Handle the case where the condition is a constant evaluatable simple integer,
4599 // which means we don't have to separately handle the true/false blocks.
4600 std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
4601     CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
4602   const Expr *condExpr = E->getCond();
4603   bool CondExprBool;
4604   if (CGF.ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
4605     const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
4606     if (!CondExprBool)
4607       std::swap(Live, Dead);
4608 
4609     if (!CGF.ContainsLabel(Dead)) {
4610       // If the true case is live, we need to track its region.
4611       if (CondExprBool)
4612         CGF.incrementProfileCounter(E);
4613       // If a throw expression we emit it and return an undefined lvalue
4614       // because it can't be used.
4615       if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Live->IgnoreParens())) {
4616         CGF.EmitCXXThrowExpr(ThrowExpr);
4617         llvm::Type *ElemTy = CGF.ConvertType(Dead->getType());
4618         llvm::Type *Ty = llvm::PointerType::getUnqual(ElemTy);
4619         return CGF.MakeAddrLValue(
4620             Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()),
4621             Dead->getType());
4622       }
4623       return CGF.EmitLValue(Live);
4624     }
4625   }
4626   return std::nullopt;
4627 }
4628 struct ConditionalInfo {
4629   llvm::BasicBlock *lhsBlock, *rhsBlock;
4630   std::optional<LValue> LHS, RHS;
4631 };
4632 
4633 // Create and generate the 3 blocks for a conditional operator.
4634 // Leaves the 'current block' in the continuation basic block.
4635 template<typename FuncTy>
4636 ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
4637                                       const AbstractConditionalOperator *E,
4638                                       const FuncTy &BranchGenFunc) {
4639   ConditionalInfo Info{CGF.createBasicBlock("cond.true"),
4640                        CGF.createBasicBlock("cond.false"), std::nullopt,
4641                        std::nullopt};
4642   llvm::BasicBlock *endBlock = CGF.createBasicBlock("cond.end");
4643 
4644   CodeGenFunction::ConditionalEvaluation eval(CGF);
4645   CGF.EmitBranchOnBoolExpr(E->getCond(), Info.lhsBlock, Info.rhsBlock,
4646                            CGF.getProfileCount(E));
4647 
4648   // Any temporaries created here are conditional.
4649   CGF.EmitBlock(Info.lhsBlock);
4650   CGF.incrementProfileCounter(E);
4651   eval.begin(CGF);
4652   Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
4653   eval.end(CGF);
4654   Info.lhsBlock = CGF.Builder.GetInsertBlock();
4655 
4656   if (Info.LHS)
4657     CGF.Builder.CreateBr(endBlock);
4658 
4659   // Any temporaries created here are conditional.
4660   CGF.EmitBlock(Info.rhsBlock);
4661   eval.begin(CGF);
4662   Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
4663   eval.end(CGF);
4664   Info.rhsBlock = CGF.Builder.GetInsertBlock();
4665   CGF.EmitBlock(endBlock);
4666 
4667   return Info;
4668 }
4669 } // namespace
4670 
4671 void CodeGenFunction::EmitIgnoredConditionalOperator(
4672     const AbstractConditionalOperator *E) {
4673   if (!E->isGLValue()) {
4674     // ?: here should be an aggregate.
4675     assert(hasAggregateEvaluationKind(E->getType()) &&
4676            "Unexpected conditional operator!");
4677     return (void)EmitAggExprToLValue(E);
4678   }
4679 
4680   OpaqueValueMapping binding(*this, E);
4681   if (HandleConditionalOperatorLValueSimpleCase(*this, E))
4682     return;
4683 
4684   EmitConditionalBlocks(*this, E, [](CodeGenFunction &CGF, const Expr *E) {
4685     CGF.EmitIgnoredExpr(E);
4686     return LValue{};
4687   });
4688 }
4689 LValue CodeGenFunction::EmitConditionalOperatorLValue(
4690     const AbstractConditionalOperator *expr) {
4691   if (!expr->isGLValue()) {
4692     // ?: here should be an aggregate.
4693     assert(hasAggregateEvaluationKind(expr->getType()) &&
4694            "Unexpected conditional operator!");
4695     return EmitAggExprToLValue(expr);
4696   }
4697 
4698   OpaqueValueMapping binding(*this, expr);
4699   if (std::optional<LValue> Res =
4700           HandleConditionalOperatorLValueSimpleCase(*this, expr))
4701     return *Res;
4702 
4703   ConditionalInfo Info = EmitConditionalBlocks(
4704       *this, expr, [](CodeGenFunction &CGF, const Expr *E) {
4705         return EmitLValueOrThrowExpression(CGF, E);
4706       });
4707 
4708   if ((Info.LHS && !Info.LHS->isSimple()) ||
4709       (Info.RHS && !Info.RHS->isSimple()))
4710     return EmitUnsupportedLValue(expr, "conditional operator");
4711 
4712   if (Info.LHS && Info.RHS) {
4713     Address lhsAddr = Info.LHS->getAddress(*this);
4714     Address rhsAddr = Info.RHS->getAddress(*this);
4715     llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue");
4716     phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock);
4717     phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock);
4718     Address result(phi, lhsAddr.getElementType(),
4719                    std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
4720     AlignmentSource alignSource =
4721         std::max(Info.LHS->getBaseInfo().getAlignmentSource(),
4722                  Info.RHS->getBaseInfo().getAlignmentSource());
4723     TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
4724         Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo());
4725     return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),
4726                           TBAAInfo);
4727   } else {
4728     assert((Info.LHS || Info.RHS) &&
4729            "both operands of glvalue conditional are throw-expressions?");
4730     return Info.LHS ? *Info.LHS : *Info.RHS;
4731   }
4732 }
4733 
4734 /// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
4735 /// type. If the cast is to a reference, we can have the usual lvalue result,
4736 /// otherwise if a cast is needed by the code generator in an lvalue context,
4737 /// then it must mean that we need the address of an aggregate in order to
4738 /// access one of its members.  This can happen for all the reasons that casts
4739 /// are permitted with aggregate result, including noop aggregate casts, and
4740 /// cast from scalar to union.
4741 LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
4742   switch (E->getCastKind()) {
4743   case CK_ToVoid:
4744   case CK_BitCast:
4745   case CK_LValueToRValueBitCast:
4746   case CK_ArrayToPointerDecay:
4747   case CK_FunctionToPointerDecay:
4748   case CK_NullToMemberPointer:
4749   case CK_NullToPointer:
4750   case CK_IntegralToPointer:
4751   case CK_PointerToIntegral:
4752   case CK_PointerToBoolean:
4753   case CK_VectorSplat:
4754   case CK_IntegralCast:
4755   case CK_BooleanToSignedIntegral:
4756   case CK_IntegralToBoolean:
4757   case CK_IntegralToFloating:
4758   case CK_FloatingToIntegral:
4759   case CK_FloatingToBoolean:
4760   case CK_FloatingCast:
4761   case CK_FloatingRealToComplex:
4762   case CK_FloatingComplexToReal:
4763   case CK_FloatingComplexToBoolean:
4764   case CK_FloatingComplexCast:
4765   case CK_FloatingComplexToIntegralComplex:
4766   case CK_IntegralRealToComplex:
4767   case CK_IntegralComplexToReal:
4768   case CK_IntegralComplexToBoolean:
4769   case CK_IntegralComplexCast:
4770   case CK_IntegralComplexToFloatingComplex:
4771   case CK_DerivedToBaseMemberPointer:
4772   case CK_BaseToDerivedMemberPointer:
4773   case CK_MemberPointerToBoolean:
4774   case CK_ReinterpretMemberPointer:
4775   case CK_AnyPointerToBlockPointerCast:
4776   case CK_ARCProduceObject:
4777   case CK_ARCConsumeObject:
4778   case CK_ARCReclaimReturnedObject:
4779   case CK_ARCExtendBlockObject:
4780   case CK_CopyAndAutoreleaseBlockObject:
4781   case CK_IntToOCLSampler:
4782   case CK_FloatingToFixedPoint:
4783   case CK_FixedPointToFloating:
4784   case CK_FixedPointCast:
4785   case CK_FixedPointToBoolean:
4786   case CK_FixedPointToIntegral:
4787   case CK_IntegralToFixedPoint:
4788   case CK_MatrixCast:
4789     return EmitUnsupportedLValue(E, "unexpected cast lvalue");
4790 
4791   case CK_Dependent:
4792     llvm_unreachable("dependent cast kind in IR gen!");
4793 
4794   case CK_BuiltinFnToFnPtr:
4795     llvm_unreachable("builtin functions are handled elsewhere");
4796 
4797   // These are never l-values; just use the aggregate emission code.
4798   case CK_NonAtomicToAtomic:
4799   case CK_AtomicToNonAtomic:
4800     return EmitAggExprToLValue(E);
4801 
4802   case CK_Dynamic: {
4803     LValue LV = EmitLValue(E->getSubExpr());
4804     Address V = LV.getAddress(*this);
4805     const auto *DCE = cast<CXXDynamicCastExpr>(E);
4806     return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType());
4807   }
4808 
4809   case CK_ConstructorConversion:
4810   case CK_UserDefinedConversion:
4811   case CK_CPointerToObjCPointerCast:
4812   case CK_BlockPointerToObjCPointerCast:
4813   case CK_LValueToRValue:
4814     return EmitLValue(E->getSubExpr());
4815 
4816   case CK_NoOp: {
4817     // CK_NoOp can model a qualification conversion, which can remove an array
4818     // bound and change the IR type.
4819     // FIXME: Once pointee types are removed from IR, remove this.
4820     LValue LV = EmitLValue(E->getSubExpr());
4821     if (LV.isSimple()) {
4822       Address V = LV.getAddress(*this);
4823       if (V.isValid()) {
4824         llvm::Type *T = ConvertTypeForMem(E->getType());
4825         if (V.getElementType() != T)
4826           LV.setAddress(V.withElementType(T));
4827       }
4828     }
4829     return LV;
4830   }
4831 
4832   case CK_UncheckedDerivedToBase:
4833   case CK_DerivedToBase: {
4834     const auto *DerivedClassTy =
4835         E->getSubExpr()->getType()->castAs<RecordType>();
4836     auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
4837 
4838     LValue LV = EmitLValue(E->getSubExpr());
4839     Address This = LV.getAddress(*this);
4840 
4841     // Perform the derived-to-base conversion
4842     Address Base = GetAddressOfBaseClass(
4843         This, DerivedClassDecl, E->path_begin(), E->path_end(),
4844         /*NullCheckValue=*/false, E->getExprLoc());
4845 
4846     // TODO: Support accesses to members of base classes in TBAA. For now, we
4847     // conservatively pretend that the complete object is of the base class
4848     // type.
4849     return MakeAddrLValue(Base, E->getType(), LV.getBaseInfo(),
4850                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4851   }
4852   case CK_ToUnion:
4853     return EmitAggExprToLValue(E);
4854   case CK_BaseToDerived: {
4855     const auto *DerivedClassTy = E->getType()->castAs<RecordType>();
4856     auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
4857 
4858     LValue LV = EmitLValue(E->getSubExpr());
4859 
4860     // Perform the base-to-derived conversion
4861     Address Derived = GetAddressOfDerivedClass(
4862         LV.getAddress(*this), DerivedClassDecl, E->path_begin(), E->path_end(),
4863         /*NullCheckValue=*/false);
4864 
4865     // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
4866     // performed and the object is not of the derived type.
4867     if (sanitizePerformTypeCheck())
4868       EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(),
4869                     Derived.getPointer(), E->getType());
4870 
4871     if (SanOpts.has(SanitizerKind::CFIDerivedCast))
4872       EmitVTablePtrCheckForCast(E->getType(), Derived,
4873                                 /*MayBeNull=*/false, CFITCK_DerivedCast,
4874                                 E->getBeginLoc());
4875 
4876     return MakeAddrLValue(Derived, E->getType(), LV.getBaseInfo(),
4877                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4878   }
4879   case CK_LValueBitCast: {
4880     // This must be a reinterpret_cast (or c-style equivalent).
4881     const auto *CE = cast<ExplicitCastExpr>(E);
4882 
4883     CGM.EmitExplicitCastExprType(CE, this);
4884     LValue LV = EmitLValue(E->getSubExpr());
4885     Address V = LV.getAddress(*this).withElementType(
4886         ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType()));
4887 
4888     if (SanOpts.has(SanitizerKind::CFIUnrelatedCast))
4889       EmitVTablePtrCheckForCast(E->getType(), V,
4890                                 /*MayBeNull=*/false, CFITCK_UnrelatedCast,
4891                                 E->getBeginLoc());
4892 
4893     return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
4894                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4895   }
4896   case CK_AddressSpaceConversion: {
4897     LValue LV = EmitLValue(E->getSubExpr());
4898     QualType DestTy = getContext().getPointerType(E->getType());
4899     llvm::Value *V = getTargetHooks().performAddrSpaceCast(
4900         *this, LV.getPointer(*this),
4901         E->getSubExpr()->getType().getAddressSpace(),
4902         E->getType().getAddressSpace(), ConvertType(DestTy));
4903     return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()),
4904                                   LV.getAddress(*this).getAlignment()),
4905                           E->getType(), LV.getBaseInfo(), LV.getTBAAInfo());
4906   }
4907   case CK_ObjCObjectLValueCast: {
4908     LValue LV = EmitLValue(E->getSubExpr());
4909     Address V = LV.getAddress(*this).withElementType(ConvertType(E->getType()));
4910     return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
4911                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4912   }
4913   case CK_ZeroToOCLOpaqueType:
4914     llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
4915   }
4916 
4917   llvm_unreachable("Unhandled lvalue cast kind?");
4918 }
4919 
4920 LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
4921   assert(OpaqueValueMappingData::shouldBindAsLValue(e));
4922   return getOrCreateOpaqueLValueMapping(e);
4923 }
4924 
4925 LValue
4926 CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
4927   assert(OpaqueValueMapping::shouldBindAsLValue(e));
4928 
4929   llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
4930       it = OpaqueLValues.find(e);
4931 
4932   if (it != OpaqueLValues.end())
4933     return it->second;
4934 
4935   assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
4936   return EmitLValue(e->getSourceExpr());
4937 }
4938 
4939 RValue
4940 CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
4941   assert(!OpaqueValueMapping::shouldBindAsLValue(e));
4942 
4943   llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
4944       it = OpaqueRValues.find(e);
4945 
4946   if (it != OpaqueRValues.end())
4947     return it->second;
4948 
4949   assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
4950   return EmitAnyExpr(e->getSourceExpr());
4951 }
4952 
4953 RValue CodeGenFunction::EmitRValueForField(LValue LV,
4954                                            const FieldDecl *FD,
4955                                            SourceLocation Loc) {
4956   QualType FT = FD->getType();
4957   LValue FieldLV = EmitLValueForField(LV, FD);
4958   switch (getEvaluationKind(FT)) {
4959   case TEK_Complex:
4960     return RValue::getComplex(EmitLoadOfComplex(FieldLV, Loc));
4961   case TEK_Aggregate:
4962     return FieldLV.asAggregateRValue(*this);
4963   case TEK_Scalar:
4964     // This routine is used to load fields one-by-one to perform a copy, so
4965     // don't load reference fields.
4966     if (FD->getType()->isReferenceType())
4967       return RValue::get(FieldLV.getPointer(*this));
4968     // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
4969     // primitive load.
4970     if (FieldLV.isBitField())
4971       return EmitLoadOfLValue(FieldLV, Loc);
4972     return RValue::get(EmitLoadOfScalar(FieldLV, Loc));
4973   }
4974   llvm_unreachable("bad evaluation kind");
4975 }
4976 
4977 //===--------------------------------------------------------------------===//
4978 //                             Expression Emission
4979 //===--------------------------------------------------------------------===//
4980 
4981 RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
4982                                      ReturnValueSlot ReturnValue) {
4983   // Builtins never have block type.
4984   if (E->getCallee()->getType()->isBlockPointerType())
4985     return EmitBlockCallExpr(E, ReturnValue);
4986 
4987   if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E))
4988     return EmitCXXMemberCallExpr(CE, ReturnValue);
4989 
4990   if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E))
4991     return EmitCUDAKernelCallExpr(CE, ReturnValue);
4992 
4993   if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E))
4994     if (const CXXMethodDecl *MD =
4995           dyn_cast_or_null<CXXMethodDecl>(CE->getCalleeDecl()))
4996       return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
4997 
4998   CGCallee callee = EmitCallee(E->getCallee());
4999 
5000   if (callee.isBuiltin()) {
5001     return EmitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(),
5002                            E, ReturnValue);
5003   }
5004 
5005   if (callee.isPseudoDestructor()) {
5006     return EmitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr());
5007   }
5008 
5009   return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue);
5010 }
5011 
5012 /// Emit a CallExpr without considering whether it might be a subclass.
5013 RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
5014                                            ReturnValueSlot ReturnValue) {
5015   CGCallee Callee = EmitCallee(E->getCallee());
5016   return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue);
5017 }
5018 
5019 // Detect the unusual situation where an inline version is shadowed by a
5020 // non-inline version. In that case we should pick the external one
5021 // everywhere. That's GCC behavior too.
5022 static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
5023   for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
5024     if (!PD->isInlineBuiltinDeclaration())
5025       return false;
5026   return true;
5027 }
5028 
5029 static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
5030   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
5031 
5032   if (auto builtinID = FD->getBuiltinID()) {
5033     std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
5034     std::string NoBuiltins = "no-builtins";
5035 
5036     StringRef Ident = CGF.CGM.getMangledName(GD);
5037     std::string FDInlineName = (Ident + ".inline").str();
5038 
5039     bool IsPredefinedLibFunction =
5040         CGF.getContext().BuiltinInfo.isPredefinedLibFunction(builtinID);
5041     bool HasAttributeNoBuiltin =
5042         CGF.CurFn->getAttributes().hasFnAttr(NoBuiltinFD) ||
5043         CGF.CurFn->getAttributes().hasFnAttr(NoBuiltins);
5044 
5045     // When directing calling an inline builtin, call it through it's mangled
5046     // name to make it clear it's not the actual builtin.
5047     if (CGF.CurFn->getName() != FDInlineName &&
5048         OnlyHasInlineBuiltinDeclaration(FD)) {
5049       llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
5050       llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr);
5051       llvm::Module *M = Fn->getParent();
5052       llvm::Function *Clone = M->getFunction(FDInlineName);
5053       if (!Clone) {
5054         Clone = llvm::Function::Create(Fn->getFunctionType(),
5055                                        llvm::GlobalValue::InternalLinkage,
5056                                        Fn->getAddressSpace(), FDInlineName, M);
5057         Clone->addFnAttr(llvm::Attribute::AlwaysInline);
5058       }
5059       return CGCallee::forDirect(Clone, GD);
5060     }
5061 
5062     // Replaceable builtins provide their own implementation of a builtin. If we
5063     // are in an inline builtin implementation, avoid trivial infinite
5064     // recursion. Honor __attribute__((no_builtin("foo"))) or
5065     // __attribute__((no_builtin)) on the current function unless foo is
5066     // not a predefined library function which means we must generate the
5067     // builtin no matter what.
5068     else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
5069       return CGCallee::forBuiltin(builtinID, FD);
5070   }
5071 
5072   llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
5073   if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
5074       FD->hasAttr<CUDAGlobalAttr>())
5075     CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
5076         cast<llvm::GlobalValue>(CalleePtr->stripPointerCasts()));
5077 
5078   return CGCallee::forDirect(CalleePtr, GD);
5079 }
5080 
5081 CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
5082   E = E->IgnoreParens();
5083 
5084   // Look through function-to-pointer decay.
5085   if (auto ICE = dyn_cast<ImplicitCastExpr>(E)) {
5086     if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
5087         ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
5088       return EmitCallee(ICE->getSubExpr());
5089     }
5090 
5091   // Resolve direct calls.
5092   } else if (auto DRE = dyn_cast<DeclRefExpr>(E)) {
5093     if (auto FD = dyn_cast<FunctionDecl>(DRE->getDecl())) {
5094       return EmitDirectCallee(*this, FD);
5095     }
5096   } else if (auto ME = dyn_cast<MemberExpr>(E)) {
5097     if (auto FD = dyn_cast<FunctionDecl>(ME->getMemberDecl())) {
5098       EmitIgnoredExpr(ME->getBase());
5099       return EmitDirectCallee(*this, FD);
5100     }
5101 
5102   // Look through template substitutions.
5103   } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {
5104     return EmitCallee(NTTP->getReplacement());
5105 
5106   // Treat pseudo-destructor calls differently.
5107   } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(E)) {
5108     return CGCallee::forPseudoDestructor(PDE);
5109   }
5110 
5111   // Otherwise, we have an indirect reference.
5112   llvm::Value *calleePtr;
5113   QualType functionType;
5114   if (auto ptrType = E->getType()->getAs<PointerType>()) {
5115     calleePtr = EmitScalarExpr(E);
5116     functionType = ptrType->getPointeeType();
5117   } else {
5118     functionType = E->getType();
5119     calleePtr = EmitLValue(E, KnownNonNull).getPointer(*this);
5120   }
5121   assert(functionType->isFunctionType());
5122 
5123   GlobalDecl GD;
5124   if (const auto *VD =
5125           dyn_cast_or_null<VarDecl>(E->getReferencedDeclOfCallee()))
5126     GD = GlobalDecl(VD);
5127 
5128   CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD);
5129   CGCallee callee(calleeInfo, calleePtr);
5130   return callee;
5131 }
5132 
5133 LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
5134   // Comma expressions just emit their LHS then their RHS as an l-value.
5135   if (E->getOpcode() == BO_Comma) {
5136     EmitIgnoredExpr(E->getLHS());
5137     EnsureInsertPoint();
5138     return EmitLValue(E->getRHS());
5139   }
5140 
5141   if (E->getOpcode() == BO_PtrMemD ||
5142       E->getOpcode() == BO_PtrMemI)
5143     return EmitPointerToDataMemberBinaryExpr(E);
5144 
5145   assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
5146 
5147   // Note that in all of these cases, __block variables need the RHS
5148   // evaluated first just in case the variable gets moved by the RHS.
5149 
5150   switch (getEvaluationKind(E->getType())) {
5151   case TEK_Scalar: {
5152     switch (E->getLHS()->getType().getObjCLifetime()) {
5153     case Qualifiers::OCL_Strong:
5154       return EmitARCStoreStrong(E, /*ignored*/ false).first;
5155 
5156     case Qualifiers::OCL_Autoreleasing:
5157       return EmitARCStoreAutoreleasing(E).first;
5158 
5159     // No reason to do any of these differently.
5160     case Qualifiers::OCL_None:
5161     case Qualifiers::OCL_ExplicitNone:
5162     case Qualifiers::OCL_Weak:
5163       break;
5164     }
5165 
5166     RValue RV = EmitAnyExpr(E->getRHS());
5167     LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);
5168     if (RV.isScalar())
5169       EmitNullabilityCheck(LV, RV.getScalarVal(), E->getExprLoc());
5170     EmitStoreThroughLValue(RV, LV);
5171     if (getLangOpts().OpenMP)
5172       CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
5173                                                                 E->getLHS());
5174     return LV;
5175   }
5176 
5177   case TEK_Complex:
5178     return EmitComplexAssignmentLValue(E);
5179 
5180   case TEK_Aggregate:
5181     return EmitAggExprToLValue(E);
5182   }
5183   llvm_unreachable("bad evaluation kind");
5184 }
5185 
5186 LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
5187   RValue RV = EmitCallExpr(E);
5188 
5189   if (!RV.isScalar())
5190     return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5191                           AlignmentSource::Decl);
5192 
5193   assert(E->getCallReturnType(getContext())->isReferenceType() &&
5194          "Can't have a scalar return unless the return type is a "
5195          "reference type!");
5196 
5197   return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
5198 }
5199 
5200 LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
5201   // FIXME: This shouldn't require another copy.
5202   return EmitAggExprToLValue(E);
5203 }
5204 
5205 LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
5206   assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
5207          && "binding l-value to type which needs a temporary");
5208   AggValueSlot Slot = CreateAggTemp(E->getType());
5209   EmitCXXConstructExpr(E, Slot);
5210   return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5211 }
5212 
5213 LValue
5214 CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
5215   return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType());
5216 }
5217 
5218 Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
5219   return CGM.GetAddrOfMSGuidDecl(E->getGuidDecl())
5220       .withElementType(ConvertType(E->getType()));
5221 }
5222 
5223 LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
5224   return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(),
5225                         AlignmentSource::Decl);
5226 }
5227 
5228 LValue
5229 CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
5230   AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
5231   Slot.setExternallyDestructed();
5232   EmitAggExpr(E->getSubExpr(), Slot);
5233   EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddress());
5234   return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5235 }
5236 
5237 LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
5238   RValue RV = EmitObjCMessageExpr(E);
5239 
5240   if (!RV.isScalar())
5241     return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5242                           AlignmentSource::Decl);
5243 
5244   assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
5245          "Can't have a scalar return unless the return type is a "
5246          "reference type!");
5247 
5248   return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
5249 }
5250 
5251 LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
5252   Address V =
5253     CGM.getObjCRuntime().GetAddrOfSelector(*this, E->getSelector());
5254   return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl);
5255 }
5256 
5257 llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
5258                                              const ObjCIvarDecl *Ivar) {
5259   return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);
5260 }
5261 
5262 llvm::Value *
5263 CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
5264                                              const ObjCIvarDecl *Ivar) {
5265   llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
5266   QualType PointerDiffType = getContext().getPointerDiffType();
5267   return Builder.CreateZExtOrTrunc(OffsetValue,
5268                                    getTypes().ConvertType(PointerDiffType));
5269 }
5270 
5271 LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
5272                                           llvm::Value *BaseValue,
5273                                           const ObjCIvarDecl *Ivar,
5274                                           unsigned CVRQualifiers) {
5275   return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,
5276                                                    Ivar, CVRQualifiers);
5277 }
5278 
5279 LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
5280   // FIXME: A lot of the code below could be shared with EmitMemberExpr.
5281   llvm::Value *BaseValue = nullptr;
5282   const Expr *BaseExpr = E->getBase();
5283   Qualifiers BaseQuals;
5284   QualType ObjectTy;
5285   if (E->isArrow()) {
5286     BaseValue = EmitScalarExpr(BaseExpr);
5287     ObjectTy = BaseExpr->getType()->getPointeeType();
5288     BaseQuals = ObjectTy.getQualifiers();
5289   } else {
5290     LValue BaseLV = EmitLValue(BaseExpr);
5291     BaseValue = BaseLV.getPointer(*this);
5292     ObjectTy = BaseExpr->getType();
5293     BaseQuals = ObjectTy.getQualifiers();
5294   }
5295 
5296   LValue LV =
5297     EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),
5298                       BaseQuals.getCVRQualifiers());
5299   setObjCGCLValueClass(getContext(), E, LV);
5300   return LV;
5301 }
5302 
5303 LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
5304   // Can only get l-value for message expression returning aggregate type
5305   RValue RV = EmitAnyExprToTemp(E);
5306   return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5307                         AlignmentSource::Decl);
5308 }
5309 
5310 RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee,
5311                                  const CallExpr *E, ReturnValueSlot ReturnValue,
5312                                  llvm::Value *Chain) {
5313   // Get the actual function type. The callee type will always be a pointer to
5314   // function type or a block pointer type.
5315   assert(CalleeType->isFunctionPointerType() &&
5316          "Call must have function pointer type!");
5317 
5318   const Decl *TargetDecl =
5319       OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
5320 
5321   assert((!isa_and_present<FunctionDecl>(TargetDecl) ||
5322           !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&
5323          "trying to emit a call to an immediate function");
5324 
5325   CalleeType = getContext().getCanonicalType(CalleeType);
5326 
5327   auto PointeeType = cast<PointerType>(CalleeType)->getPointeeType();
5328 
5329   CGCallee Callee = OrigCallee;
5330 
5331   if (SanOpts.has(SanitizerKind::Function) &&
5332       (!TargetDecl || !isa<FunctionDecl>(TargetDecl)) &&
5333       !isa<FunctionNoProtoType>(PointeeType)) {
5334     if (llvm::Constant *PrefixSig =
5335             CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
5336       SanitizerScope SanScope(this);
5337       auto *TypeHash = getUBSanFunctionTypeHash(PointeeType);
5338 
5339       llvm::Type *PrefixSigType = PrefixSig->getType();
5340       llvm::StructType *PrefixStructTy = llvm::StructType::get(
5341           CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true);
5342 
5343       llvm::Value *CalleePtr = Callee.getFunctionPointer();
5344 
5345       // On 32-bit Arm, the low bit of a function pointer indicates whether
5346       // it's using the Arm or Thumb instruction set. The actual first
5347       // instruction lives at the same address either way, so we must clear
5348       // that low bit before using the function address to find the prefix
5349       // structure.
5350       //
5351       // This applies to both Arm and Thumb target triples, because
5352       // either one could be used in an interworking context where it
5353       // might be passed function pointers of both types.
5354       llvm::Value *AlignedCalleePtr;
5355       if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {
5356         llvm::Value *CalleeAddress =
5357             Builder.CreatePtrToInt(CalleePtr, IntPtrTy);
5358         llvm::Value *Mask = llvm::ConstantInt::get(IntPtrTy, ~1);
5359         llvm::Value *AlignedCalleeAddress =
5360             Builder.CreateAnd(CalleeAddress, Mask);
5361         AlignedCalleePtr =
5362             Builder.CreateIntToPtr(AlignedCalleeAddress, CalleePtr->getType());
5363       } else {
5364         AlignedCalleePtr = CalleePtr;
5365       }
5366 
5367       llvm::Value *CalleePrefixStruct = Builder.CreateBitCast(
5368           AlignedCalleePtr, llvm::PointerType::getUnqual(PrefixStructTy));
5369       llvm::Value *CalleeSigPtr =
5370           Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, -1, 0);
5371       llvm::Value *CalleeSig =
5372           Builder.CreateAlignedLoad(PrefixSigType, CalleeSigPtr, getIntAlign());
5373       llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig);
5374 
5375       llvm::BasicBlock *Cont = createBasicBlock("cont");
5376       llvm::BasicBlock *TypeCheck = createBasicBlock("typecheck");
5377       Builder.CreateCondBr(CalleeSigMatch, TypeCheck, Cont);
5378 
5379       EmitBlock(TypeCheck);
5380       llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(
5381           Int32Ty,
5382           Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, -1, 1),
5383           getPointerAlign());
5384       llvm::Value *CalleeTypeHashMatch =
5385           Builder.CreateICmpEQ(CalleeTypeHash, TypeHash);
5386       llvm::Constant *StaticData[] = {EmitCheckSourceLocation(E->getBeginLoc()),
5387                                       EmitCheckTypeDescriptor(CalleeType)};
5388       EmitCheck(std::make_pair(CalleeTypeHashMatch, SanitizerKind::Function),
5389                 SanitizerHandler::FunctionTypeMismatch, StaticData,
5390                 {CalleePtr});
5391 
5392       Builder.CreateBr(Cont);
5393       EmitBlock(Cont);
5394     }
5395   }
5396 
5397   const auto *FnType = cast<FunctionType>(PointeeType);
5398 
5399   // If we are checking indirect calls and this call is indirect, check that the
5400   // function pointer is a member of the bit set for the function type.
5401   if (SanOpts.has(SanitizerKind::CFIICall) &&
5402       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5403     SanitizerScope SanScope(this);
5404     EmitSanitizerStatReport(llvm::SanStat_CFI_ICall);
5405 
5406     llvm::Metadata *MD;
5407     if (CGM.getCodeGenOpts().SanitizeCfiICallGeneralizePointers)
5408       MD = CGM.CreateMetadataIdentifierGeneralized(QualType(FnType, 0));
5409     else
5410       MD = CGM.CreateMetadataIdentifierForType(QualType(FnType, 0));
5411 
5412     llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
5413 
5414     llvm::Value *CalleePtr = Callee.getFunctionPointer();
5415     llvm::Value *CastedCallee = Builder.CreateBitCast(CalleePtr, Int8PtrTy);
5416     llvm::Value *TypeTest = Builder.CreateCall(
5417         CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedCallee, TypeId});
5418 
5419     auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
5420     llvm::Constant *StaticData[] = {
5421         llvm::ConstantInt::get(Int8Ty, CFITCK_ICall),
5422         EmitCheckSourceLocation(E->getBeginLoc()),
5423         EmitCheckTypeDescriptor(QualType(FnType, 0)),
5424     };
5425     if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
5426       EmitCfiSlowPathCheck(SanitizerKind::CFIICall, TypeTest, CrossDsoTypeId,
5427                            CastedCallee, StaticData);
5428     } else {
5429       EmitCheck(std::make_pair(TypeTest, SanitizerKind::CFIICall),
5430                 SanitizerHandler::CFICheckFail, StaticData,
5431                 {CastedCallee, llvm::UndefValue::get(IntPtrTy)});
5432     }
5433   }
5434 
5435   CallArgList Args;
5436   if (Chain)
5437     Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)),
5438              CGM.getContext().VoidPtrTy);
5439 
5440   // C++17 requires that we evaluate arguments to a call using assignment syntax
5441   // right-to-left, and that we evaluate arguments to certain other operators
5442   // left-to-right. Note that we allow this to override the order dictated by
5443   // the calling convention on the MS ABI, which means that parameter
5444   // destruction order is not necessarily reverse construction order.
5445   // FIXME: Revisit this based on C++ committee response to unimplementability.
5446   EvaluationOrder Order = EvaluationOrder::Default;
5447   if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) {
5448     if (OCE->isAssignmentOp())
5449       Order = EvaluationOrder::ForceRightToLeft;
5450     else {
5451       switch (OCE->getOperator()) {
5452       case OO_LessLess:
5453       case OO_GreaterGreater:
5454       case OO_AmpAmp:
5455       case OO_PipePipe:
5456       case OO_Comma:
5457       case OO_ArrowStar:
5458         Order = EvaluationOrder::ForceLeftToRight;
5459         break;
5460       default:
5461         break;
5462       }
5463     }
5464   }
5465 
5466   EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(),
5467                E->getDirectCallee(), /*ParamsToSkip*/ 0, Order);
5468 
5469   const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
5470       Args, FnType, /*ChainCall=*/Chain);
5471 
5472   // C99 6.5.2.2p6:
5473   //   If the expression that denotes the called function has a type
5474   //   that does not include a prototype, [the default argument
5475   //   promotions are performed]. If the number of arguments does not
5476   //   equal the number of parameters, the behavior is undefined. If
5477   //   the function is defined with a type that includes a prototype,
5478   //   and either the prototype ends with an ellipsis (, ...) or the
5479   //   types of the arguments after promotion are not compatible with
5480   //   the types of the parameters, the behavior is undefined. If the
5481   //   function is defined with a type that does not include a
5482   //   prototype, and the types of the arguments after promotion are
5483   //   not compatible with those of the parameters after promotion,
5484   //   the behavior is undefined [except in some trivial cases].
5485   // That is, in the general case, we should assume that a call
5486   // through an unprototyped function type works like a *non-variadic*
5487   // call.  The way we make this work is to cast to the exact type
5488   // of the promoted arguments.
5489   //
5490   // Chain calls use this same code path to add the invisible chain parameter
5491   // to the function type.
5492   if (isa<FunctionNoProtoType>(FnType) || Chain) {
5493     llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
5494     int AS = Callee.getFunctionPointer()->getType()->getPointerAddressSpace();
5495     CalleeTy = CalleeTy->getPointerTo(AS);
5496 
5497     llvm::Value *CalleePtr = Callee.getFunctionPointer();
5498     CalleePtr = Builder.CreateBitCast(CalleePtr, CalleeTy, "callee.knr.cast");
5499     Callee.setFunctionPointer(CalleePtr);
5500   }
5501 
5502   // HIP function pointer contains kernel handle when it is used in triple
5503   // chevron. The kernel stub needs to be loaded from kernel handle and used
5504   // as callee.
5505   if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
5506       isa<CUDAKernelCallExpr>(E) &&
5507       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5508     llvm::Value *Handle = Callee.getFunctionPointer();
5509     auto *Cast =
5510         Builder.CreateBitCast(Handle, Handle->getType()->getPointerTo());
5511     auto *Stub = Builder.CreateLoad(
5512         Address(Cast, Handle->getType(), CGM.getPointerAlign()));
5513     Callee.setFunctionPointer(Stub);
5514   }
5515   llvm::CallBase *CallOrInvoke = nullptr;
5516   RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &CallOrInvoke,
5517                          E == MustTailCall, E->getExprLoc());
5518 
5519   // Generate function declaration DISuprogram in order to be used
5520   // in debug info about call sites.
5521   if (CGDebugInfo *DI = getDebugInfo()) {
5522     if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
5523       FunctionArgList Args;
5524       QualType ResTy = BuildFunctionArgList(CalleeDecl, Args);
5525       DI->EmitFuncDeclForCallSite(CallOrInvoke,
5526                                   DI->getFunctionType(CalleeDecl, ResTy, Args),
5527                                   CalleeDecl);
5528     }
5529   }
5530 
5531   return Call;
5532 }
5533 
5534 LValue CodeGenFunction::
5535 EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
5536   Address BaseAddr = Address::invalid();
5537   if (E->getOpcode() == BO_PtrMemI) {
5538     BaseAddr = EmitPointerWithAlignment(E->getLHS());
5539   } else {
5540     BaseAddr = EmitLValue(E->getLHS()).getAddress(*this);
5541   }
5542 
5543   llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
5544   const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
5545 
5546   LValueBaseInfo BaseInfo;
5547   TBAAAccessInfo TBAAInfo;
5548   Address MemberAddr =
5549     EmitCXXMemberDataPointerAddress(E, BaseAddr, OffsetV, MPT, &BaseInfo,
5550                                     &TBAAInfo);
5551 
5552   return MakeAddrLValue(MemberAddr, MPT->getPointeeType(), BaseInfo, TBAAInfo);
5553 }
5554 
5555 /// Given the address of a temporary variable, produce an r-value of
5556 /// its type.
5557 RValue CodeGenFunction::convertTempToRValue(Address addr,
5558                                             QualType type,
5559                                             SourceLocation loc) {
5560   LValue lvalue = MakeAddrLValue(addr, type, AlignmentSource::Decl);
5561   switch (getEvaluationKind(type)) {
5562   case TEK_Complex:
5563     return RValue::getComplex(EmitLoadOfComplex(lvalue, loc));
5564   case TEK_Aggregate:
5565     return lvalue.asAggregateRValue(*this);
5566   case TEK_Scalar:
5567     return RValue::get(EmitLoadOfScalar(lvalue, loc));
5568   }
5569   llvm_unreachable("bad evaluation kind");
5570 }
5571 
5572 void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
5573   assert(Val->getType()->isFPOrFPVectorTy());
5574   if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
5575     return;
5576 
5577   llvm::MDBuilder MDHelper(getLLVMContext());
5578   llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
5579 
5580   cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node);
5581 }
5582 
5583 void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {
5584   llvm::Type *EltTy = Val->getType()->getScalarType();
5585   if (!EltTy->isFloatTy())
5586     return;
5587 
5588   if ((getLangOpts().OpenCL &&
5589        !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
5590       (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
5591        !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
5592     // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 3ulp
5593     //
5594     // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
5595     // build option allows an application to specify that single precision
5596     // floating-point divide (x/y and 1/x) and sqrt used in the program
5597     // source are correctly rounded.
5598     //
5599     // TODO: CUDA has a prec-sqrt flag
5600     SetFPAccuracy(Val, 3.0f);
5601   }
5602 }
5603 
5604 void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {
5605   llvm::Type *EltTy = Val->getType()->getScalarType();
5606   if (!EltTy->isFloatTy())
5607     return;
5608 
5609   if ((getLangOpts().OpenCL &&
5610        !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||
5611       (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&
5612        !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {
5613     // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5ulp
5614     //
5615     // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt
5616     // build option allows an application to specify that single precision
5617     // floating-point divide (x/y and 1/x) and sqrt used in the program
5618     // source are correctly rounded.
5619     //
5620     // TODO: CUDA has a prec-div flag
5621     SetFPAccuracy(Val, 2.5f);
5622   }
5623 }
5624 
5625 namespace {
5626   struct LValueOrRValue {
5627     LValue LV;
5628     RValue RV;
5629   };
5630 }
5631 
5632 static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
5633                                            const PseudoObjectExpr *E,
5634                                            bool forLValue,
5635                                            AggValueSlot slot) {
5636   SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
5637 
5638   // Find the result expression, if any.
5639   const Expr *resultExpr = E->getResultExpr();
5640   LValueOrRValue result;
5641 
5642   for (PseudoObjectExpr::const_semantics_iterator
5643          i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
5644     const Expr *semantic = *i;
5645 
5646     // If this semantic expression is an opaque value, bind it
5647     // to the result of its source expression.
5648     if (const auto *ov = dyn_cast<OpaqueValueExpr>(semantic)) {
5649       // Skip unique OVEs.
5650       if (ov->isUnique()) {
5651         assert(ov != resultExpr &&
5652                "A unique OVE cannot be used as the result expression");
5653         continue;
5654       }
5655 
5656       // If this is the result expression, we may need to evaluate
5657       // directly into the slot.
5658       typedef CodeGenFunction::OpaqueValueMappingData OVMA;
5659       OVMA opaqueData;
5660       if (ov == resultExpr && ov->isPRValue() && !forLValue &&
5661           CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) {
5662         CGF.EmitAggExpr(ov->getSourceExpr(), slot);
5663         LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(),
5664                                        AlignmentSource::Decl);
5665         opaqueData = OVMA::bind(CGF, ov, LV);
5666         result.RV = slot.asRValue();
5667 
5668       // Otherwise, emit as normal.
5669       } else {
5670         opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr());
5671 
5672         // If this is the result, also evaluate the result now.
5673         if (ov == resultExpr) {
5674           if (forLValue)
5675             result.LV = CGF.EmitLValue(ov);
5676           else
5677             result.RV = CGF.EmitAnyExpr(ov, slot);
5678         }
5679       }
5680 
5681       opaques.push_back(opaqueData);
5682 
5683     // Otherwise, if the expression is the result, evaluate it
5684     // and remember the result.
5685     } else if (semantic == resultExpr) {
5686       if (forLValue)
5687         result.LV = CGF.EmitLValue(semantic);
5688       else
5689         result.RV = CGF.EmitAnyExpr(semantic, slot);
5690 
5691     // Otherwise, evaluate the expression in an ignored context.
5692     } else {
5693       CGF.EmitIgnoredExpr(semantic);
5694     }
5695   }
5696 
5697   // Unbind all the opaques now.
5698   for (unsigned i = 0, e = opaques.size(); i != e; ++i)
5699     opaques[i].unbind(CGF);
5700 
5701   return result;
5702 }
5703 
5704 RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
5705                                                AggValueSlot slot) {
5706   return emitPseudoObjectExpr(*this, E, false, slot).RV;
5707 }
5708 
5709 LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
5710   return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;
5711 }
5712