xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp (revision 4d3fc8b0570b29fb0d6ee9525f104d52176ff0d4)
1 //===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Expr nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCUDARuntime.h"
14 #include "CGCXXABI.h"
15 #include "CGCall.h"
16 #include "CGCleanup.h"
17 #include "CGDebugInfo.h"
18 #include "CGObjCRuntime.h"
19 #include "CGOpenMPRuntime.h"
20 #include "CGRecordLayout.h"
21 #include "CodeGenFunction.h"
22 #include "CodeGenModule.h"
23 #include "ConstantEmitter.h"
24 #include "TargetInfo.h"
25 #include "clang/AST/ASTContext.h"
26 #include "clang/AST/Attr.h"
27 #include "clang/AST/DeclObjC.h"
28 #include "clang/AST/NSAPI.h"
29 #include "clang/Basic/Builtins.h"
30 #include "clang/Basic/CodeGenOptions.h"
31 #include "clang/Basic/SourceManager.h"
32 #include "llvm/ADT/Hashing.h"
33 #include "llvm/ADT/StringExtras.h"
34 #include "llvm/IR/DataLayout.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/IR/LLVMContext.h"
37 #include "llvm/IR/MDBuilder.h"
38 #include "llvm/IR/MatrixBuilder.h"
39 #include "llvm/Support/ConvertUTF.h"
40 #include "llvm/Support/MathExtras.h"
41 #include "llvm/Support/Path.h"
42 #include "llvm/Support/SaveAndRestore.h"
43 #include "llvm/Transforms/Utils/SanitizerStats.h"
44 
45 #include <string>
46 
47 using namespace clang;
48 using namespace CodeGen;
49 
50 //===--------------------------------------------------------------------===//
51 //                        Miscellaneous Helper Methods
52 //===--------------------------------------------------------------------===//
53 
54 llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
55   unsigned addressSpace =
56       cast<llvm::PointerType>(value->getType())->getAddressSpace();
57 
58   llvm::PointerType *destType = Int8PtrTy;
59   if (addressSpace)
60     destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
61 
62   if (value->getType() == destType) return value;
63   return Builder.CreateBitCast(value, destType);
64 }
65 
66 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
67 /// block.
68 Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
69                                                      CharUnits Align,
70                                                      const Twine &Name,
71                                                      llvm::Value *ArraySize) {
72   auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
73   Alloca->setAlignment(Align.getAsAlign());
74   return Address(Alloca, Ty, Align);
75 }
76 
77 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
78 /// block. The alloca is casted to default address space if necessary.
79 Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
80                                           const Twine &Name,
81                                           llvm::Value *ArraySize,
82                                           Address *AllocaAddr) {
83   auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
84   if (AllocaAddr)
85     *AllocaAddr = Alloca;
86   llvm::Value *V = Alloca.getPointer();
87   // Alloca always returns a pointer in alloca address space, which may
88   // be different from the type defined by the language. For example,
89   // in C++ the auto variables are in the default address space. Therefore
90   // cast alloca to the default address space when necessary.
91   if (getASTAllocaAddressSpace() != LangAS::Default) {
92     auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default);
93     llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
94     // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
95     // otherwise alloca is inserted at the current insertion point of the
96     // builder.
97     if (!ArraySize)
98       Builder.SetInsertPoint(getPostAllocaInsertPoint());
99     V = getTargetHooks().performAddrSpaceCast(
100         *this, V, getASTAllocaAddressSpace(), LangAS::Default,
101         Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
102   }
103 
104   return Address(V, Ty, Align);
105 }
106 
107 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
108 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
109 /// insertion point of the builder.
110 llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
111                                                     const Twine &Name,
112                                                     llvm::Value *ArraySize) {
113   if (ArraySize)
114     return Builder.CreateAlloca(Ty, ArraySize, Name);
115   return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
116                               ArraySize, Name, AllocaInsertPt);
117 }
118 
119 /// CreateDefaultAlignTempAlloca - This creates an alloca with the
120 /// default alignment of the corresponding LLVM type, which is *not*
121 /// guaranteed to be related in any way to the expected alignment of
122 /// an AST type that might have been lowered to Ty.
123 Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
124                                                       const Twine &Name) {
125   CharUnits Align =
126       CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlignment(Ty));
127   return CreateTempAlloca(Ty, Align, Name);
128 }
129 
130 Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
131   CharUnits Align = getContext().getTypeAlignInChars(Ty);
132   return CreateTempAlloca(ConvertType(Ty), Align, Name);
133 }
134 
135 Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
136                                        Address *Alloca) {
137   // FIXME: Should we prefer the preferred type alignment here?
138   return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);
139 }
140 
141 Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
142                                        const Twine &Name, Address *Alloca) {
143   Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,
144                                     /*ArraySize=*/nullptr, Alloca);
145 
146   if (Ty->isConstantMatrixType()) {
147     auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
148     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
149                                                 ArrayTy->getNumElements());
150 
151     Result = Address(
152         Builder.CreateBitCast(Result.getPointer(), VectorTy->getPointerTo()),
153         VectorTy, Result.getAlignment());
154   }
155   return Result;
156 }
157 
158 Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align,
159                                                   const Twine &Name) {
160   return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);
161 }
162 
163 Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
164                                                   const Twine &Name) {
165   return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),
166                                   Name);
167 }
168 
169 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
170 /// expression and compare the result against zero, returning an Int1Ty value.
171 llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
172   PGO.setCurrentStmt(E);
173   if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
174     llvm::Value *MemPtr = EmitScalarExpr(E);
175     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
176   }
177 
178   QualType BoolTy = getContext().BoolTy;
179   SourceLocation Loc = E->getExprLoc();
180   CGFPOptionsRAII FPOptsRAII(*this, E);
181   if (!E->getType()->isAnyComplexType())
182     return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc);
183 
184   return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy,
185                                        Loc);
186 }
187 
188 /// EmitIgnoredExpr - Emit code to compute the specified expression,
189 /// ignoring the result.
190 void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
191   if (E->isPRValue())
192     return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true);
193 
194   // if this is a bitfield-resulting conditional operator, we can special case
195   // emit this. The normal 'EmitLValue' version of this is particularly
196   // difficult to codegen for, since creating a single "LValue" for two
197   // different sized arguments here is not particularly doable.
198   if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
199           E->IgnoreParenNoopCasts(getContext()))) {
200     if (CondOp->getObjectKind() == OK_BitField)
201       return EmitIgnoredConditionalOperator(CondOp);
202   }
203 
204   // Just emit it as an l-value and drop the result.
205   EmitLValue(E);
206 }
207 
208 /// EmitAnyExpr - Emit code to compute the specified expression which
209 /// can have any type.  The result is returned as an RValue struct.
210 /// If this is an aggregate expression, AggSlot indicates where the
211 /// result should be returned.
212 RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
213                                     AggValueSlot aggSlot,
214                                     bool ignoreResult) {
215   switch (getEvaluationKind(E->getType())) {
216   case TEK_Scalar:
217     return RValue::get(EmitScalarExpr(E, ignoreResult));
218   case TEK_Complex:
219     return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));
220   case TEK_Aggregate:
221     if (!ignoreResult && aggSlot.isIgnored())
222       aggSlot = CreateAggTemp(E->getType(), "agg-temp");
223     EmitAggExpr(E, aggSlot);
224     return aggSlot.asRValue();
225   }
226   llvm_unreachable("bad evaluation kind");
227 }
228 
229 /// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
230 /// always be accessible even if no aggregate location is provided.
231 RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
232   AggValueSlot AggSlot = AggValueSlot::ignored();
233 
234   if (hasAggregateEvaluationKind(E->getType()))
235     AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
236   return EmitAnyExpr(E, AggSlot);
237 }
238 
239 /// EmitAnyExprToMem - Evaluate an expression into a given memory
240 /// location.
241 void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
242                                        Address Location,
243                                        Qualifiers Quals,
244                                        bool IsInit) {
245   // FIXME: This function should take an LValue as an argument.
246   switch (getEvaluationKind(E->getType())) {
247   case TEK_Complex:
248     EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()),
249                               /*isInit*/ false);
250     return;
251 
252   case TEK_Aggregate: {
253     EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
254                                          AggValueSlot::IsDestructed_t(IsInit),
255                                          AggValueSlot::DoesNotNeedGCBarriers,
256                                          AggValueSlot::IsAliased_t(!IsInit),
257                                          AggValueSlot::MayOverlap));
258     return;
259   }
260 
261   case TEK_Scalar: {
262     RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
263     LValue LV = MakeAddrLValue(Location, E->getType());
264     EmitStoreThroughLValue(RV, LV);
265     return;
266   }
267   }
268   llvm_unreachable("bad evaluation kind");
269 }
270 
271 static void
272 pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
273                      const Expr *E, Address ReferenceTemporary) {
274   // Objective-C++ ARC:
275   //   If we are binding a reference to a temporary that has ownership, we
276   //   need to perform retain/release operations on the temporary.
277   //
278   // FIXME: This should be looking at E, not M.
279   if (auto Lifetime = M->getType().getObjCLifetime()) {
280     switch (Lifetime) {
281     case Qualifiers::OCL_None:
282     case Qualifiers::OCL_ExplicitNone:
283       // Carry on to normal cleanup handling.
284       break;
285 
286     case Qualifiers::OCL_Autoreleasing:
287       // Nothing to do; cleaned up by an autorelease pool.
288       return;
289 
290     case Qualifiers::OCL_Strong:
291     case Qualifiers::OCL_Weak:
292       switch (StorageDuration Duration = M->getStorageDuration()) {
293       case SD_Static:
294         // Note: we intentionally do not register a cleanup to release
295         // the object on program termination.
296         return;
297 
298       case SD_Thread:
299         // FIXME: We should probably register a cleanup in this case.
300         return;
301 
302       case SD_Automatic:
303       case SD_FullExpression:
304         CodeGenFunction::Destroyer *Destroy;
305         CleanupKind CleanupKind;
306         if (Lifetime == Qualifiers::OCL_Strong) {
307           const ValueDecl *VD = M->getExtendingDecl();
308           bool Precise =
309               VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>();
310           CleanupKind = CGF.getARCCleanupKind();
311           Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
312                             : &CodeGenFunction::destroyARCStrongImprecise;
313         } else {
314           // __weak objects always get EH cleanups; otherwise, exceptions
315           // could cause really nasty crashes instead of mere leaks.
316           CleanupKind = NormalAndEHCleanup;
317           Destroy = &CodeGenFunction::destroyARCWeak;
318         }
319         if (Duration == SD_FullExpression)
320           CGF.pushDestroy(CleanupKind, ReferenceTemporary,
321                           M->getType(), *Destroy,
322                           CleanupKind & EHCleanup);
323         else
324           CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,
325                                           M->getType(),
326                                           *Destroy, CleanupKind & EHCleanup);
327         return;
328 
329       case SD_Dynamic:
330         llvm_unreachable("temporary cannot have dynamic storage duration");
331       }
332       llvm_unreachable("unknown storage duration");
333     }
334   }
335 
336   CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
337   if (const RecordType *RT =
338           E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
339     // Get the destructor for the reference temporary.
340     auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
341     if (!ClassDecl->hasTrivialDestructor())
342       ReferenceTemporaryDtor = ClassDecl->getDestructor();
343   }
344 
345   if (!ReferenceTemporaryDtor)
346     return;
347 
348   // Call the destructor for the temporary.
349   switch (M->getStorageDuration()) {
350   case SD_Static:
351   case SD_Thread: {
352     llvm::FunctionCallee CleanupFn;
353     llvm::Constant *CleanupArg;
354     if (E->getType()->isArrayType()) {
355       CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
356           ReferenceTemporary, E->getType(),
357           CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions,
358           dyn_cast_or_null<VarDecl>(M->getExtendingDecl()));
359       CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);
360     } else {
361       CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
362           GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
363       CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
364     }
365     CGF.CGM.getCXXABI().registerGlobalDtor(
366         CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
367     break;
368   }
369 
370   case SD_FullExpression:
371     CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(),
372                     CodeGenFunction::destroyCXXObject,
373                     CGF.getLangOpts().Exceptions);
374     break;
375 
376   case SD_Automatic:
377     CGF.pushLifetimeExtendedDestroy(NormalAndEHCleanup,
378                                     ReferenceTemporary, E->getType(),
379                                     CodeGenFunction::destroyCXXObject,
380                                     CGF.getLangOpts().Exceptions);
381     break;
382 
383   case SD_Dynamic:
384     llvm_unreachable("temporary cannot have dynamic storage duration");
385   }
386 }
387 
388 static Address createReferenceTemporary(CodeGenFunction &CGF,
389                                         const MaterializeTemporaryExpr *M,
390                                         const Expr *Inner,
391                                         Address *Alloca = nullptr) {
392   auto &TCG = CGF.getTargetHooks();
393   switch (M->getStorageDuration()) {
394   case SD_FullExpression:
395   case SD_Automatic: {
396     // If we have a constant temporary array or record try to promote it into a
397     // constant global under the same rules a normal constant would've been
398     // promoted. This is easier on the optimizer and generally emits fewer
399     // instructions.
400     QualType Ty = Inner->getType();
401     if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
402         (Ty->isArrayType() || Ty->isRecordType()) &&
403         CGF.CGM.isTypeConstant(Ty, true))
404       if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) {
405         auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
406         auto *GV = new llvm::GlobalVariable(
407             CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
408             llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
409             llvm::GlobalValue::NotThreadLocal,
410             CGF.getContext().getTargetAddressSpace(AS));
411         CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
412         GV->setAlignment(alignment.getAsAlign());
413         llvm::Constant *C = GV;
414         if (AS != LangAS::Default)
415           C = TCG.performAddrSpaceCast(
416               CGF.CGM, GV, AS, LangAS::Default,
417               GV->getValueType()->getPointerTo(
418                   CGF.getContext().getTargetAddressSpace(LangAS::Default)));
419         // FIXME: Should we put the new global into a COMDAT?
420         return Address(C, GV->getValueType(), alignment);
421       }
422     return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
423   }
424   case SD_Thread:
425   case SD_Static:
426     return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);
427 
428   case SD_Dynamic:
429     llvm_unreachable("temporary can't have dynamic storage duration");
430   }
431   llvm_unreachable("unknown storage duration");
432 }
433 
434 /// Helper method to check if the underlying ABI is AAPCS
435 static bool isAAPCS(const TargetInfo &TargetInfo) {
436   return TargetInfo.getABI().startswith("aapcs");
437 }
438 
439 LValue CodeGenFunction::
440 EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
441   const Expr *E = M->getSubExpr();
442 
443   assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
444           !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
445          "Reference should never be pseudo-strong!");
446 
447   // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
448   // as that will cause the lifetime adjustment to be lost for ARC
449   auto ownership = M->getType().getObjCLifetime();
450   if (ownership != Qualifiers::OCL_None &&
451       ownership != Qualifiers::OCL_ExplicitNone) {
452     Address Object = createReferenceTemporary(*this, M, E);
453     if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
454       llvm::Type *Ty = ConvertTypeForMem(E->getType());
455       Object = Address(llvm::ConstantExpr::getBitCast(
456                            Var, Ty->getPointerTo(Object.getAddressSpace())),
457                        Ty, Object.getAlignment());
458 
459       // createReferenceTemporary will promote the temporary to a global with a
460       // constant initializer if it can.  It can only do this to a value of
461       // ARC-manageable type if the value is global and therefore "immune" to
462       // ref-counting operations.  Therefore we have no need to emit either a
463       // dynamic initialization or a cleanup and we can just return the address
464       // of the temporary.
465       if (Var->hasInitializer())
466         return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
467 
468       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
469     }
470     LValue RefTempDst = MakeAddrLValue(Object, M->getType(),
471                                        AlignmentSource::Decl);
472 
473     switch (getEvaluationKind(E->getType())) {
474     default: llvm_unreachable("expected scalar or aggregate expression");
475     case TEK_Scalar:
476       EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);
477       break;
478     case TEK_Aggregate: {
479       EmitAggExpr(E, AggValueSlot::forAddr(Object,
480                                            E->getType().getQualifiers(),
481                                            AggValueSlot::IsDestructed,
482                                            AggValueSlot::DoesNotNeedGCBarriers,
483                                            AggValueSlot::IsNotAliased,
484                                            AggValueSlot::DoesNotOverlap));
485       break;
486     }
487     }
488 
489     pushTemporaryCleanup(*this, M, E, Object);
490     return RefTempDst;
491   }
492 
493   SmallVector<const Expr *, 2> CommaLHSs;
494   SmallVector<SubobjectAdjustment, 2> Adjustments;
495   E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);
496 
497   for (const auto &Ignored : CommaLHSs)
498     EmitIgnoredExpr(Ignored);
499 
500   if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) {
501     if (opaque->getType()->isRecordType()) {
502       assert(Adjustments.empty());
503       return EmitOpaqueValueLValue(opaque);
504     }
505   }
506 
507   // Create and initialize the reference temporary.
508   Address Alloca = Address::invalid();
509   Address Object = createReferenceTemporary(*this, M, E, &Alloca);
510   if (auto *Var = dyn_cast<llvm::GlobalVariable>(
511           Object.getPointer()->stripPointerCasts())) {
512     llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
513     Object = Address(llvm::ConstantExpr::getBitCast(
514                          cast<llvm::Constant>(Object.getPointer()),
515                          TemporaryType->getPointerTo()),
516                      TemporaryType,
517                      Object.getAlignment());
518     // If the temporary is a global and has a constant initializer or is a
519     // constant temporary that we promoted to a global, we may have already
520     // initialized it.
521     if (!Var->hasInitializer()) {
522       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
523       EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
524     }
525   } else {
526     switch (M->getStorageDuration()) {
527     case SD_Automatic:
528       if (auto *Size = EmitLifetimeStart(
529               CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
530               Alloca.getPointer())) {
531         pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker,
532                                                   Alloca, Size);
533       }
534       break;
535 
536     case SD_FullExpression: {
537       if (!ShouldEmitLifetimeMarkers)
538         break;
539 
540       // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
541       // marker. Instead, start the lifetime of a conditional temporary earlier
542       // so that it's unconditional. Don't do this with sanitizers which need
543       // more precise lifetime marks.
544       ConditionalEvaluation *OldConditional = nullptr;
545       CGBuilderTy::InsertPoint OldIP;
546       if (isInConditionalBranch() && !E->getType().isDestructedType() &&
547           !SanOpts.has(SanitizerKind::HWAddress) &&
548           !SanOpts.has(SanitizerKind::Memory) &&
549           !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) {
550         OldConditional = OutermostConditional;
551         OutermostConditional = nullptr;
552 
553         OldIP = Builder.saveIP();
554         llvm::BasicBlock *Block = OldConditional->getStartingBlock();
555         Builder.restoreIP(CGBuilderTy::InsertPoint(
556             Block, llvm::BasicBlock::iterator(Block->back())));
557       }
558 
559       if (auto *Size = EmitLifetimeStart(
560               CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
561               Alloca.getPointer())) {
562         pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca,
563                                              Size);
564       }
565 
566       if (OldConditional) {
567         OutermostConditional = OldConditional;
568         Builder.restoreIP(OldIP);
569       }
570       break;
571     }
572 
573     default:
574       break;
575     }
576     EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
577   }
578   pushTemporaryCleanup(*this, M, E, Object);
579 
580   // Perform derived-to-base casts and/or field accesses, to get from the
581   // temporary object we created (and, potentially, for which we extended
582   // the lifetime) to the subobject we're binding the reference to.
583   for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) {
584     switch (Adjustment.Kind) {
585     case SubobjectAdjustment::DerivedToBaseAdjustment:
586       Object =
587           GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass,
588                                 Adjustment.DerivedToBase.BasePath->path_begin(),
589                                 Adjustment.DerivedToBase.BasePath->path_end(),
590                                 /*NullCheckValue=*/ false, E->getExprLoc());
591       break;
592 
593     case SubobjectAdjustment::FieldAdjustment: {
594       LValue LV = MakeAddrLValue(Object, E->getType(), AlignmentSource::Decl);
595       LV = EmitLValueForField(LV, Adjustment.Field);
596       assert(LV.isSimple() &&
597              "materialized temporary field is not a simple lvalue");
598       Object = LV.getAddress(*this);
599       break;
600     }
601 
602     case SubobjectAdjustment::MemberPointerAdjustment: {
603       llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS);
604       Object = EmitCXXMemberDataPointerAddress(E, Object, Ptr,
605                                                Adjustment.Ptr.MPT);
606       break;
607     }
608     }
609   }
610 
611   return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
612 }
613 
614 RValue
615 CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
616   // Emit the expression as an lvalue.
617   LValue LV = EmitLValue(E);
618   assert(LV.isSimple());
619   llvm::Value *Value = LV.getPointer(*this);
620 
621   if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
622     // C++11 [dcl.ref]p5 (as amended by core issue 453):
623     //   If a glvalue to which a reference is directly bound designates neither
624     //   an existing object or function of an appropriate type nor a region of
625     //   storage of suitable size and alignment to contain an object of the
626     //   reference's type, the behavior is undefined.
627     QualType Ty = E->getType();
628     EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty);
629   }
630 
631   return RValue::get(Value);
632 }
633 
634 
635 /// getAccessedFieldNo - Given an encoded value and a result number, return the
636 /// input field number being accessed.
637 unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
638                                              const llvm::Constant *Elts) {
639   return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
640       ->getZExtValue();
641 }
642 
643 /// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
644 static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
645                                     llvm::Value *High) {
646   llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
647   llvm::Value *K47 = Builder.getInt64(47);
648   llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
649   llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
650   llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
651   llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
652   return Builder.CreateMul(B1, KMul);
653 }
654 
655 bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
656   return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
657          TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
658 }
659 
660 bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
661   CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
662   return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
663          (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
664           TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
665           TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
666 }
667 
668 bool CodeGenFunction::sanitizePerformTypeCheck() const {
669   return SanOpts.has(SanitizerKind::Null) ||
670          SanOpts.has(SanitizerKind::Alignment) ||
671          SanOpts.has(SanitizerKind::ObjectSize) ||
672          SanOpts.has(SanitizerKind::Vptr);
673 }
674 
675 void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
676                                     llvm::Value *Ptr, QualType Ty,
677                                     CharUnits Alignment,
678                                     SanitizerSet SkippedChecks,
679                                     llvm::Value *ArraySize) {
680   if (!sanitizePerformTypeCheck())
681     return;
682 
683   // Don't check pointers outside the default address space. The null check
684   // isn't correct, the object-size check isn't supported by LLVM, and we can't
685   // communicate the addresses to the runtime handler for the vptr check.
686   if (Ptr->getType()->getPointerAddressSpace())
687     return;
688 
689   // Don't check pointers to volatile data. The behavior here is implementation-
690   // defined.
691   if (Ty.isVolatileQualified())
692     return;
693 
694   SanitizerScope SanScope(this);
695 
696   SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks;
697   llvm::BasicBlock *Done = nullptr;
698 
699   // Quickly determine whether we have a pointer to an alloca. It's possible
700   // to skip null checks, and some alignment checks, for these pointers. This
701   // can reduce compile-time significantly.
702   auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
703 
704   llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());
705   llvm::Value *IsNonNull = nullptr;
706   bool IsGuaranteedNonNull =
707       SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca;
708   bool AllowNullPointers = isNullPointerAllowed(TCK);
709   if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
710       !IsGuaranteedNonNull) {
711     // The glvalue must not be an empty glvalue.
712     IsNonNull = Builder.CreateIsNotNull(Ptr);
713 
714     // The IR builder can constant-fold the null check if the pointer points to
715     // a constant.
716     IsGuaranteedNonNull = IsNonNull == True;
717 
718     // Skip the null check if the pointer is known to be non-null.
719     if (!IsGuaranteedNonNull) {
720       if (AllowNullPointers) {
721         // When performing pointer casts, it's OK if the value is null.
722         // Skip the remaining checks in that case.
723         Done = createBasicBlock("null");
724         llvm::BasicBlock *Rest = createBasicBlock("not.null");
725         Builder.CreateCondBr(IsNonNull, Rest, Done);
726         EmitBlock(Rest);
727       } else {
728         Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null));
729       }
730     }
731   }
732 
733   if (SanOpts.has(SanitizerKind::ObjectSize) &&
734       !SkippedChecks.has(SanitizerKind::ObjectSize) &&
735       !Ty->isIncompleteType()) {
736     uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
737     llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);
738     if (ArraySize)
739       Size = Builder.CreateMul(Size, ArraySize);
740 
741     // Degenerate case: new X[0] does not need an objectsize check.
742     llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
743     if (!ConstantSize || !ConstantSize->isNullValue()) {
744       // The glvalue must refer to a large enough storage region.
745       // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
746       //        to check this.
747       // FIXME: Get object address space
748       llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
749       llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
750       llvm::Value *Min = Builder.getFalse();
751       llvm::Value *NullIsUnknown = Builder.getFalse();
752       llvm::Value *Dynamic = Builder.getFalse();
753       llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy);
754       llvm::Value *LargeEnough = Builder.CreateICmpUGE(
755           Builder.CreateCall(F, {CastAddr, Min, NullIsUnknown, Dynamic}), Size);
756       Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
757     }
758   }
759 
760   llvm::MaybeAlign AlignVal;
761   llvm::Value *PtrAsInt = nullptr;
762 
763   if (SanOpts.has(SanitizerKind::Alignment) &&
764       !SkippedChecks.has(SanitizerKind::Alignment)) {
765     AlignVal = Alignment.getAsMaybeAlign();
766     if (!Ty->isIncompleteType() && !AlignVal)
767       AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
768                                              /*ForPointeeType=*/true)
769                      .getAsMaybeAlign();
770 
771     // The glvalue must be suitably aligned.
772     if (AlignVal && *AlignVal > llvm::Align(1) &&
773         (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
774       PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
775       llvm::Value *Align = Builder.CreateAnd(
776           PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
777       llvm::Value *Aligned =
778           Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
779       if (Aligned != True)
780         Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment));
781     }
782   }
783 
784   if (Checks.size() > 0) {
785     llvm::Constant *StaticData[] = {
786         EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty),
787         llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
788         llvm::ConstantInt::get(Int8Ty, TCK)};
789     EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
790               PtrAsInt ? PtrAsInt : Ptr);
791   }
792 
793   // If possible, check that the vptr indicates that there is a subobject of
794   // type Ty at offset zero within this object.
795   //
796   // C++11 [basic.life]p5,6:
797   //   [For storage which does not refer to an object within its lifetime]
798   //   The program has undefined behavior if:
799   //    -- the [pointer or glvalue] is used to access a non-static data member
800   //       or call a non-static member function
801   if (SanOpts.has(SanitizerKind::Vptr) &&
802       !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
803     // Ensure that the pointer is non-null before loading it. If there is no
804     // compile-time guarantee, reuse the run-time null check or emit a new one.
805     if (!IsGuaranteedNonNull) {
806       if (!IsNonNull)
807         IsNonNull = Builder.CreateIsNotNull(Ptr);
808       if (!Done)
809         Done = createBasicBlock("vptr.null");
810       llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null");
811       Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);
812       EmitBlock(VptrNotNull);
813     }
814 
815     // Compute a hash of the mangled name of the type.
816     //
817     // FIXME: This is not guaranteed to be deterministic! Move to a
818     //        fingerprinting mechanism once LLVM provides one. For the time
819     //        being the implementation happens to be deterministic.
820     SmallString<64> MangledName;
821     llvm::raw_svector_ostream Out(MangledName);
822     CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(),
823                                                      Out);
824 
825     // Contained in NoSanitizeList based on the mangled type.
826     if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr,
827                                                            Out.str())) {
828       llvm::hash_code TypeHash = hash_value(Out.str());
829 
830       // Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
831       llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
832       llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
833       Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), IntPtrTy,
834                        getPointerAlign());
835       llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
836       llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
837 
838       llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
839       Hash = Builder.CreateTrunc(Hash, IntPtrTy);
840 
841       // Look the hash up in our cache.
842       const int CacheSize = 128;
843       llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);
844       llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,
845                                                      "__ubsan_vptr_type_cache");
846       llvm::Value *Slot = Builder.CreateAnd(Hash,
847                                             llvm::ConstantInt::get(IntPtrTy,
848                                                                    CacheSize-1));
849       llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
850       llvm::Value *CacheVal = Builder.CreateAlignedLoad(
851           IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices),
852           getPointerAlign());
853 
854       // If the hash isn't in the cache, call a runtime handler to perform the
855       // hard work of checking whether the vptr is for an object of the right
856       // type. This will either fill in the cache and return, or produce a
857       // diagnostic.
858       llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash);
859       llvm::Constant *StaticData[] = {
860         EmitCheckSourceLocation(Loc),
861         EmitCheckTypeDescriptor(Ty),
862         CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()),
863         llvm::ConstantInt::get(Int8Ty, TCK)
864       };
865       llvm::Value *DynamicData[] = { Ptr, Hash };
866       EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
867                 SanitizerHandler::DynamicTypeCacheMiss, StaticData,
868                 DynamicData);
869     }
870   }
871 
872   if (Done) {
873     Builder.CreateBr(Done);
874     EmitBlock(Done);
875   }
876 }
877 
878 /// Determine whether this expression refers to a flexible array member in a
879 /// struct. We disable array bounds checks for such members.
880 static bool isFlexibleArrayMemberExpr(const Expr *E,
881                                       unsigned StrictFlexArraysLevel) {
882   // For compatibility with existing code, we treat arrays of length 0 or
883   // 1 as flexible array members.
884   // FIXME: This is inconsistent with the warning code in SemaChecking. Unify
885   // the two mechanisms.
886   const ArrayType *AT = E->getType()->castAsArrayTypeUnsafe();
887   if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) {
888     // FIXME: Sema doesn't treat [1] as a flexible array member if the bound
889     // was produced by macro expansion.
890     if (StrictFlexArraysLevel >= 2 && CAT->getSize().ugt(0))
891       return false;
892     // FIXME: While the default -fstrict-flex-arrays=0 permits Size>1 trailing
893     // arrays to be treated as flexible-array-members, we still emit ubsan
894     // checks as if they are not.
895     if (CAT->getSize().ugt(1))
896       return false;
897   } else if (!isa<IncompleteArrayType>(AT))
898     return false;
899 
900   E = E->IgnoreParens();
901 
902   // A flexible array member must be the last member in the class.
903   if (const auto *ME = dyn_cast<MemberExpr>(E)) {
904     // FIXME: If the base type of the member expr is not FD->getParent(),
905     // this should not be treated as a flexible array member access.
906     if (const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
907       // FIXME: Sema doesn't treat a T[1] union member as a flexible array
908       // member, only a T[0] or T[] member gets that treatment.
909       // Under StrictFlexArraysLevel, obey c99+ that disallows FAM in union, see
910       // C11 6.7.2.1 §18
911       if (FD->getParent()->isUnion())
912         return StrictFlexArraysLevel < 2;
913       RecordDecl::field_iterator FI(
914           DeclContext::decl_iterator(const_cast<FieldDecl *>(FD)));
915       return ++FI == FD->getParent()->field_end();
916     }
917   } else if (const auto *IRE = dyn_cast<ObjCIvarRefExpr>(E)) {
918     return IRE->getDecl()->getNextIvar() == nullptr;
919   }
920 
921   return false;
922 }
923 
924 llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
925                                                    QualType EltTy) {
926   ASTContext &C = getContext();
927   uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity();
928   if (!EltSize)
929     return nullptr;
930 
931   auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts());
932   if (!ArrayDeclRef)
933     return nullptr;
934 
935   auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());
936   if (!ParamDecl)
937     return nullptr;
938 
939   auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
940   if (!POSAttr)
941     return nullptr;
942 
943   // Don't load the size if it's a lower bound.
944   int POSType = POSAttr->getType();
945   if (POSType != 0 && POSType != 1)
946     return nullptr;
947 
948   // Find the implicit size parameter.
949   auto PassedSizeIt = SizeArguments.find(ParamDecl);
950   if (PassedSizeIt == SizeArguments.end())
951     return nullptr;
952 
953   const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
954   assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
955   Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
956   llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false,
957                                               C.getSizeType(), E->getExprLoc());
958   llvm::Value *SizeOfElement =
959       llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);
960   return Builder.CreateUDiv(SizeInBytes, SizeOfElement);
961 }
962 
963 /// If Base is known to point to the start of an array, return the length of
964 /// that array. Return 0 if the length cannot be determined.
965 static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
966                                           const Expr *Base,
967                                           QualType &IndexedType,
968                                           unsigned StrictFlexArraysLevel) {
969   // For the vector indexing extension, the bound is the number of elements.
970   if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
971     IndexedType = Base->getType();
972     return CGF.Builder.getInt32(VT->getNumElements());
973   }
974 
975   Base = Base->IgnoreParens();
976 
977   if (const auto *CE = dyn_cast<CastExpr>(Base)) {
978     if (CE->getCastKind() == CK_ArrayToPointerDecay &&
979         !isFlexibleArrayMemberExpr(CE->getSubExpr(), StrictFlexArraysLevel)) {
980       IndexedType = CE->getSubExpr()->getType();
981       const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
982       if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
983         return CGF.Builder.getInt(CAT->getSize());
984       else if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
985         return CGF.getVLASize(VAT).NumElts;
986       // Ignore pass_object_size here. It's not applicable on decayed pointers.
987     }
988   }
989 
990   QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
991   if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {
992     IndexedType = Base->getType();
993     return POS;
994   }
995 
996   return nullptr;
997 }
998 
999 void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
1000                                       llvm::Value *Index, QualType IndexType,
1001                                       bool Accessed) {
1002   assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
1003          "should not be called unless adding bounds checks");
1004   SanitizerScope SanScope(this);
1005 
1006   const unsigned StrictFlexArraysLevel = getLangOpts().StrictFlexArrays;
1007 
1008   QualType IndexedType;
1009   llvm::Value *Bound =
1010       getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel);
1011   if (!Bound)
1012     return;
1013 
1014   bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
1015   llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
1016   llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);
1017 
1018   llvm::Constant *StaticData[] = {
1019     EmitCheckSourceLocation(E->getExprLoc()),
1020     EmitCheckTypeDescriptor(IndexedType),
1021     EmitCheckTypeDescriptor(IndexType)
1022   };
1023   llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal)
1024                                 : Builder.CreateICmpULE(IndexVal, BoundVal);
1025   EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds),
1026             SanitizerHandler::OutOfBounds, StaticData, Index);
1027 }
1028 
1029 
1030 CodeGenFunction::ComplexPairTy CodeGenFunction::
1031 EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
1032                          bool isInc, bool isPre) {
1033   ComplexPairTy InVal = EmitLoadOfComplex(LV, E->getExprLoc());
1034 
1035   llvm::Value *NextVal;
1036   if (isa<llvm::IntegerType>(InVal.first->getType())) {
1037     uint64_t AmountVal = isInc ? 1 : -1;
1038     NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
1039 
1040     // Add the inc/dec to the real part.
1041     NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1042   } else {
1043     QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1044     llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
1045     if (!isInc)
1046       FVal.changeSign();
1047     NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
1048 
1049     // Add the inc/dec to the real part.
1050     NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1051   }
1052 
1053   ComplexPairTy IncVal(NextVal, InVal.second);
1054 
1055   // Store the updated result through the lvalue.
1056   EmitStoreOfComplex(IncVal, LV, /*init*/ false);
1057   if (getLangOpts().OpenMP)
1058     CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
1059                                                               E->getSubExpr());
1060 
1061   // If this is a postinc, return the value read from memory, otherwise use the
1062   // updated value.
1063   return isPre ? IncVal : InVal;
1064 }
1065 
1066 void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1067                                              CodeGenFunction *CGF) {
1068   // Bind VLAs in the cast type.
1069   if (CGF && E->getType()->isVariablyModifiedType())
1070     CGF->EmitVariablyModifiedType(E->getType());
1071 
1072   if (CGDebugInfo *DI = getModuleDebugInfo())
1073     DI->EmitExplicitCastType(E->getType());
1074 }
1075 
1076 //===----------------------------------------------------------------------===//
1077 //                         LValue Expression Emission
1078 //===----------------------------------------------------------------------===//
1079 
1080 /// EmitPointerWithAlignment - Given an expression of pointer type, try to
1081 /// derive a more accurate bound on the alignment of the pointer.
1082 Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E,
1083                                                   LValueBaseInfo *BaseInfo,
1084                                                   TBAAAccessInfo *TBAAInfo) {
1085   // We allow this with ObjC object pointers because of fragile ABIs.
1086   assert(E->getType()->isPointerType() ||
1087          E->getType()->isObjCObjectPointerType());
1088   E = E->IgnoreParens();
1089 
1090   // Casts:
1091   if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
1092     if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
1093       CGM.EmitExplicitCastExprType(ECE, this);
1094 
1095     switch (CE->getCastKind()) {
1096     // Non-converting casts (but not C's implicit conversion from void*).
1097     case CK_BitCast:
1098     case CK_NoOp:
1099     case CK_AddressSpaceConversion:
1100       if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1101         if (PtrTy->getPointeeType()->isVoidType())
1102           break;
1103 
1104         LValueBaseInfo InnerBaseInfo;
1105         TBAAAccessInfo InnerTBAAInfo;
1106         Address Addr = EmitPointerWithAlignment(CE->getSubExpr(),
1107                                                 &InnerBaseInfo,
1108                                                 &InnerTBAAInfo);
1109         if (BaseInfo) *BaseInfo = InnerBaseInfo;
1110         if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1111 
1112         if (isa<ExplicitCastExpr>(CE)) {
1113           LValueBaseInfo TargetTypeBaseInfo;
1114           TBAAAccessInfo TargetTypeTBAAInfo;
1115           CharUnits Align = CGM.getNaturalPointeeTypeAlignment(
1116               E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);
1117           if (TBAAInfo)
1118             *TBAAInfo = CGM.mergeTBAAInfoForCast(*TBAAInfo,
1119                                                  TargetTypeTBAAInfo);
1120           // If the source l-value is opaque, honor the alignment of the
1121           // casted-to type.
1122           if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1123             if (BaseInfo)
1124               BaseInfo->mergeForCast(TargetTypeBaseInfo);
1125             Addr = Address(Addr.getPointer(), Addr.getElementType(), Align);
1126           }
1127         }
1128 
1129         if (SanOpts.has(SanitizerKind::CFIUnrelatedCast) &&
1130             CE->getCastKind() == CK_BitCast) {
1131           if (auto PT = E->getType()->getAs<PointerType>())
1132             EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr,
1133                                       /*MayBeNull=*/true,
1134                                       CodeGenFunction::CFITCK_UnrelatedCast,
1135                                       CE->getBeginLoc());
1136         }
1137 
1138         llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType());
1139         Addr = Builder.CreateElementBitCast(Addr, ElemTy);
1140         if (CE->getCastKind() == CK_AddressSpaceConversion)
1141           Addr = Builder.CreateAddrSpaceCast(Addr, ConvertType(E->getType()));
1142         return Addr;
1143       }
1144       break;
1145 
1146     // Array-to-pointer decay.
1147     case CK_ArrayToPointerDecay:
1148       return EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo);
1149 
1150     // Derived-to-base conversions.
1151     case CK_UncheckedDerivedToBase:
1152     case CK_DerivedToBase: {
1153       // TODO: Support accesses to members of base classes in TBAA. For now, we
1154       // conservatively pretend that the complete object is of the base class
1155       // type.
1156       if (TBAAInfo)
1157         *TBAAInfo = CGM.getTBAAAccessInfo(E->getType());
1158       Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), BaseInfo);
1159       auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1160       return GetAddressOfBaseClass(Addr, Derived,
1161                                    CE->path_begin(), CE->path_end(),
1162                                    ShouldNullCheckClassCastValue(CE),
1163                                    CE->getExprLoc());
1164     }
1165 
1166     // TODO: Is there any reason to treat base-to-derived conversions
1167     // specially?
1168     default:
1169       break;
1170     }
1171   }
1172 
1173   // Unary &.
1174   if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
1175     if (UO->getOpcode() == UO_AddrOf) {
1176       LValue LV = EmitLValue(UO->getSubExpr());
1177       if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1178       if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1179       return LV.getAddress(*this);
1180     }
1181   }
1182 
1183   // std::addressof and variants.
1184   if (auto *Call = dyn_cast<CallExpr>(E)) {
1185     switch (Call->getBuiltinCallee()) {
1186     default:
1187       break;
1188     case Builtin::BIaddressof:
1189     case Builtin::BI__addressof:
1190     case Builtin::BI__builtin_addressof: {
1191       LValue LV = EmitLValue(Call->getArg(0));
1192       if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1193       if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1194       return LV.getAddress(*this);
1195     }
1196     }
1197   }
1198 
1199   // TODO: conditional operators, comma.
1200 
1201   // Otherwise, use the alignment of the type.
1202   CharUnits Align =
1203       CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
1204   llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType());
1205   return Address(EmitScalarExpr(E), ElemTy, Align);
1206 }
1207 
1208 llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1209   llvm::Value *V = RV.getScalarVal();
1210   if (auto MPT = T->getAs<MemberPointerType>())
1211     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, V, MPT);
1212   return Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
1213 }
1214 
1215 RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1216   if (Ty->isVoidType())
1217     return RValue::get(nullptr);
1218 
1219   switch (getEvaluationKind(Ty)) {
1220   case TEK_Complex: {
1221     llvm::Type *EltTy =
1222       ConvertType(Ty->castAs<ComplexType>()->getElementType());
1223     llvm::Value *U = llvm::UndefValue::get(EltTy);
1224     return RValue::getComplex(std::make_pair(U, U));
1225   }
1226 
1227   // If this is a use of an undefined aggregate type, the aggregate must have an
1228   // identifiable address.  Just because the contents of the value are undefined
1229   // doesn't mean that the address can't be taken and compared.
1230   case TEK_Aggregate: {
1231     Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
1232     return RValue::getAggregate(DestPtr);
1233   }
1234 
1235   case TEK_Scalar:
1236     return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
1237   }
1238   llvm_unreachable("bad evaluation kind");
1239 }
1240 
1241 RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1242                                               const char *Name) {
1243   ErrorUnsupported(E, Name);
1244   return GetUndefRValue(E->getType());
1245 }
1246 
1247 LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1248                                               const char *Name) {
1249   ErrorUnsupported(E, Name);
1250   llvm::Type *ElTy = ConvertType(E->getType());
1251   llvm::Type *Ty = llvm::PointerType::getUnqual(ElTy);
1252   return MakeAddrLValue(
1253       Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType());
1254 }
1255 
1256 bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1257   const Expr *Base = Obj;
1258   while (!isa<CXXThisExpr>(Base)) {
1259     // The result of a dynamic_cast can be null.
1260     if (isa<CXXDynamicCastExpr>(Base))
1261       return false;
1262 
1263     if (const auto *CE = dyn_cast<CastExpr>(Base)) {
1264       Base = CE->getSubExpr();
1265     } else if (const auto *PE = dyn_cast<ParenExpr>(Base)) {
1266       Base = PE->getSubExpr();
1267     } else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) {
1268       if (UO->getOpcode() == UO_Extension)
1269         Base = UO->getSubExpr();
1270       else
1271         return false;
1272     } else {
1273       return false;
1274     }
1275   }
1276   return true;
1277 }
1278 
1279 LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1280   LValue LV;
1281   if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E))
1282     LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true);
1283   else
1284     LV = EmitLValue(E);
1285   if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) {
1286     SanitizerSet SkippedChecks;
1287     if (const auto *ME = dyn_cast<MemberExpr>(E)) {
1288       bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase());
1289       if (IsBaseCXXThis)
1290         SkippedChecks.set(SanitizerKind::Alignment, true);
1291       if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
1292         SkippedChecks.set(SanitizerKind::Null, true);
1293     }
1294     EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(),
1295                   LV.getAlignment(), SkippedChecks);
1296   }
1297   return LV;
1298 }
1299 
1300 /// EmitLValue - Emit code to compute a designator that specifies the location
1301 /// of the expression.
1302 ///
1303 /// This can return one of two things: a simple address or a bitfield reference.
1304 /// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1305 /// an LLVM pointer type.
1306 ///
1307 /// If this returns a bitfield reference, nothing about the pointee type of the
1308 /// LLVM value is known: For example, it may not be a pointer to an integer.
1309 ///
1310 /// If this returns a normal address, and if the lvalue's C type is fixed size,
1311 /// this method guarantees that the returned pointer type will point to an LLVM
1312 /// type of the same size of the lvalue's type.  If the lvalue has a variable
1313 /// length type, this is not possible.
1314 ///
1315 LValue CodeGenFunction::EmitLValue(const Expr *E) {
1316   ApplyDebugLocation DL(*this, E);
1317   switch (E->getStmtClass()) {
1318   default: return EmitUnsupportedLValue(E, "l-value expression");
1319 
1320   case Expr::ObjCPropertyRefExprClass:
1321     llvm_unreachable("cannot emit a property reference directly");
1322 
1323   case Expr::ObjCSelectorExprClass:
1324     return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
1325   case Expr::ObjCIsaExprClass:
1326     return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
1327   case Expr::BinaryOperatorClass:
1328     return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
1329   case Expr::CompoundAssignOperatorClass: {
1330     QualType Ty = E->getType();
1331     if (const AtomicType *AT = Ty->getAs<AtomicType>())
1332       Ty = AT->getValueType();
1333     if (!Ty->isAnyComplexType())
1334       return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1335     return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1336   }
1337   case Expr::CallExprClass:
1338   case Expr::CXXMemberCallExprClass:
1339   case Expr::CXXOperatorCallExprClass:
1340   case Expr::UserDefinedLiteralClass:
1341     return EmitCallExprLValue(cast<CallExpr>(E));
1342   case Expr::CXXRewrittenBinaryOperatorClass:
1343     return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm());
1344   case Expr::VAArgExprClass:
1345     return EmitVAArgExprLValue(cast<VAArgExpr>(E));
1346   case Expr::DeclRefExprClass:
1347     return EmitDeclRefLValue(cast<DeclRefExpr>(E));
1348   case Expr::ConstantExprClass: {
1349     const ConstantExpr *CE = cast<ConstantExpr>(E);
1350     if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
1351       QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit())
1352                              ->getCallReturnType(getContext())
1353                              ->getPointeeType();
1354       return MakeNaturalAlignAddrLValue(Result, RetType);
1355     }
1356     return EmitLValue(cast<ConstantExpr>(E)->getSubExpr());
1357   }
1358   case Expr::ParenExprClass:
1359     return EmitLValue(cast<ParenExpr>(E)->getSubExpr());
1360   case Expr::GenericSelectionExprClass:
1361     return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr());
1362   case Expr::PredefinedExprClass:
1363     return EmitPredefinedLValue(cast<PredefinedExpr>(E));
1364   case Expr::StringLiteralClass:
1365     return EmitStringLiteralLValue(cast<StringLiteral>(E));
1366   case Expr::ObjCEncodeExprClass:
1367     return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
1368   case Expr::PseudoObjectExprClass:
1369     return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
1370   case Expr::InitListExprClass:
1371     return EmitInitListLValue(cast<InitListExpr>(E));
1372   case Expr::CXXTemporaryObjectExprClass:
1373   case Expr::CXXConstructExprClass:
1374     return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
1375   case Expr::CXXBindTemporaryExprClass:
1376     return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
1377   case Expr::CXXUuidofExprClass:
1378     return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));
1379   case Expr::LambdaExprClass:
1380     return EmitAggExprToLValue(E);
1381 
1382   case Expr::ExprWithCleanupsClass: {
1383     const auto *cleanups = cast<ExprWithCleanups>(E);
1384     RunCleanupsScope Scope(*this);
1385     LValue LV = EmitLValue(cleanups->getSubExpr());
1386     if (LV.isSimple()) {
1387       // Defend against branches out of gnu statement expressions surrounded by
1388       // cleanups.
1389       Address Addr = LV.getAddress(*this);
1390       llvm::Value *V = Addr.getPointer();
1391       Scope.ForceCleanup({&V});
1392       return LValue::MakeAddr(Addr.withPointer(V), LV.getType(), getContext(),
1393                               LV.getBaseInfo(), LV.getTBAAInfo());
1394     }
1395     // FIXME: Is it possible to create an ExprWithCleanups that produces a
1396     // bitfield lvalue or some other non-simple lvalue?
1397     return LV;
1398   }
1399 
1400   case Expr::CXXDefaultArgExprClass: {
1401     auto *DAE = cast<CXXDefaultArgExpr>(E);
1402     CXXDefaultArgExprScope Scope(*this, DAE);
1403     return EmitLValue(DAE->getExpr());
1404   }
1405   case Expr::CXXDefaultInitExprClass: {
1406     auto *DIE = cast<CXXDefaultInitExpr>(E);
1407     CXXDefaultInitExprScope Scope(*this, DIE);
1408     return EmitLValue(DIE->getExpr());
1409   }
1410   case Expr::CXXTypeidExprClass:
1411     return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
1412 
1413   case Expr::ObjCMessageExprClass:
1414     return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
1415   case Expr::ObjCIvarRefExprClass:
1416     return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
1417   case Expr::StmtExprClass:
1418     return EmitStmtExprLValue(cast<StmtExpr>(E));
1419   case Expr::UnaryOperatorClass:
1420     return EmitUnaryOpLValue(cast<UnaryOperator>(E));
1421   case Expr::ArraySubscriptExprClass:
1422     return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
1423   case Expr::MatrixSubscriptExprClass:
1424     return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E));
1425   case Expr::OMPArraySectionExprClass:
1426     return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E));
1427   case Expr::ExtVectorElementExprClass:
1428     return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
1429   case Expr::MemberExprClass:
1430     return EmitMemberExpr(cast<MemberExpr>(E));
1431   case Expr::CompoundLiteralExprClass:
1432     return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
1433   case Expr::ConditionalOperatorClass:
1434     return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
1435   case Expr::BinaryConditionalOperatorClass:
1436     return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
1437   case Expr::ChooseExprClass:
1438     return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr());
1439   case Expr::OpaqueValueExprClass:
1440     return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
1441   case Expr::SubstNonTypeTemplateParmExprClass:
1442     return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement());
1443   case Expr::ImplicitCastExprClass:
1444   case Expr::CStyleCastExprClass:
1445   case Expr::CXXFunctionalCastExprClass:
1446   case Expr::CXXStaticCastExprClass:
1447   case Expr::CXXDynamicCastExprClass:
1448   case Expr::CXXReinterpretCastExprClass:
1449   case Expr::CXXConstCastExprClass:
1450   case Expr::CXXAddrspaceCastExprClass:
1451   case Expr::ObjCBridgedCastExprClass:
1452     return EmitCastLValue(cast<CastExpr>(E));
1453 
1454   case Expr::MaterializeTemporaryExprClass:
1455     return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
1456 
1457   case Expr::CoawaitExprClass:
1458     return EmitCoawaitLValue(cast<CoawaitExpr>(E));
1459   case Expr::CoyieldExprClass:
1460     return EmitCoyieldLValue(cast<CoyieldExpr>(E));
1461   }
1462 }
1463 
1464 /// Given an object of the given canonical type, can we safely copy a
1465 /// value out of it based on its initializer?
1466 static bool isConstantEmittableObjectType(QualType type) {
1467   assert(type.isCanonical());
1468   assert(!type->isReferenceType());
1469 
1470   // Must be const-qualified but non-volatile.
1471   Qualifiers qs = type.getLocalQualifiers();
1472   if (!qs.hasConst() || qs.hasVolatile()) return false;
1473 
1474   // Otherwise, all object types satisfy this except C++ classes with
1475   // mutable subobjects or non-trivial copy/destroy behavior.
1476   if (const auto *RT = dyn_cast<RecordType>(type))
1477     if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1478       if (RD->hasMutableFields() || !RD->isTrivial())
1479         return false;
1480 
1481   return true;
1482 }
1483 
1484 /// Can we constant-emit a load of a reference to a variable of the
1485 /// given type?  This is different from predicates like
1486 /// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1487 /// in situations that don't necessarily satisfy the language's rules
1488 /// for this (e.g. C++'s ODR-use rules).  For example, we want to able
1489 /// to do this with const float variables even if those variables
1490 /// aren't marked 'constexpr'.
1491 enum ConstantEmissionKind {
1492   CEK_None,
1493   CEK_AsReferenceOnly,
1494   CEK_AsValueOrReference,
1495   CEK_AsValueOnly
1496 };
1497 static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1498   type = type.getCanonicalType();
1499   if (const auto *ref = dyn_cast<ReferenceType>(type)) {
1500     if (isConstantEmittableObjectType(ref->getPointeeType()))
1501       return CEK_AsValueOrReference;
1502     return CEK_AsReferenceOnly;
1503   }
1504   if (isConstantEmittableObjectType(type))
1505     return CEK_AsValueOnly;
1506   return CEK_None;
1507 }
1508 
1509 /// Try to emit a reference to the given value without producing it as
1510 /// an l-value.  This is just an optimization, but it avoids us needing
1511 /// to emit global copies of variables if they're named without triggering
1512 /// a formal use in a context where we can't emit a direct reference to them,
1513 /// for instance if a block or lambda or a member of a local class uses a
1514 /// const int variable or constexpr variable from an enclosing function.
1515 CodeGenFunction::ConstantEmission
1516 CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
1517   ValueDecl *value = refExpr->getDecl();
1518 
1519   // The value needs to be an enum constant or a constant variable.
1520   ConstantEmissionKind CEK;
1521   if (isa<ParmVarDecl>(value)) {
1522     CEK = CEK_None;
1523   } else if (auto *var = dyn_cast<VarDecl>(value)) {
1524     CEK = checkVarTypeForConstantEmission(var->getType());
1525   } else if (isa<EnumConstantDecl>(value)) {
1526     CEK = CEK_AsValueOnly;
1527   } else {
1528     CEK = CEK_None;
1529   }
1530   if (CEK == CEK_None) return ConstantEmission();
1531 
1532   Expr::EvalResult result;
1533   bool resultIsReference;
1534   QualType resultType;
1535 
1536   // It's best to evaluate all the way as an r-value if that's permitted.
1537   if (CEK != CEK_AsReferenceOnly &&
1538       refExpr->EvaluateAsRValue(result, getContext())) {
1539     resultIsReference = false;
1540     resultType = refExpr->getType();
1541 
1542   // Otherwise, try to evaluate as an l-value.
1543   } else if (CEK != CEK_AsValueOnly &&
1544              refExpr->EvaluateAsLValue(result, getContext())) {
1545     resultIsReference = true;
1546     resultType = value->getType();
1547 
1548   // Failure.
1549   } else {
1550     return ConstantEmission();
1551   }
1552 
1553   // In any case, if the initializer has side-effects, abandon ship.
1554   if (result.HasSideEffects)
1555     return ConstantEmission();
1556 
1557   // In CUDA/HIP device compilation, a lambda may capture a reference variable
1558   // referencing a global host variable by copy. In this case the lambda should
1559   // make a copy of the value of the global host variable. The DRE of the
1560   // captured reference variable cannot be emitted as load from the host
1561   // global variable as compile time constant, since the host variable is not
1562   // accessible on device. The DRE of the captured reference variable has to be
1563   // loaded from captures.
1564   if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
1565       refExpr->refersToEnclosingVariableOrCapture()) {
1566     auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
1567     if (MD && MD->getParent()->isLambda() &&
1568         MD->getOverloadedOperator() == OO_Call) {
1569       const APValue::LValueBase &base = result.Val.getLValueBase();
1570       if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
1571         if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
1572           if (!VD->hasAttr<CUDADeviceAttr>()) {
1573             return ConstantEmission();
1574           }
1575         }
1576       }
1577     }
1578   }
1579 
1580   // Emit as a constant.
1581   auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(),
1582                                                result.Val, resultType);
1583 
1584   // Make sure we emit a debug reference to the global variable.
1585   // This should probably fire even for
1586   if (isa<VarDecl>(value)) {
1587     if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
1588       EmitDeclRefExprDbgValue(refExpr, result.Val);
1589   } else {
1590     assert(isa<EnumConstantDecl>(value));
1591     EmitDeclRefExprDbgValue(refExpr, result.Val);
1592   }
1593 
1594   // If we emitted a reference constant, we need to dereference that.
1595   if (resultIsReference)
1596     return ConstantEmission::forReference(C);
1597 
1598   return ConstantEmission::forValue(C);
1599 }
1600 
1601 static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
1602                                                         const MemberExpr *ME) {
1603   if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) {
1604     // Try to emit static variable member expressions as DREs.
1605     return DeclRefExpr::Create(
1606         CGF.getContext(), NestedNameSpecifierLoc(), SourceLocation(), VD,
1607         /*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(),
1608         ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse());
1609   }
1610   return nullptr;
1611 }
1612 
1613 CodeGenFunction::ConstantEmission
1614 CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
1615   if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, ME))
1616     return tryEmitAsConstant(DRE);
1617   return ConstantEmission();
1618 }
1619 
1620 llvm::Value *CodeGenFunction::emitScalarConstant(
1621     const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
1622   assert(Constant && "not a constant");
1623   if (Constant.isReference())
1624     return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E),
1625                             E->getExprLoc())
1626         .getScalarVal();
1627   return Constant.getValue();
1628 }
1629 
1630 llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
1631                                                SourceLocation Loc) {
1632   return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(),
1633                           lvalue.getType(), Loc, lvalue.getBaseInfo(),
1634                           lvalue.getTBAAInfo(), lvalue.isNontemporal());
1635 }
1636 
1637 static bool hasBooleanRepresentation(QualType Ty) {
1638   if (Ty->isBooleanType())
1639     return true;
1640 
1641   if (const EnumType *ET = Ty->getAs<EnumType>())
1642     return ET->getDecl()->getIntegerType()->isBooleanType();
1643 
1644   if (const AtomicType *AT = Ty->getAs<AtomicType>())
1645     return hasBooleanRepresentation(AT->getValueType());
1646 
1647   return false;
1648 }
1649 
1650 static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
1651                             llvm::APInt &Min, llvm::APInt &End,
1652                             bool StrictEnums, bool IsBool) {
1653   const EnumType *ET = Ty->getAs<EnumType>();
1654   bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
1655                                 ET && !ET->getDecl()->isFixed();
1656   if (!IsBool && !IsRegularCPlusPlusEnum)
1657     return false;
1658 
1659   if (IsBool) {
1660     Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);
1661     End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);
1662   } else {
1663     const EnumDecl *ED = ET->getDecl();
1664     llvm::Type *LTy = CGF.ConvertTypeForMem(ED->getIntegerType());
1665     unsigned Bitwidth = LTy->getScalarSizeInBits();
1666     unsigned NumNegativeBits = ED->getNumNegativeBits();
1667     unsigned NumPositiveBits = ED->getNumPositiveBits();
1668 
1669     if (NumNegativeBits) {
1670       unsigned NumBits = std::max(NumNegativeBits, NumPositiveBits + 1);
1671       assert(NumBits <= Bitwidth);
1672       End = llvm::APInt(Bitwidth, 1) << (NumBits - 1);
1673       Min = -End;
1674     } else {
1675       assert(NumPositiveBits <= Bitwidth);
1676       End = llvm::APInt(Bitwidth, 1) << NumPositiveBits;
1677       Min = llvm::APInt::getZero(Bitwidth);
1678     }
1679   }
1680   return true;
1681 }
1682 
1683 llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
1684   llvm::APInt Min, End;
1685   if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums,
1686                        hasBooleanRepresentation(Ty)))
1687     return nullptr;
1688 
1689   llvm::MDBuilder MDHelper(getLLVMContext());
1690   return MDHelper.createRange(Min, End);
1691 }
1692 
1693 bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
1694                                            SourceLocation Loc) {
1695   bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool);
1696   bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum);
1697   if (!HasBoolCheck && !HasEnumCheck)
1698     return false;
1699 
1700   bool IsBool = hasBooleanRepresentation(Ty) ||
1701                 NSAPI(CGM.getContext()).isObjCBOOLType(Ty);
1702   bool NeedsBoolCheck = HasBoolCheck && IsBool;
1703   bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>();
1704   if (!NeedsBoolCheck && !NeedsEnumCheck)
1705     return false;
1706 
1707   // Single-bit booleans don't need to be checked. Special-case this to avoid
1708   // a bit width mismatch when handling bitfield values. This is handled by
1709   // EmitFromMemory for the non-bitfield case.
1710   if (IsBool &&
1711       cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1)
1712     return false;
1713 
1714   llvm::APInt Min, End;
1715   if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
1716     return true;
1717 
1718   auto &Ctx = getLLVMContext();
1719   SanitizerScope SanScope(this);
1720   llvm::Value *Check;
1721   --End;
1722   if (!Min) {
1723     Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End));
1724   } else {
1725     llvm::Value *Upper =
1726         Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End));
1727     llvm::Value *Lower =
1728         Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min));
1729     Check = Builder.CreateAnd(Upper, Lower);
1730   }
1731   llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
1732                                   EmitCheckTypeDescriptor(Ty)};
1733   SanitizerMask Kind =
1734       NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1735   EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
1736             StaticArgs, EmitCheckValue(Value));
1737   return true;
1738 }
1739 
1740 llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
1741                                                QualType Ty,
1742                                                SourceLocation Loc,
1743                                                LValueBaseInfo BaseInfo,
1744                                                TBAAAccessInfo TBAAInfo,
1745                                                bool isNontemporal) {
1746   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1747     // Boolean vectors use `iN` as storage type.
1748     if (ClangVecTy->isExtVectorBoolType()) {
1749       llvm::Type *ValTy = ConvertType(Ty);
1750       unsigned ValNumElems =
1751           cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1752       // Load the `iP` storage object (P is the padded vector size).
1753       auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits");
1754       const auto *RawIntTy = RawIntV->getType();
1755       assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
1756       // Bitcast iP --> <P x i1>.
1757       auto *PaddedVecTy = llvm::FixedVectorType::get(
1758           Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1759       llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy);
1760       // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1761       V = emitBoolVecConversion(V, ValNumElems, "extractvec");
1762 
1763       return EmitFromMemory(V, Ty);
1764     }
1765 
1766     // Handle vectors of size 3 like size 4 for better performance.
1767     const llvm::Type *EltTy = Addr.getElementType();
1768     const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
1769 
1770     if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) {
1771 
1772       // Bitcast to vec4 type.
1773       llvm::VectorType *vec4Ty =
1774           llvm::FixedVectorType::get(VTy->getElementType(), 4);
1775       Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4");
1776       // Now load value.
1777       llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
1778 
1779       // Shuffle vector to get vec3.
1780       V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec");
1781       return EmitFromMemory(V, Ty);
1782     }
1783   }
1784 
1785   // Atomic operations have to be done on integral types.
1786   LValue AtomicLValue =
1787       LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1788   if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) {
1789     return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();
1790   }
1791 
1792   llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
1793   if (isNontemporal) {
1794     llvm::MDNode *Node = llvm::MDNode::get(
1795         Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1796     Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
1797   }
1798 
1799   CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
1800 
1801   if (EmitScalarRangeCheck(Load, Ty, Loc)) {
1802     // In order to prevent the optimizer from throwing away the check, don't
1803     // attach range metadata to the load.
1804   } else if (CGM.getCodeGenOpts().OptimizationLevel > 0)
1805     if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty))
1806       Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
1807 
1808   return EmitFromMemory(Load, Ty);
1809 }
1810 
1811 llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
1812   // Bool has a different representation in memory than in registers.
1813   if (hasBooleanRepresentation(Ty)) {
1814     // This should really always be an i1, but sometimes it's already
1815     // an i8, and it's awkward to track those cases down.
1816     if (Value->getType()->isIntegerTy(1))
1817       return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool");
1818     assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1819            "wrong value rep of bool");
1820   }
1821 
1822   return Value;
1823 }
1824 
1825 llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
1826   // Bool has a different representation in memory than in registers.
1827   if (hasBooleanRepresentation(Ty)) {
1828     assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1829            "wrong value rep of bool");
1830     return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
1831   }
1832   if (Ty->isExtVectorBoolType()) {
1833     const auto *RawIntTy = Value->getType();
1834     // Bitcast iP --> <P x i1>.
1835     auto *PaddedVecTy = llvm::FixedVectorType::get(
1836         Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1837     auto *V = Builder.CreateBitCast(Value, PaddedVecTy);
1838     // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1839     llvm::Type *ValTy = ConvertType(Ty);
1840     unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1841     return emitBoolVecConversion(V, ValNumElems, "extractvec");
1842   }
1843 
1844   return Value;
1845 }
1846 
1847 // Convert the pointer of \p Addr to a pointer to a vector (the value type of
1848 // MatrixType), if it points to a array (the memory type of MatrixType).
1849 static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF,
1850                                          bool IsVector = true) {
1851   auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
1852   if (ArrayTy && IsVector) {
1853     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
1854                                                 ArrayTy->getNumElements());
1855 
1856     return Address(CGF.Builder.CreateElementBitCast(Addr, VectorTy));
1857   }
1858   auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType());
1859   if (VectorTy && !IsVector) {
1860     auto *ArrayTy = llvm::ArrayType::get(
1861         VectorTy->getElementType(),
1862         cast<llvm::FixedVectorType>(VectorTy)->getNumElements());
1863 
1864     return Address(CGF.Builder.CreateElementBitCast(Addr, ArrayTy));
1865   }
1866 
1867   return Addr;
1868 }
1869 
1870 // Emit a store of a matrix LValue. This may require casting the original
1871 // pointer to memory address (ArrayType) to a pointer to the value type
1872 // (VectorType).
1873 static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
1874                                     bool isInit, CodeGenFunction &CGF) {
1875   Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF,
1876                                            value->getType()->isVectorTy());
1877   CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(),
1878                         lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit,
1879                         lvalue.isNontemporal());
1880 }
1881 
1882 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
1883                                         bool Volatile, QualType Ty,
1884                                         LValueBaseInfo BaseInfo,
1885                                         TBAAAccessInfo TBAAInfo,
1886                                         bool isInit, bool isNontemporal) {
1887   llvm::Type *SrcTy = Value->getType();
1888   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1889     auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
1890     if (VecTy && ClangVecTy->isExtVectorBoolType()) {
1891       auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType());
1892       // Expand to the memory bit width.
1893       unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
1894       // <N x i1> --> <P x i1>.
1895       Value = emitBoolVecConversion(Value, MemNumElems, "insertvec");
1896       // <P x i1> --> iP.
1897       Value = Builder.CreateBitCast(Value, MemIntTy);
1898     } else if (!CGM.getCodeGenOpts().PreserveVec3Type) {
1899       // Handle vec3 special.
1900       if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
1901         // Our source is a vec3, do a shuffle vector to make it a vec4.
1902         Value = Builder.CreateShuffleVector(Value, ArrayRef<int>{0, 1, 2, -1},
1903                                             "extractVec");
1904         SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4);
1905       }
1906       if (Addr.getElementType() != SrcTy) {
1907         Addr = Builder.CreateElementBitCast(Addr, SrcTy, "storetmp");
1908       }
1909     }
1910   }
1911 
1912   Value = EmitToMemory(Value, Ty);
1913 
1914   LValue AtomicLValue =
1915       LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1916   if (Ty->isAtomicType() ||
1917       (!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) {
1918     EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit);
1919     return;
1920   }
1921 
1922   llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
1923   if (isNontemporal) {
1924     llvm::MDNode *Node =
1925         llvm::MDNode::get(Store->getContext(),
1926                           llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1927     Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
1928   }
1929 
1930   CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
1931 }
1932 
1933 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
1934                                         bool isInit) {
1935   if (lvalue.getType()->isConstantMatrixType()) {
1936     EmitStoreOfMatrixScalar(value, lvalue, isInit, *this);
1937     return;
1938   }
1939 
1940   EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(),
1941                     lvalue.getType(), lvalue.getBaseInfo(),
1942                     lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal());
1943 }
1944 
1945 // Emit a load of a LValue of matrix type. This may require casting the pointer
1946 // to memory address (ArrayType) to a pointer to the value type (VectorType).
1947 static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
1948                                      CodeGenFunction &CGF) {
1949   assert(LV.getType()->isConstantMatrixType());
1950   Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF);
1951   LV.setAddress(Addr);
1952   return RValue::get(CGF.EmitLoadOfScalar(LV, Loc));
1953 }
1954 
1955 /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
1956 /// method emits the address of the lvalue, then loads the result as an rvalue,
1957 /// returning the rvalue.
1958 RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
1959   if (LV.isObjCWeak()) {
1960     // load of a __weak object.
1961     Address AddrWeakObj = LV.getAddress(*this);
1962     return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
1963                                                              AddrWeakObj));
1964   }
1965   if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
1966     // In MRC mode, we do a load+autorelease.
1967     if (!getLangOpts().ObjCAutoRefCount) {
1968       return RValue::get(EmitARCLoadWeak(LV.getAddress(*this)));
1969     }
1970 
1971     // In ARC mode, we load retained and then consume the value.
1972     llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this));
1973     Object = EmitObjCConsumeObject(LV.getType(), Object);
1974     return RValue::get(Object);
1975   }
1976 
1977   if (LV.isSimple()) {
1978     assert(!LV.getType()->isFunctionType());
1979 
1980     if (LV.getType()->isConstantMatrixType())
1981       return EmitLoadOfMatrixLValue(LV, Loc, *this);
1982 
1983     // Everything needs a load.
1984     return RValue::get(EmitLoadOfScalar(LV, Loc));
1985   }
1986 
1987   if (LV.isVectorElt()) {
1988     llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(),
1989                                               LV.isVolatileQualified());
1990     return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
1991                                                     "vecext"));
1992   }
1993 
1994   // If this is a reference to a subset of the elements of a vector, either
1995   // shuffle the input or extract/insert them as appropriate.
1996   if (LV.isExtVectorElt()) {
1997     return EmitLoadOfExtVectorElementLValue(LV);
1998   }
1999 
2000   // Global Register variables always invoke intrinsics
2001   if (LV.isGlobalReg())
2002     return EmitLoadOfGlobalRegLValue(LV);
2003 
2004   if (LV.isMatrixElt()) {
2005     llvm::Value *Idx = LV.getMatrixIdx();
2006     if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2007       const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();
2008       llvm::MatrixBuilder MB(Builder);
2009       MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2010     }
2011     llvm::LoadInst *Load =
2012         Builder.CreateLoad(LV.getMatrixAddress(), LV.isVolatileQualified());
2013     return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext"));
2014   }
2015 
2016   assert(LV.isBitField() && "Unknown LValue type!");
2017   return EmitLoadOfBitfieldLValue(LV, Loc);
2018 }
2019 
2020 RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
2021                                                  SourceLocation Loc) {
2022   const CGBitFieldInfo &Info = LV.getBitFieldInfo();
2023 
2024   // Get the output type.
2025   llvm::Type *ResLTy = ConvertType(LV.getType());
2026 
2027   Address Ptr = LV.getBitFieldAddress();
2028   llvm::Value *Val =
2029       Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load");
2030 
2031   bool UseVolatile = LV.isVolatileQualified() &&
2032                      Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2033   const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2034   const unsigned StorageSize =
2035       UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2036   if (Info.IsSigned) {
2037     assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
2038     unsigned HighBits = StorageSize - Offset - Info.Size;
2039     if (HighBits)
2040       Val = Builder.CreateShl(Val, HighBits, "bf.shl");
2041     if (Offset + HighBits)
2042       Val = Builder.CreateAShr(Val, Offset + HighBits, "bf.ashr");
2043   } else {
2044     if (Offset)
2045       Val = Builder.CreateLShr(Val, Offset, "bf.lshr");
2046     if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2047       Val = Builder.CreateAnd(
2048           Val, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), "bf.clear");
2049   }
2050   Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast");
2051   EmitScalarRangeCheck(Val, LV.getType(), Loc);
2052   return RValue::get(Val);
2053 }
2054 
2055 // If this is a reference to a subset of the elements of a vector, create an
2056 // appropriate shufflevector.
2057 RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2058   llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(),
2059                                         LV.isVolatileQualified());
2060 
2061   const llvm::Constant *Elts = LV.getExtVectorElts();
2062 
2063   // If the result of the expression is a non-vector type, we must be extracting
2064   // a single element.  Just codegen as an extractelement.
2065   const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2066   if (!ExprVT) {
2067     unsigned InIdx = getAccessedFieldNo(0, Elts);
2068     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2069     return RValue::get(Builder.CreateExtractElement(Vec, Elt));
2070   }
2071 
2072   // Always use shuffle vector to try to retain the original program structure
2073   unsigned NumResultElts = ExprVT->getNumElements();
2074 
2075   SmallVector<int, 4> Mask;
2076   for (unsigned i = 0; i != NumResultElts; ++i)
2077     Mask.push_back(getAccessedFieldNo(i, Elts));
2078 
2079   Vec = Builder.CreateShuffleVector(Vec, Mask);
2080   return RValue::get(Vec);
2081 }
2082 
2083 /// Generates lvalue for partial ext_vector access.
2084 Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2085   Address VectorAddress = LV.getExtVectorAddress();
2086   QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2087   llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT);
2088 
2089   Address CastToPointerElement =
2090     Builder.CreateElementBitCast(VectorAddress, VectorElementTy,
2091                                  "conv.ptr.element");
2092 
2093   const llvm::Constant *Elts = LV.getExtVectorElts();
2094   unsigned ix = getAccessedFieldNo(0, Elts);
2095 
2096   Address VectorBasePtrPlusIx =
2097     Builder.CreateConstInBoundsGEP(CastToPointerElement, ix,
2098                                    "vector.elt");
2099 
2100   return VectorBasePtrPlusIx;
2101 }
2102 
2103 /// Load of global gamed gegisters are always calls to intrinsics.
2104 RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2105   assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2106          "Bad type for register variable");
2107   llvm::MDNode *RegName = cast<llvm::MDNode>(
2108       cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata());
2109 
2110   // We accept integer and pointer types only
2111   llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType());
2112   llvm::Type *Ty = OrigTy;
2113   if (OrigTy->isPointerTy())
2114     Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2115   llvm::Type *Types[] = { Ty };
2116 
2117   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
2118   llvm::Value *Call = Builder.CreateCall(
2119       F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));
2120   if (OrigTy->isPointerTy())
2121     Call = Builder.CreateIntToPtr(Call, OrigTy);
2122   return RValue::get(Call);
2123 }
2124 
2125 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
2126 /// lvalue, where both are guaranteed to the have the same type, and that type
2127 /// is 'Ty'.
2128 void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2129                                              bool isInit) {
2130   if (!Dst.isSimple()) {
2131     if (Dst.isVectorElt()) {
2132       // Read/modify/write the vector, inserting the new element.
2133       llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
2134                                             Dst.isVolatileQualified());
2135       auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
2136       if (IRStoreTy) {
2137         auto *IRVecTy = llvm::FixedVectorType::get(
2138             Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
2139         Vec = Builder.CreateBitCast(Vec, IRVecTy);
2140         // iN --> <N x i1>.
2141       }
2142       Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
2143                                         Dst.getVectorIdx(), "vecins");
2144       if (IRStoreTy) {
2145         // <N x i1> --> <iN>.
2146         Vec = Builder.CreateBitCast(Vec, IRStoreTy);
2147       }
2148       Builder.CreateStore(Vec, Dst.getVectorAddress(),
2149                           Dst.isVolatileQualified());
2150       return;
2151     }
2152 
2153     // If this is an update of extended vector elements, insert them as
2154     // appropriate.
2155     if (Dst.isExtVectorElt())
2156       return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2157 
2158     if (Dst.isGlobalReg())
2159       return EmitStoreThroughGlobalRegLValue(Src, Dst);
2160 
2161     if (Dst.isMatrixElt()) {
2162       llvm::Value *Idx = Dst.getMatrixIdx();
2163       if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2164         const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2165         llvm::MatrixBuilder MB(Builder);
2166         MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2167       }
2168       llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress());
2169       llvm::Value *Vec =
2170           Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins");
2171       Builder.CreateStore(Vec, Dst.getMatrixAddress(),
2172                           Dst.isVolatileQualified());
2173       return;
2174     }
2175 
2176     assert(Dst.isBitField() && "Unknown LValue type");
2177     return EmitStoreThroughBitfieldLValue(Src, Dst);
2178   }
2179 
2180   // There's special magic for assigning into an ARC-qualified l-value.
2181   if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2182     switch (Lifetime) {
2183     case Qualifiers::OCL_None:
2184       llvm_unreachable("present but none");
2185 
2186     case Qualifiers::OCL_ExplicitNone:
2187       // nothing special
2188       break;
2189 
2190     case Qualifiers::OCL_Strong:
2191       if (isInit) {
2192         Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal()));
2193         break;
2194       }
2195       EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
2196       return;
2197 
2198     case Qualifiers::OCL_Weak:
2199       if (isInit)
2200         // Initialize and then skip the primitive store.
2201         EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal());
2202       else
2203         EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(),
2204                          /*ignore*/ true);
2205       return;
2206 
2207     case Qualifiers::OCL_Autoreleasing:
2208       Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),
2209                                                      Src.getScalarVal()));
2210       // fall into the normal path
2211       break;
2212     }
2213   }
2214 
2215   if (Dst.isObjCWeak() && !Dst.isNonGC()) {
2216     // load of a __weak object.
2217     Address LvalueDst = Dst.getAddress(*this);
2218     llvm::Value *src = Src.getScalarVal();
2219      CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
2220     return;
2221   }
2222 
2223   if (Dst.isObjCStrong() && !Dst.isNonGC()) {
2224     // load of a __strong object.
2225     Address LvalueDst = Dst.getAddress(*this);
2226     llvm::Value *src = Src.getScalarVal();
2227     if (Dst.isObjCIvar()) {
2228       assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
2229       llvm::Type *ResultType = IntPtrTy;
2230       Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp());
2231       llvm::Value *RHS = dst.getPointer();
2232       RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
2233       llvm::Value *LHS =
2234         Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
2235                                "sub.ptr.lhs.cast");
2236       llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
2237       CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
2238                                               BytesBetween);
2239     } else if (Dst.isGlobalObjCRef()) {
2240       CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
2241                                                 Dst.isThreadLocalRef());
2242     }
2243     else
2244       CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
2245     return;
2246   }
2247 
2248   assert(Src.isScalar() && "Can't emit an agg store with this method");
2249   EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
2250 }
2251 
2252 void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
2253                                                      llvm::Value **Result) {
2254   const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
2255   llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
2256   Address Ptr = Dst.getBitFieldAddress();
2257 
2258   // Get the source value, truncated to the width of the bit-field.
2259   llvm::Value *SrcVal = Src.getScalarVal();
2260 
2261   // Cast the source to the storage type and shift it into place.
2262   SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(),
2263                                  /*isSigned=*/false);
2264   llvm::Value *MaskedVal = SrcVal;
2265 
2266   const bool UseVolatile =
2267       CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
2268       Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2269   const unsigned StorageSize =
2270       UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2271   const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2272   // See if there are other bits in the bitfield's storage we'll need to load
2273   // and mask together with source before storing.
2274   if (StorageSize != Info.Size) {
2275     assert(StorageSize > Info.Size && "Invalid bitfield size.");
2276     llvm::Value *Val =
2277         Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load");
2278 
2279     // Mask the source value as needed.
2280     if (!hasBooleanRepresentation(Dst.getType()))
2281       SrcVal = Builder.CreateAnd(
2282           SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.Size),
2283           "bf.value");
2284     MaskedVal = SrcVal;
2285     if (Offset)
2286       SrcVal = Builder.CreateShl(SrcVal, Offset, "bf.shl");
2287 
2288     // Mask out the original value.
2289     Val = Builder.CreateAnd(
2290         Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.Size),
2291         "bf.clear");
2292 
2293     // Or together the unchanged values and the source value.
2294     SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set");
2295   } else {
2296     assert(Offset == 0);
2297     // According to the AACPS:
2298     // When a volatile bit-field is written, and its container does not overlap
2299     // with any non-bit-field member, its container must be read exactly once
2300     // and written exactly once using the access width appropriate to the type
2301     // of the container. The two accesses are not atomic.
2302     if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) &&
2303         CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
2304       Builder.CreateLoad(Ptr, true, "bf.load");
2305   }
2306 
2307   // Write the new value back out.
2308   Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified());
2309 
2310   // Return the new value of the bit-field, if requested.
2311   if (Result) {
2312     llvm::Value *ResultVal = MaskedVal;
2313 
2314     // Sign extend the value if needed.
2315     if (Info.IsSigned) {
2316       assert(Info.Size <= StorageSize);
2317       unsigned HighBits = StorageSize - Info.Size;
2318       if (HighBits) {
2319         ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl");
2320         ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr");
2321       }
2322     }
2323 
2324     ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned,
2325                                       "bf.result.cast");
2326     *Result = EmitFromMemory(ResultVal, Dst.getType());
2327   }
2328 }
2329 
2330 void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
2331                                                                LValue Dst) {
2332   // This access turns into a read/modify/write of the vector.  Load the input
2333   // value now.
2334   llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(),
2335                                         Dst.isVolatileQualified());
2336   const llvm::Constant *Elts = Dst.getExtVectorElts();
2337 
2338   llvm::Value *SrcVal = Src.getScalarVal();
2339 
2340   if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
2341     unsigned NumSrcElts = VTy->getNumElements();
2342     unsigned NumDstElts =
2343         cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
2344     if (NumDstElts == NumSrcElts) {
2345       // Use shuffle vector is the src and destination are the same number of
2346       // elements and restore the vector mask since it is on the side it will be
2347       // stored.
2348       SmallVector<int, 4> Mask(NumDstElts);
2349       for (unsigned i = 0; i != NumSrcElts; ++i)
2350         Mask[getAccessedFieldNo(i, Elts)] = i;
2351 
2352       Vec = Builder.CreateShuffleVector(SrcVal, Mask);
2353     } else if (NumDstElts > NumSrcElts) {
2354       // Extended the source vector to the same length and then shuffle it
2355       // into the destination.
2356       // FIXME: since we're shuffling with undef, can we just use the indices
2357       //        into that?  This could be simpler.
2358       SmallVector<int, 4> ExtMask;
2359       for (unsigned i = 0; i != NumSrcElts; ++i)
2360         ExtMask.push_back(i);
2361       ExtMask.resize(NumDstElts, -1);
2362       llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask);
2363       // build identity
2364       SmallVector<int, 4> Mask;
2365       for (unsigned i = 0; i != NumDstElts; ++i)
2366         Mask.push_back(i);
2367 
2368       // When the vector size is odd and .odd or .hi is used, the last element
2369       // of the Elts constant array will be one past the size of the vector.
2370       // Ignore the last element here, if it is greater than the mask size.
2371       if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size())
2372         NumSrcElts--;
2373 
2374       // modify when what gets shuffled in
2375       for (unsigned i = 0; i != NumSrcElts; ++i)
2376         Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts;
2377       Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);
2378     } else {
2379       // We should never shorten the vector
2380       llvm_unreachable("unexpected shorten vector length");
2381     }
2382   } else {
2383     // If the Src is a scalar (not a vector) it must be updating one element.
2384     unsigned InIdx = getAccessedFieldNo(0, Elts);
2385     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2386     Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
2387   }
2388 
2389   Builder.CreateStore(Vec, Dst.getExtVectorAddress(),
2390                       Dst.isVolatileQualified());
2391 }
2392 
2393 /// Store of global named registers are always calls to intrinsics.
2394 void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
2395   assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
2396          "Bad type for register variable");
2397   llvm::MDNode *RegName = cast<llvm::MDNode>(
2398       cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata());
2399   assert(RegName && "Register LValue is not metadata");
2400 
2401   // We accept integer and pointer types only
2402   llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType());
2403   llvm::Type *Ty = OrigTy;
2404   if (OrigTy->isPointerTy())
2405     Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2406   llvm::Type *Types[] = { Ty };
2407 
2408   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
2409   llvm::Value *Value = Src.getScalarVal();
2410   if (OrigTy->isPointerTy())
2411     Value = Builder.CreatePtrToInt(Value, Ty);
2412   Builder.CreateCall(
2413       F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value});
2414 }
2415 
2416 // setObjCGCLValueClass - sets class of the lvalue for the purpose of
2417 // generating write-barries API. It is currently a global, ivar,
2418 // or neither.
2419 static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
2420                                  LValue &LV,
2421                                  bool IsMemberAccess=false) {
2422   if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
2423     return;
2424 
2425   if (isa<ObjCIvarRefExpr>(E)) {
2426     QualType ExpTy = E->getType();
2427     if (IsMemberAccess && ExpTy->isPointerType()) {
2428       // If ivar is a structure pointer, assigning to field of
2429       // this struct follows gcc's behavior and makes it a non-ivar
2430       // writer-barrier conservatively.
2431       ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2432       if (ExpTy->isRecordType()) {
2433         LV.setObjCIvar(false);
2434         return;
2435       }
2436     }
2437     LV.setObjCIvar(true);
2438     auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E));
2439     LV.setBaseIvarExp(Exp->getBase());
2440     LV.setObjCArray(E->getType()->isArrayType());
2441     return;
2442   }
2443 
2444   if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) {
2445     if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
2446       if (VD->hasGlobalStorage()) {
2447         LV.setGlobalObjCRef(true);
2448         LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
2449       }
2450     }
2451     LV.setObjCArray(E->getType()->isArrayType());
2452     return;
2453   }
2454 
2455   if (const auto *Exp = dyn_cast<UnaryOperator>(E)) {
2456     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2457     return;
2458   }
2459 
2460   if (const auto *Exp = dyn_cast<ParenExpr>(E)) {
2461     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2462     if (LV.isObjCIvar()) {
2463       // If cast is to a structure pointer, follow gcc's behavior and make it
2464       // a non-ivar write-barrier.
2465       QualType ExpTy = E->getType();
2466       if (ExpTy->isPointerType())
2467         ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2468       if (ExpTy->isRecordType())
2469         LV.setObjCIvar(false);
2470     }
2471     return;
2472   }
2473 
2474   if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) {
2475     setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
2476     return;
2477   }
2478 
2479   if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) {
2480     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2481     return;
2482   }
2483 
2484   if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) {
2485     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2486     return;
2487   }
2488 
2489   if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
2490     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2491     return;
2492   }
2493 
2494   if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
2495     setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
2496     if (LV.isObjCIvar() && !LV.isObjCArray())
2497       // Using array syntax to assigning to what an ivar points to is not
2498       // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
2499       LV.setObjCIvar(false);
2500     else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
2501       // Using array syntax to assigning to what global points to is not
2502       // same as assigning to the global itself. {id *G;} G[i] = 0;
2503       LV.setGlobalObjCRef(false);
2504     return;
2505   }
2506 
2507   if (const auto *Exp = dyn_cast<MemberExpr>(E)) {
2508     setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
2509     // We don't know if member is an 'ivar', but this flag is looked at
2510     // only in the context of LV.isObjCIvar().
2511     LV.setObjCArray(E->getType()->isArrayType());
2512     return;
2513   }
2514 }
2515 
2516 static llvm::Value *
2517 EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
2518                                 llvm::Value *V, llvm::Type *IRType,
2519                                 StringRef Name = StringRef()) {
2520   unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
2521   return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
2522 }
2523 
2524 static LValue EmitThreadPrivateVarDeclLValue(
2525     CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
2526     llvm::Type *RealVarTy, SourceLocation Loc) {
2527   if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
2528     Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
2529         CGF, VD, Addr, Loc);
2530   else
2531     Addr =
2532         CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
2533 
2534   Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy);
2535   return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2536 }
2537 
2538 static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
2539                                            const VarDecl *VD, QualType T) {
2540   llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2541       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2542   // Return an invalid address if variable is MT_To and unified
2543   // memory is not enabled. For all other cases: MT_Link and
2544   // MT_To with unified memory, return a valid address.
2545   if (!Res || (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2546                !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
2547     return Address::invalid();
2548   assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2549           (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2550            CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
2551          "Expected link clause OR to clause with unified memory enabled.");
2552   QualType PtrTy = CGF.getContext().getPointerType(VD->getType());
2553   Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
2554   return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>());
2555 }
2556 
2557 Address
2558 CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
2559                                      LValueBaseInfo *PointeeBaseInfo,
2560                                      TBAAAccessInfo *PointeeTBAAInfo) {
2561   llvm::LoadInst *Load =
2562       Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
2563   CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo());
2564 
2565   QualType PointeeType = RefLVal.getType()->getPointeeType();
2566   CharUnits Align = CGM.getNaturalTypeAlignment(
2567       PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
2568       /* forPointeeType= */ true);
2569   return Address(Load, ConvertTypeForMem(PointeeType), Align);
2570 }
2571 
2572 LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
2573   LValueBaseInfo PointeeBaseInfo;
2574   TBAAAccessInfo PointeeTBAAInfo;
2575   Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo,
2576                                             &PointeeTBAAInfo);
2577   return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(),
2578                         PointeeBaseInfo, PointeeTBAAInfo);
2579 }
2580 
2581 Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
2582                                            const PointerType *PtrTy,
2583                                            LValueBaseInfo *BaseInfo,
2584                                            TBAAAccessInfo *TBAAInfo) {
2585   llvm::Value *Addr = Builder.CreateLoad(Ptr);
2586   return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
2587                  CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
2588                                              TBAAInfo,
2589                                              /*forPointeeType=*/true));
2590 }
2591 
2592 LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
2593                                                 const PointerType *PtrTy) {
2594   LValueBaseInfo BaseInfo;
2595   TBAAAccessInfo TBAAInfo;
2596   Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo);
2597   return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
2598 }
2599 
2600 static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
2601                                       const Expr *E, const VarDecl *VD) {
2602   QualType T = E->getType();
2603 
2604   // If it's thread_local, emit a call to its wrapper function instead.
2605   if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
2606       CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
2607     return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T);
2608   // Check if the variable is marked as declare target with link clause in
2609   // device codegen.
2610   if (CGF.getLangOpts().OpenMPIsDevice) {
2611     Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
2612     if (Addr.isValid())
2613       return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2614   }
2615 
2616   llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
2617   llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
2618   V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
2619   CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2620   Address Addr(V, RealVarTy, Alignment);
2621   // Emit reference to the private copy of the variable if it is an OpenMP
2622   // threadprivate variable.
2623   if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
2624       VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2625     return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
2626                                           E->getExprLoc());
2627   }
2628   LValue LV = VD->getType()->isReferenceType() ?
2629       CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2630                                     AlignmentSource::Decl) :
2631       CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2632   setObjCGCLValueClass(CGF.getContext(), E, LV);
2633   return LV;
2634 }
2635 
2636 static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM,
2637                                                GlobalDecl GD) {
2638   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2639   if (FD->hasAttr<WeakRefAttr>()) {
2640     ConstantAddress aliasee = CGM.GetWeakRefReference(FD);
2641     return aliasee.getPointer();
2642   }
2643 
2644   llvm::Constant *V = CGM.GetAddrOfFunction(GD);
2645   if (!FD->hasPrototype()) {
2646     if (const FunctionProtoType *Proto =
2647             FD->getType()->getAs<FunctionProtoType>()) {
2648       // Ugly case: for a K&R-style definition, the type of the definition
2649       // isn't the same as the type of a use.  Correct for this with a
2650       // bitcast.
2651       QualType NoProtoType =
2652           CGM.getContext().getFunctionNoProtoType(Proto->getReturnType());
2653       NoProtoType = CGM.getContext().getPointerType(NoProtoType);
2654       V = llvm::ConstantExpr::getBitCast(V,
2655                                       CGM.getTypes().ConvertType(NoProtoType));
2656     }
2657   }
2658   return V;
2659 }
2660 
2661 static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
2662                                      GlobalDecl GD) {
2663   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2664   llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD);
2665   CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
2666   return CGF.MakeAddrLValue(V, E->getType(), Alignment,
2667                             AlignmentSource::Decl);
2668 }
2669 
2670 static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
2671                                       llvm::Value *ThisValue) {
2672   QualType TagType = CGF.getContext().getTagDeclType(FD->getParent());
2673   LValue LV = CGF.MakeNaturalAlignAddrLValue(ThisValue, TagType);
2674   return CGF.EmitLValueForField(LV, FD);
2675 }
2676 
2677 /// Named Registers are named metadata pointing to the register name
2678 /// which will be read from/written to as an argument to the intrinsic
2679 /// @llvm.read/write_register.
2680 /// So far, only the name is being passed down, but other options such as
2681 /// register type, allocation type or even optimization options could be
2682 /// passed down via the metadata node.
2683 static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
2684   SmallString<64> Name("llvm.named.register.");
2685   AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
2686   assert(Asm->getLabel().size() < 64-Name.size() &&
2687       "Register name too big");
2688   Name.append(Asm->getLabel());
2689   llvm::NamedMDNode *M =
2690     CGM.getModule().getOrInsertNamedMetadata(Name);
2691   if (M->getNumOperands() == 0) {
2692     llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(),
2693                                               Asm->getLabel());
2694     llvm::Metadata *Ops[] = {Str};
2695     M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
2696   }
2697 
2698   CharUnits Alignment = CGM.getContext().getDeclAlign(VD);
2699 
2700   llvm::Value *Ptr =
2701     llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));
2702   return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType());
2703 }
2704 
2705 /// Determine whether we can emit a reference to \p VD from the current
2706 /// context, despite not necessarily having seen an odr-use of the variable in
2707 /// this context.
2708 static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
2709                                                const DeclRefExpr *E,
2710                                                const VarDecl *VD,
2711                                                bool IsConstant) {
2712   // For a variable declared in an enclosing scope, do not emit a spurious
2713   // reference even if we have a capture, as that will emit an unwarranted
2714   // reference to our capture state, and will likely generate worse code than
2715   // emitting a local copy.
2716   if (E->refersToEnclosingVariableOrCapture())
2717     return false;
2718 
2719   // For a local declaration declared in this function, we can always reference
2720   // it even if we don't have an odr-use.
2721   if (VD->hasLocalStorage()) {
2722     return VD->getDeclContext() ==
2723            dyn_cast_or_null<DeclContext>(CGF.CurCodeDecl);
2724   }
2725 
2726   // For a global declaration, we can emit a reference to it if we know
2727   // for sure that we are able to emit a definition of it.
2728   VD = VD->getDefinition(CGF.getContext());
2729   if (!VD)
2730     return false;
2731 
2732   // Don't emit a spurious reference if it might be to a variable that only
2733   // exists on a different device / target.
2734   // FIXME: This is unnecessarily broad. Check whether this would actually be a
2735   // cross-target reference.
2736   if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
2737       CGF.getLangOpts().OpenCL) {
2738     return false;
2739   }
2740 
2741   // We can emit a spurious reference only if the linkage implies that we'll
2742   // be emitting a non-interposable symbol that will be retained until link
2743   // time.
2744   switch (CGF.CGM.getLLVMLinkageVarDefinition(VD, IsConstant)) {
2745   case llvm::GlobalValue::ExternalLinkage:
2746   case llvm::GlobalValue::LinkOnceODRLinkage:
2747   case llvm::GlobalValue::WeakODRLinkage:
2748   case llvm::GlobalValue::InternalLinkage:
2749   case llvm::GlobalValue::PrivateLinkage:
2750     return true;
2751   default:
2752     return false;
2753   }
2754 }
2755 
2756 LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
2757   const NamedDecl *ND = E->getDecl();
2758   QualType T = E->getType();
2759 
2760   assert(E->isNonOdrUse() != NOUR_Unevaluated &&
2761          "should not emit an unevaluated operand");
2762 
2763   if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2764     // Global Named registers access via intrinsics only
2765     if (VD->getStorageClass() == SC_Register &&
2766         VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
2767       return EmitGlobalNamedRegister(VD, CGM);
2768 
2769     // If this DeclRefExpr does not constitute an odr-use of the variable,
2770     // we're not permitted to emit a reference to it in general, and it might
2771     // not be captured if capture would be necessary for a use. Emit the
2772     // constant value directly instead.
2773     if (E->isNonOdrUse() == NOUR_Constant &&
2774         (VD->getType()->isReferenceType() ||
2775          !canEmitSpuriousReferenceToVariable(*this, E, VD, true))) {
2776       VD->getAnyInitializer(VD);
2777       llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
2778           E->getLocation(), *VD->evaluateValue(), VD->getType());
2779       assert(Val && "failed to emit constant expression");
2780 
2781       Address Addr = Address::invalid();
2782       if (!VD->getType()->isReferenceType()) {
2783         // Spill the constant value to a global.
2784         Addr = CGM.createUnnamedGlobalFrom(*VD, Val,
2785                                            getContext().getDeclAlign(VD));
2786         llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());
2787         auto *PTy = llvm::PointerType::get(
2788             VarTy, getContext().getTargetAddressSpace(VD->getType()));
2789         Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
2790       } else {
2791         // Should we be using the alignment of the constant pointer we emitted?
2792         CharUnits Alignment =
2793             CGM.getNaturalTypeAlignment(E->getType(),
2794                                         /* BaseInfo= */ nullptr,
2795                                         /* TBAAInfo= */ nullptr,
2796                                         /* forPointeeType= */ true);
2797         Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
2798       }
2799       return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2800     }
2801 
2802     // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
2803 
2804     // Check for captured variables.
2805     if (E->refersToEnclosingVariableOrCapture()) {
2806       VD = VD->getCanonicalDecl();
2807       if (auto *FD = LambdaCaptureFields.lookup(VD))
2808         return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2809       if (CapturedStmtInfo) {
2810         auto I = LocalDeclMap.find(VD);
2811         if (I != LocalDeclMap.end()) {
2812           LValue CapLVal;
2813           if (VD->getType()->isReferenceType())
2814             CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(),
2815                                                 AlignmentSource::Decl);
2816           else
2817             CapLVal = MakeAddrLValue(I->second, T);
2818           // Mark lvalue as nontemporal if the variable is marked as nontemporal
2819           // in simd context.
2820           if (getLangOpts().OpenMP &&
2821               CGM.getOpenMPRuntime().isNontemporalDecl(VD))
2822             CapLVal.setNontemporal(/*Value=*/true);
2823           return CapLVal;
2824         }
2825         LValue CapLVal =
2826             EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
2827                                     CapturedStmtInfo->getContextValue());
2828         Address LValueAddress = CapLVal.getAddress(*this);
2829         CapLVal = MakeAddrLValue(
2830             Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
2831                     getContext().getDeclAlign(VD)),
2832             CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl),
2833             CapLVal.getTBAAInfo());
2834         // Mark lvalue as nontemporal if the variable is marked as nontemporal
2835         // in simd context.
2836         if (getLangOpts().OpenMP &&
2837             CGM.getOpenMPRuntime().isNontemporalDecl(VD))
2838           CapLVal.setNontemporal(/*Value=*/true);
2839         return CapLVal;
2840       }
2841 
2842       assert(isa<BlockDecl>(CurCodeDecl));
2843       Address addr = GetAddrOfBlockDecl(VD);
2844       return MakeAddrLValue(addr, T, AlignmentSource::Decl);
2845     }
2846   }
2847 
2848   // FIXME: We should be able to assert this for FunctionDecls as well!
2849   // FIXME: We should be able to assert this for all DeclRefExprs, not just
2850   // those with a valid source location.
2851   assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
2852           !E->getLocation().isValid()) &&
2853          "Should not use decl without marking it used!");
2854 
2855   if (ND->hasAttr<WeakRefAttr>()) {
2856     const auto *VD = cast<ValueDecl>(ND);
2857     ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
2858     return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl);
2859   }
2860 
2861   if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2862     // Check if this is a global variable.
2863     if (VD->hasLinkage() || VD->isStaticDataMember())
2864       return EmitGlobalVarDeclLValue(*this, E, VD);
2865 
2866     Address addr = Address::invalid();
2867 
2868     // The variable should generally be present in the local decl map.
2869     auto iter = LocalDeclMap.find(VD);
2870     if (iter != LocalDeclMap.end()) {
2871       addr = iter->second;
2872 
2873     // Otherwise, it might be static local we haven't emitted yet for
2874     // some reason; most likely, because it's in an outer function.
2875     } else if (VD->isStaticLocal()) {
2876       llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
2877           *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false));
2878       addr = Address(
2879           var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
2880 
2881     // No other cases for now.
2882     } else {
2883       llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
2884     }
2885 
2886 
2887     // Check for OpenMP threadprivate variables.
2888     if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
2889         VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2890       return EmitThreadPrivateVarDeclLValue(
2891           *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()),
2892           E->getExprLoc());
2893     }
2894 
2895     // Drill into block byref variables.
2896     bool isBlockByref = VD->isEscapingByref();
2897     if (isBlockByref) {
2898       addr = emitBlockByrefAddress(addr, VD);
2899     }
2900 
2901     // Drill into reference types.
2902     LValue LV = VD->getType()->isReferenceType() ?
2903         EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) :
2904         MakeAddrLValue(addr, T, AlignmentSource::Decl);
2905 
2906     bool isLocalStorage = VD->hasLocalStorage();
2907 
2908     bool NonGCable = isLocalStorage &&
2909                      !VD->getType()->isReferenceType() &&
2910                      !isBlockByref;
2911     if (NonGCable) {
2912       LV.getQuals().removeObjCGCAttr();
2913       LV.setNonGC(true);
2914     }
2915 
2916     bool isImpreciseLifetime =
2917       (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
2918     if (isImpreciseLifetime)
2919       LV.setARCPreciseLifetime(ARCImpreciseLifetime);
2920     setObjCGCLValueClass(getContext(), E, LV);
2921     return LV;
2922   }
2923 
2924   if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
2925     LValue LV = EmitFunctionDeclLValue(*this, E, FD);
2926 
2927     // Emit debuginfo for the function declaration if the target wants to.
2928     if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) {
2929       if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) {
2930         auto *Fn =
2931             cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts());
2932         if (!Fn->getSubprogram())
2933           DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn);
2934       }
2935     }
2936 
2937     return LV;
2938   }
2939 
2940   // FIXME: While we're emitting a binding from an enclosing scope, all other
2941   // DeclRefExprs we see should be implicitly treated as if they also refer to
2942   // an enclosing scope.
2943   if (const auto *BD = dyn_cast<BindingDecl>(ND))
2944     return EmitLValue(BD->getBinding());
2945 
2946   // We can form DeclRefExprs naming GUID declarations when reconstituting
2947   // non-type template parameters into expressions.
2948   if (const auto *GD = dyn_cast<MSGuidDecl>(ND))
2949     return MakeAddrLValue(CGM.GetAddrOfMSGuidDecl(GD), T,
2950                           AlignmentSource::Decl);
2951 
2952   if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND))
2953     return MakeAddrLValue(CGM.GetAddrOfTemplateParamObject(TPO), T,
2954                           AlignmentSource::Decl);
2955 
2956   llvm_unreachable("Unhandled DeclRefExpr");
2957 }
2958 
2959 LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
2960   // __extension__ doesn't affect lvalue-ness.
2961   if (E->getOpcode() == UO_Extension)
2962     return EmitLValue(E->getSubExpr());
2963 
2964   QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
2965   switch (E->getOpcode()) {
2966   default: llvm_unreachable("Unknown unary operator lvalue!");
2967   case UO_Deref: {
2968     QualType T = E->getSubExpr()->getType()->getPointeeType();
2969     assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
2970 
2971     LValueBaseInfo BaseInfo;
2972     TBAAAccessInfo TBAAInfo;
2973     Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &BaseInfo,
2974                                             &TBAAInfo);
2975     LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
2976     LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
2977 
2978     // We should not generate __weak write barrier on indirect reference
2979     // of a pointer to object; as in void foo (__weak id *param); *param = 0;
2980     // But, we continue to generate __strong write barrier on indirect write
2981     // into a pointer to object.
2982     if (getLangOpts().ObjC &&
2983         getLangOpts().getGC() != LangOptions::NonGC &&
2984         LV.isObjCWeak())
2985       LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
2986     return LV;
2987   }
2988   case UO_Real:
2989   case UO_Imag: {
2990     LValue LV = EmitLValue(E->getSubExpr());
2991     assert(LV.isSimple() && "real/imag on non-ordinary l-value");
2992 
2993     // __real is valid on scalars.  This is a faster way of testing that.
2994     // __imag can only produce an rvalue on scalars.
2995     if (E->getOpcode() == UO_Real &&
2996         !LV.getAddress(*this).getElementType()->isStructTy()) {
2997       assert(E->getSubExpr()->getType()->isArithmeticType());
2998       return LV;
2999     }
3000 
3001     QualType T = ExprTy->castAs<ComplexType>()->getElementType();
3002 
3003     Address Component =
3004         (E->getOpcode() == UO_Real
3005              ? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType())
3006              : emitAddrOfImagComponent(LV.getAddress(*this), LV.getType()));
3007     LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(),
3008                                    CGM.getTBAAInfoForSubobject(LV, T));
3009     ElemLV.getQuals().addQualifiers(LV.getQuals());
3010     return ElemLV;
3011   }
3012   case UO_PreInc:
3013   case UO_PreDec: {
3014     LValue LV = EmitLValue(E->getSubExpr());
3015     bool isInc = E->getOpcode() == UO_PreInc;
3016 
3017     if (E->getType()->isAnyComplexType())
3018       EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
3019     else
3020       EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
3021     return LV;
3022   }
3023   }
3024 }
3025 
3026 LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
3027   return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
3028                         E->getType(), AlignmentSource::Decl);
3029 }
3030 
3031 LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
3032   return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
3033                         E->getType(), AlignmentSource::Decl);
3034 }
3035 
3036 LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
3037   auto SL = E->getFunctionName();
3038   assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3039   StringRef FnName = CurFn->getName();
3040   if (FnName.startswith("\01"))
3041     FnName = FnName.substr(1);
3042   StringRef NameItems[] = {
3043       PredefinedExpr::getIdentKindName(E->getIdentKind()), FnName};
3044   std::string GVName = llvm::join(NameItems, NameItems + 2, ".");
3045   if (auto *BD = dyn_cast_or_null<BlockDecl>(CurCodeDecl)) {
3046     std::string Name = std::string(SL->getString());
3047     if (!Name.empty()) {
3048       unsigned Discriminator =
3049           CGM.getCXXABI().getMangleContext().getBlockId(BD, true);
3050       if (Discriminator)
3051         Name += "_" + Twine(Discriminator + 1).str();
3052       auto C = CGM.GetAddrOfConstantCString(Name, GVName.c_str());
3053       return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3054     } else {
3055       auto C =
3056           CGM.GetAddrOfConstantCString(std::string(FnName), GVName.c_str());
3057       return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3058     }
3059   }
3060   auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName);
3061   return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3062 }
3063 
3064 /// Emit a type description suitable for use by a runtime sanitizer library. The
3065 /// format of a type descriptor is
3066 ///
3067 /// \code
3068 ///   { i16 TypeKind, i16 TypeInfo }
3069 /// \endcode
3070 ///
3071 /// followed by an array of i8 containing the type name. TypeKind is 0 for an
3072 /// integer, 1 for a floating point value, and -1 for anything else.
3073 llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
3074   // Only emit each type's descriptor once.
3075   if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))
3076     return C;
3077 
3078   uint16_t TypeKind = -1;
3079   uint16_t TypeInfo = 0;
3080 
3081   if (T->isIntegerType()) {
3082     TypeKind = 0;
3083     TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
3084                (T->isSignedIntegerType() ? 1 : 0);
3085   } else if (T->isFloatingType()) {
3086     TypeKind = 1;
3087     TypeInfo = getContext().getTypeSize(T);
3088   }
3089 
3090   // Format the type name as if for a diagnostic, including quotes and
3091   // optionally an 'aka'.
3092   SmallString<32> Buffer;
3093   CGM.getDiags().ConvertArgToString(DiagnosticsEngine::ak_qualtype,
3094                                     (intptr_t)T.getAsOpaquePtr(),
3095                                     StringRef(), StringRef(), None, Buffer,
3096                                     None);
3097 
3098   llvm::Constant *Components[] = {
3099     Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
3100     llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
3101   };
3102   llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
3103 
3104   auto *GV = new llvm::GlobalVariable(
3105       CGM.getModule(), Descriptor->getType(),
3106       /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3107   GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3108   CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
3109 
3110   // Remember the descriptor for this type.
3111   CGM.setTypeDescriptorInMap(T, GV);
3112 
3113   return GV;
3114 }
3115 
3116 llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
3117   llvm::Type *TargetTy = IntPtrTy;
3118 
3119   if (V->getType() == TargetTy)
3120     return V;
3121 
3122   // Floating-point types which fit into intptr_t are bitcast to integers
3123   // and then passed directly (after zero-extension, if necessary).
3124   if (V->getType()->isFloatingPointTy()) {
3125     unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedSize();
3126     if (Bits <= TargetTy->getIntegerBitWidth())
3127       V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(),
3128                                                          Bits));
3129   }
3130 
3131   // Integers which fit in intptr_t are zero-extended and passed directly.
3132   if (V->getType()->isIntegerTy() &&
3133       V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3134     return Builder.CreateZExt(V, TargetTy);
3135 
3136   // Pointers are passed directly, everything else is passed by address.
3137   if (!V->getType()->isPointerTy()) {
3138     Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
3139     Builder.CreateStore(V, Ptr);
3140     V = Ptr.getPointer();
3141   }
3142   return Builder.CreatePtrToInt(V, TargetTy);
3143 }
3144 
3145 /// Emit a representation of a SourceLocation for passing to a handler
3146 /// in a sanitizer runtime library. The format for this data is:
3147 /// \code
3148 ///   struct SourceLocation {
3149 ///     const char *Filename;
3150 ///     int32_t Line, Column;
3151 ///   };
3152 /// \endcode
3153 /// For an invalid SourceLocation, the Filename pointer is null.
3154 llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
3155   llvm::Constant *Filename;
3156   int Line, Column;
3157 
3158   PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
3159   if (PLoc.isValid()) {
3160     StringRef FilenameString = PLoc.getFilename();
3161 
3162     int PathComponentsToStrip =
3163         CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
3164     if (PathComponentsToStrip < 0) {
3165       assert(PathComponentsToStrip != INT_MIN);
3166       int PathComponentsToKeep = -PathComponentsToStrip;
3167       auto I = llvm::sys::path::rbegin(FilenameString);
3168       auto E = llvm::sys::path::rend(FilenameString);
3169       while (I != E && --PathComponentsToKeep)
3170         ++I;
3171 
3172       FilenameString = FilenameString.substr(I - E);
3173     } else if (PathComponentsToStrip > 0) {
3174       auto I = llvm::sys::path::begin(FilenameString);
3175       auto E = llvm::sys::path::end(FilenameString);
3176       while (I != E && PathComponentsToStrip--)
3177         ++I;
3178 
3179       if (I != E)
3180         FilenameString =
3181             FilenameString.substr(I - llvm::sys::path::begin(FilenameString));
3182       else
3183         FilenameString = llvm::sys::path::filename(FilenameString);
3184     }
3185 
3186     auto FilenameGV =
3187         CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src");
3188     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
3189                           cast<llvm::GlobalVariable>(FilenameGV.getPointer()));
3190     Filename = FilenameGV.getPointer();
3191     Line = PLoc.getLine();
3192     Column = PLoc.getColumn();
3193   } else {
3194     Filename = llvm::Constant::getNullValue(Int8PtrTy);
3195     Line = Column = 0;
3196   }
3197 
3198   llvm::Constant *Data[] = {Filename, Builder.getInt32(Line),
3199                             Builder.getInt32(Column)};
3200 
3201   return llvm::ConstantStruct::getAnon(Data);
3202 }
3203 
3204 namespace {
3205 /// Specify under what conditions this check can be recovered
3206 enum class CheckRecoverableKind {
3207   /// Always terminate program execution if this check fails.
3208   Unrecoverable,
3209   /// Check supports recovering, runtime has both fatal (noreturn) and
3210   /// non-fatal handlers for this check.
3211   Recoverable,
3212   /// Runtime conditionally aborts, always need to support recovery.
3213   AlwaysRecoverable
3214 };
3215 }
3216 
3217 static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) {
3218   assert(Kind.countPopulation() == 1);
3219   if (Kind == SanitizerKind::Function || Kind == SanitizerKind::Vptr)
3220     return CheckRecoverableKind::AlwaysRecoverable;
3221   else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3222     return CheckRecoverableKind::Unrecoverable;
3223   else
3224     return CheckRecoverableKind::Recoverable;
3225 }
3226 
3227 namespace {
3228 struct SanitizerHandlerInfo {
3229   char const *const Name;
3230   unsigned Version;
3231 };
3232 }
3233 
3234 const SanitizerHandlerInfo SanitizerHandlers[] = {
3235 #define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3236     LIST_SANITIZER_CHECKS
3237 #undef SANITIZER_CHECK
3238 };
3239 
3240 static void emitCheckHandlerCall(CodeGenFunction &CGF,
3241                                  llvm::FunctionType *FnType,
3242                                  ArrayRef<llvm::Value *> FnArgs,
3243                                  SanitizerHandler CheckHandler,
3244                                  CheckRecoverableKind RecoverKind, bool IsFatal,
3245                                  llvm::BasicBlock *ContBB) {
3246   assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3247   Optional<ApplyDebugLocation> DL;
3248   if (!CGF.Builder.getCurrentDebugLocation()) {
3249     // Ensure that the call has at least an artificial debug location.
3250     DL.emplace(CGF, SourceLocation());
3251   }
3252   bool NeedsAbortSuffix =
3253       IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3254   bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
3255   const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
3256   const StringRef CheckName = CheckInfo.Name;
3257   std::string FnName = "__ubsan_handle_" + CheckName.str();
3258   if (CheckInfo.Version && !MinimalRuntime)
3259     FnName += "_v" + llvm::utostr(CheckInfo.Version);
3260   if (MinimalRuntime)
3261     FnName += "_minimal";
3262   if (NeedsAbortSuffix)
3263     FnName += "_abort";
3264   bool MayReturn =
3265       !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3266 
3267   llvm::AttrBuilder B(CGF.getLLVMContext());
3268   if (!MayReturn) {
3269     B.addAttribute(llvm::Attribute::NoReturn)
3270         .addAttribute(llvm::Attribute::NoUnwind);
3271   }
3272   B.addUWTableAttr(llvm::UWTableKind::Default);
3273 
3274   llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
3275       FnType, FnName,
3276       llvm::AttributeList::get(CGF.getLLVMContext(),
3277                                llvm::AttributeList::FunctionIndex, B),
3278       /*Local=*/true);
3279   llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs);
3280   if (!MayReturn) {
3281     HandlerCall->setDoesNotReturn();
3282     CGF.Builder.CreateUnreachable();
3283   } else {
3284     CGF.Builder.CreateBr(ContBB);
3285   }
3286 }
3287 
3288 void CodeGenFunction::EmitCheck(
3289     ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3290     SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
3291     ArrayRef<llvm::Value *> DynamicArgs) {
3292   assert(IsSanitizerScope);
3293   assert(Checked.size() > 0);
3294   assert(CheckHandler >= 0 &&
3295          size_t(CheckHandler) < llvm::array_lengthof(SanitizerHandlers));
3296   const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
3297 
3298   llvm::Value *FatalCond = nullptr;
3299   llvm::Value *RecoverableCond = nullptr;
3300   llvm::Value *TrapCond = nullptr;
3301   for (int i = 0, n = Checked.size(); i < n; ++i) {
3302     llvm::Value *Check = Checked[i].first;
3303     // -fsanitize-trap= overrides -fsanitize-recover=.
3304     llvm::Value *&Cond =
3305         CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second)
3306             ? TrapCond
3307             : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second)
3308                   ? RecoverableCond
3309                   : FatalCond;
3310     Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check;
3311   }
3312 
3313   if (TrapCond)
3314     EmitTrapCheck(TrapCond, CheckHandler);
3315   if (!FatalCond && !RecoverableCond)
3316     return;
3317 
3318   llvm::Value *JointCond;
3319   if (FatalCond && RecoverableCond)
3320     JointCond = Builder.CreateAnd(FatalCond, RecoverableCond);
3321   else
3322     JointCond = FatalCond ? FatalCond : RecoverableCond;
3323   assert(JointCond);
3324 
3325   CheckRecoverableKind RecoverKind = getRecoverableKind(Checked[0].second);
3326   assert(SanOpts.has(Checked[0].second));
3327 #ifndef NDEBUG
3328   for (int i = 1, n = Checked.size(); i < n; ++i) {
3329     assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
3330            "All recoverable kinds in a single check must be same!");
3331     assert(SanOpts.has(Checked[i].second));
3332   }
3333 #endif
3334 
3335   llvm::BasicBlock *Cont = createBasicBlock("cont");
3336   llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName);
3337   llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers);
3338   // Give hint that we very much don't expect to execute the handler
3339   // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
3340   llvm::MDBuilder MDHelper(getLLVMContext());
3341   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3342   Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);
3343   EmitBlock(Handlers);
3344 
3345   // Handler functions take an i8* pointing to the (handler-specific) static
3346   // information block, followed by a sequence of intptr_t arguments
3347   // representing operand values.
3348   SmallVector<llvm::Value *, 4> Args;
3349   SmallVector<llvm::Type *, 4> ArgTypes;
3350   if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
3351     Args.reserve(DynamicArgs.size() + 1);
3352     ArgTypes.reserve(DynamicArgs.size() + 1);
3353 
3354     // Emit handler arguments and create handler function type.
3355     if (!StaticArgs.empty()) {
3356       llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3357       auto *InfoPtr =
3358           new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
3359                                    llvm::GlobalVariable::PrivateLinkage, Info);
3360       InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3361       CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
3362       Args.push_back(Builder.CreateBitCast(InfoPtr, Int8PtrTy));
3363       ArgTypes.push_back(Int8PtrTy);
3364     }
3365 
3366     for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3367       Args.push_back(EmitCheckValue(DynamicArgs[i]));
3368       ArgTypes.push_back(IntPtrTy);
3369     }
3370   }
3371 
3372   llvm::FunctionType *FnType =
3373     llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);
3374 
3375   if (!FatalCond || !RecoverableCond) {
3376     // Simple case: we need to generate a single handler call, either
3377     // fatal, or non-fatal.
3378     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,
3379                          (FatalCond != nullptr), Cont);
3380   } else {
3381     // Emit two handler calls: first one for set of unrecoverable checks,
3382     // another one for recoverable.
3383     llvm::BasicBlock *NonFatalHandlerBB =
3384         createBasicBlock("non_fatal." + CheckName);
3385     llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName);
3386     Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
3387     EmitBlock(FatalHandlerBB);
3388     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,
3389                          NonFatalHandlerBB);
3390     EmitBlock(NonFatalHandlerBB);
3391     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,
3392                          Cont);
3393   }
3394 
3395   EmitBlock(Cont);
3396 }
3397 
3398 void CodeGenFunction::EmitCfiSlowPathCheck(
3399     SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3400     llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) {
3401   llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");
3402 
3403   llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");
3404   llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);
3405 
3406   llvm::MDBuilder MDHelper(getLLVMContext());
3407   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3408   BI->setMetadata(llvm::LLVMContext::MD_prof, Node);
3409 
3410   EmitBlock(CheckBB);
3411 
3412   bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Kind);
3413 
3414   llvm::CallInst *CheckCall;
3415   llvm::FunctionCallee SlowPathFn;
3416   if (WithDiag) {
3417     llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3418     auto *InfoPtr =
3419         new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
3420                                  llvm::GlobalVariable::PrivateLinkage, Info);
3421     InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3422     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
3423 
3424     SlowPathFn = CGM.getModule().getOrInsertFunction(
3425         "__cfi_slowpath_diag",
3426         llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy},
3427                                 false));
3428     CheckCall = Builder.CreateCall(
3429         SlowPathFn, {TypeId, Ptr, Builder.CreateBitCast(InfoPtr, Int8PtrTy)});
3430   } else {
3431     SlowPathFn = CGM.getModule().getOrInsertFunction(
3432         "__cfi_slowpath",
3433         llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false));
3434     CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
3435   }
3436 
3437   CGM.setDSOLocal(
3438       cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));
3439   CheckCall->setDoesNotThrow();
3440 
3441   EmitBlock(Cont);
3442 }
3443 
3444 // Emit a stub for __cfi_check function so that the linker knows about this
3445 // symbol in LTO mode.
3446 void CodeGenFunction::EmitCfiCheckStub() {
3447   llvm::Module *M = &CGM.getModule();
3448   auto &Ctx = M->getContext();
3449   llvm::Function *F = llvm::Function::Create(
3450       llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false),
3451       llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M);
3452   CGM.setDSOLocal(F);
3453   llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F);
3454   // FIXME: consider emitting an intrinsic call like
3455   // call void @llvm.cfi_check(i64 %0, i8* %1, i8* %2)
3456   // which can be lowered in CrossDSOCFI pass to the actual contents of
3457   // __cfi_check. This would allow inlining of __cfi_check calls.
3458   llvm::CallInst::Create(
3459       llvm::Intrinsic::getDeclaration(M, llvm::Intrinsic::trap), "", BB);
3460   llvm::ReturnInst::Create(Ctx, nullptr, BB);
3461 }
3462 
3463 // This function is basically a switch over the CFI failure kind, which is
3464 // extracted from CFICheckFailData (1st function argument). Each case is either
3465 // llvm.trap or a call to one of the two runtime handlers, based on
3466 // -fsanitize-trap and -fsanitize-recover settings.  Default case (invalid
3467 // failure kind) traps, but this should really never happen.  CFICheckFailData
3468 // can be nullptr if the calling module has -fsanitize-trap behavior for this
3469 // check kind; in this case __cfi_check_fail traps as well.
3470 void CodeGenFunction::EmitCfiCheckFail() {
3471   SanitizerScope SanScope(this);
3472   FunctionArgList Args;
3473   ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy,
3474                             ImplicitParamDecl::Other);
3475   ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy,
3476                             ImplicitParamDecl::Other);
3477   Args.push_back(&ArgData);
3478   Args.push_back(&ArgAddr);
3479 
3480   const CGFunctionInfo &FI =
3481     CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, Args);
3482 
3483   llvm::Function *F = llvm::Function::Create(
3484       llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false),
3485       llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule());
3486 
3487   CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);
3488   CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
3489   F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3490 
3491   StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args,
3492                 SourceLocation());
3493 
3494   // This function is not affected by NoSanitizeList. This function does
3495   // not have a source location, but "src:*" would still apply. Revert any
3496   // changes to SanOpts made in StartFunction.
3497   SanOpts = CGM.getLangOpts().Sanitize;
3498 
3499   llvm::Value *Data =
3500       EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,
3501                        CGM.getContext().VoidPtrTy, ArgData.getLocation());
3502   llvm::Value *Addr =
3503       EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,
3504                        CGM.getContext().VoidPtrTy, ArgAddr.getLocation());
3505 
3506   // Data == nullptr means the calling module has trap behaviour for this check.
3507   llvm::Value *DataIsNotNullPtr =
3508       Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy));
3509   EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
3510 
3511   llvm::StructType *SourceLocationTy =
3512       llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty);
3513   llvm::StructType *CfiCheckFailDataTy =
3514       llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy);
3515 
3516   llvm::Value *V = Builder.CreateConstGEP2_32(
3517       CfiCheckFailDataTy,
3518       Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
3519       0);
3520 
3521   Address CheckKindAddr(V, Int8Ty, getIntAlign());
3522   llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
3523 
3524   llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3525       CGM.getLLVMContext(),
3526       llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
3527   llvm::Value *ValidVtable = Builder.CreateZExt(
3528       Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
3529                          {Addr, AllVtables}),
3530       IntPtrTy);
3531 
3532   const std::pair<int, SanitizerMask> CheckKinds[] = {
3533       {CFITCK_VCall, SanitizerKind::CFIVCall},
3534       {CFITCK_NVCall, SanitizerKind::CFINVCall},
3535       {CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast},
3536       {CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast},
3537       {CFITCK_ICall, SanitizerKind::CFIICall}};
3538 
3539   SmallVector<std::pair<llvm::Value *, SanitizerMask>, 5> Checks;
3540   for (auto CheckKindMaskPair : CheckKinds) {
3541     int Kind = CheckKindMaskPair.first;
3542     SanitizerMask Mask = CheckKindMaskPair.second;
3543     llvm::Value *Cond =
3544         Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind));
3545     if (CGM.getLangOpts().Sanitize.has(Mask))
3546       EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
3547                 {Data, Addr, ValidVtable});
3548     else
3549       EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail);
3550   }
3551 
3552   FinishFunction();
3553   // The only reference to this function will be created during LTO link.
3554   // Make sure it survives until then.
3555   CGM.addUsedGlobal(F);
3556 }
3557 
3558 void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
3559   if (SanOpts.has(SanitizerKind::Unreachable)) {
3560     SanitizerScope SanScope(this);
3561     EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
3562                              SanitizerKind::Unreachable),
3563               SanitizerHandler::BuiltinUnreachable,
3564               EmitCheckSourceLocation(Loc), None);
3565   }
3566   Builder.CreateUnreachable();
3567 }
3568 
3569 void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
3570                                     SanitizerHandler CheckHandlerID) {
3571   llvm::BasicBlock *Cont = createBasicBlock("cont");
3572 
3573   // If we're optimizing, collapse all calls to trap down to just one per
3574   // check-type per function to save on code size.
3575   if (TrapBBs.size() <= CheckHandlerID)
3576     TrapBBs.resize(CheckHandlerID + 1);
3577   llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3578 
3579   if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB) {
3580     TrapBB = createBasicBlock("trap");
3581     Builder.CreateCondBr(Checked, Cont, TrapBB);
3582     EmitBlock(TrapBB);
3583 
3584     llvm::CallInst *TrapCall =
3585         Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),
3586                            llvm::ConstantInt::get(CGM.Int8Ty, CheckHandlerID));
3587 
3588     if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3589       auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3590                                     CGM.getCodeGenOpts().TrapFuncName);
3591       TrapCall->addFnAttr(A);
3592     }
3593     TrapCall->setDoesNotReturn();
3594     TrapCall->setDoesNotThrow();
3595     Builder.CreateUnreachable();
3596   } else {
3597     auto Call = TrapBB->begin();
3598     assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
3599 
3600     Call->applyMergedLocation(Call->getDebugLoc(),
3601                               Builder.getCurrentDebugLocation());
3602     Builder.CreateCondBr(Checked, Cont, TrapBB);
3603   }
3604 
3605   EmitBlock(Cont);
3606 }
3607 
3608 llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
3609   llvm::CallInst *TrapCall =
3610       Builder.CreateCall(CGM.getIntrinsic(IntrID));
3611 
3612   if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3613     auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3614                                   CGM.getCodeGenOpts().TrapFuncName);
3615     TrapCall->addFnAttr(A);
3616   }
3617 
3618   return TrapCall;
3619 }
3620 
3621 Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
3622                                                  LValueBaseInfo *BaseInfo,
3623                                                  TBAAAccessInfo *TBAAInfo) {
3624   assert(E->getType()->isArrayType() &&
3625          "Array to pointer decay must have array source type!");
3626 
3627   // Expressions of array type can't be bitfields or vector elements.
3628   LValue LV = EmitLValue(E);
3629   Address Addr = LV.getAddress(*this);
3630 
3631   // If the array type was an incomplete type, we need to make sure
3632   // the decay ends up being the right type.
3633   llvm::Type *NewTy = ConvertType(E->getType());
3634   Addr = Builder.CreateElementBitCast(Addr, NewTy);
3635 
3636   // Note that VLA pointers are always decayed, so we don't need to do
3637   // anything here.
3638   if (!E->getType()->isVariableArrayType()) {
3639     assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3640            "Expected pointer to array");
3641     Addr = Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3642   }
3643 
3644   // The result of this decay conversion points to an array element within the
3645   // base lvalue. However, since TBAA currently does not support representing
3646   // accesses to elements of member arrays, we conservatively represent accesses
3647   // to the pointee object as if it had no any base lvalue specified.
3648   // TODO: Support TBAA for member arrays.
3649   QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
3650   if (BaseInfo) *BaseInfo = LV.getBaseInfo();
3651   if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType);
3652 
3653   return Builder.CreateElementBitCast(Addr, ConvertTypeForMem(EltType));
3654 }
3655 
3656 /// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
3657 /// array to pointer, return the array subexpression.
3658 static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
3659   // If this isn't just an array->pointer decay, bail out.
3660   const auto *CE = dyn_cast<CastExpr>(E);
3661   if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
3662     return nullptr;
3663 
3664   // If this is a decay from variable width array, bail out.
3665   const Expr *SubExpr = CE->getSubExpr();
3666   if (SubExpr->getType()->isVariableArrayType())
3667     return nullptr;
3668 
3669   return SubExpr;
3670 }
3671 
3672 static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
3673                                           llvm::Type *elemType,
3674                                           llvm::Value *ptr,
3675                                           ArrayRef<llvm::Value*> indices,
3676                                           bool inbounds,
3677                                           bool signedIndices,
3678                                           SourceLocation loc,
3679                                     const llvm::Twine &name = "arrayidx") {
3680   if (inbounds) {
3681     return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices,
3682                                       CodeGenFunction::NotSubtraction, loc,
3683                                       name);
3684   } else {
3685     return CGF.Builder.CreateGEP(elemType, ptr, indices, name);
3686   }
3687 }
3688 
3689 static CharUnits getArrayElementAlign(CharUnits arrayAlign,
3690                                       llvm::Value *idx,
3691                                       CharUnits eltSize) {
3692   // If we have a constant index, we can use the exact offset of the
3693   // element we're accessing.
3694   if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
3695     CharUnits offset = constantIdx->getZExtValue() * eltSize;
3696     return arrayAlign.alignmentAtOffset(offset);
3697 
3698   // Otherwise, use the worst-case alignment for any element.
3699   } else {
3700     return arrayAlign.alignmentOfArrayElement(eltSize);
3701   }
3702 }
3703 
3704 static QualType getFixedSizeElementType(const ASTContext &ctx,
3705                                         const VariableArrayType *vla) {
3706   QualType eltType;
3707   do {
3708     eltType = vla->getElementType();
3709   } while ((vla = ctx.getAsVariableArrayType(eltType)));
3710   return eltType;
3711 }
3712 
3713 /// Given an array base, check whether its member access belongs to a record
3714 /// with preserve_access_index attribute or not.
3715 static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
3716   if (!ArrayBase || !CGF.getDebugInfo())
3717     return false;
3718 
3719   // Only support base as either a MemberExpr or DeclRefExpr.
3720   // DeclRefExpr to cover cases like:
3721   //    struct s { int a; int b[10]; };
3722   //    struct s *p;
3723   //    p[1].a
3724   // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
3725   // p->b[5] is a MemberExpr example.
3726   const Expr *E = ArrayBase->IgnoreImpCasts();
3727   if (const auto *ME = dyn_cast<MemberExpr>(E))
3728     return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3729 
3730   if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
3731     const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());
3732     if (!VarDef)
3733       return false;
3734 
3735     const auto *PtrT = VarDef->getType()->getAs<PointerType>();
3736     if (!PtrT)
3737       return false;
3738 
3739     const auto *PointeeT = PtrT->getPointeeType()
3740                              ->getUnqualifiedDesugaredType();
3741     if (const auto *RecT = dyn_cast<RecordType>(PointeeT))
3742       return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3743     return false;
3744   }
3745 
3746   return false;
3747 }
3748 
3749 static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
3750                                      ArrayRef<llvm::Value *> indices,
3751                                      QualType eltType, bool inbounds,
3752                                      bool signedIndices, SourceLocation loc,
3753                                      QualType *arrayType = nullptr,
3754                                      const Expr *Base = nullptr,
3755                                      const llvm::Twine &name = "arrayidx") {
3756   // All the indices except that last must be zero.
3757 #ifndef NDEBUG
3758   for (auto idx : indices.drop_back())
3759     assert(isa<llvm::ConstantInt>(idx) &&
3760            cast<llvm::ConstantInt>(idx)->isZero());
3761 #endif
3762 
3763   // Determine the element size of the statically-sized base.  This is
3764   // the thing that the indices are expressed in terms of.
3765   if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) {
3766     eltType = getFixedSizeElementType(CGF.getContext(), vla);
3767   }
3768 
3769   // We can use that to compute the best alignment of the element.
3770   CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
3771   CharUnits eltAlign =
3772     getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
3773 
3774   llvm::Value *eltPtr;
3775   auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
3776   if (!LastIndex ||
3777       (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) {
3778     eltPtr = emitArraySubscriptGEP(
3779         CGF, addr.getElementType(), addr.getPointer(), indices, inbounds,
3780         signedIndices, loc, name);
3781   } else {
3782     // Remember the original array subscript for bpf target
3783     unsigned idx = LastIndex->getZExtValue();
3784     llvm::DIType *DbgInfo = nullptr;
3785     if (arrayType)
3786       DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);
3787     eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(),
3788                                                         addr.getPointer(),
3789                                                         indices.size() - 1,
3790                                                         idx, DbgInfo);
3791   }
3792 
3793   return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
3794 }
3795 
3796 LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
3797                                                bool Accessed) {
3798   // The index must always be an integer, which is not an aggregate.  Emit it
3799   // in lexical order (this complexity is, sadly, required by C++17).
3800   llvm::Value *IdxPre =
3801       (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr;
3802   bool SignedIndices = false;
3803   auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
3804     auto *Idx = IdxPre;
3805     if (E->getLHS() != E->getIdx()) {
3806       assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
3807       Idx = EmitScalarExpr(E->getIdx());
3808     }
3809 
3810     QualType IdxTy = E->getIdx()->getType();
3811     bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
3812     SignedIndices |= IdxSigned;
3813 
3814     if (SanOpts.has(SanitizerKind::ArrayBounds))
3815       EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed);
3816 
3817     // Extend or truncate the index type to 32 or 64-bits.
3818     if (Promote && Idx->getType() != IntPtrTy)
3819       Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
3820 
3821     return Idx;
3822   };
3823   IdxPre = nullptr;
3824 
3825   // If the base is a vector type, then we are forming a vector element lvalue
3826   // with this subscript.
3827   if (E->getBase()->getType()->isVectorType() &&
3828       !isa<ExtVectorElementExpr>(E->getBase())) {
3829     // Emit the vector as an lvalue to get its address.
3830     LValue LHS = EmitLValue(E->getBase());
3831     auto *Idx = EmitIdxAfterBase(/*Promote*/false);
3832     assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
3833     return LValue::MakeVectorElt(LHS.getAddress(*this), Idx,
3834                                  E->getBase()->getType(), LHS.getBaseInfo(),
3835                                  TBAAAccessInfo());
3836   }
3837 
3838   // All the other cases basically behave like simple offsetting.
3839 
3840   // Handle the extvector case we ignored above.
3841   if (isa<ExtVectorElementExpr>(E->getBase())) {
3842     LValue LV = EmitLValue(E->getBase());
3843     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3844     Address Addr = EmitExtVectorElementLValue(LV);
3845 
3846     QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
3847     Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true,
3848                                  SignedIndices, E->getExprLoc());
3849     return MakeAddrLValue(Addr, EltType, LV.getBaseInfo(),
3850                           CGM.getTBAAInfoForSubobject(LV, EltType));
3851   }
3852 
3853   LValueBaseInfo EltBaseInfo;
3854   TBAAAccessInfo EltTBAAInfo;
3855   Address Addr = Address::invalid();
3856   if (const VariableArrayType *vla =
3857            getContext().getAsVariableArrayType(E->getType())) {
3858     // The base must be a pointer, which is not an aggregate.  Emit
3859     // it.  It needs to be emitted first in case it's what captures
3860     // the VLA bounds.
3861     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3862     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3863 
3864     // The element count here is the total number of non-VLA elements.
3865     llvm::Value *numElements = getVLASize(vla).NumElts;
3866 
3867     // Effectively, the multiply by the VLA size is part of the GEP.
3868     // GEP indexes are signed, and scaling an index isn't permitted to
3869     // signed-overflow, so we use the same semantics for our explicit
3870     // multiply.  We suppress this if overflow is not undefined behavior.
3871     if (getLangOpts().isSignedOverflowDefined()) {
3872       Idx = Builder.CreateMul(Idx, numElements);
3873     } else {
3874       Idx = Builder.CreateNSWMul(Idx, numElements);
3875     }
3876 
3877     Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
3878                                  !getLangOpts().isSignedOverflowDefined(),
3879                                  SignedIndices, E->getExprLoc());
3880 
3881   } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
3882     // Indexing over an interface, as in "NSString *P; P[4];"
3883 
3884     // Emit the base pointer.
3885     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3886     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3887 
3888     CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);
3889     llvm::Value *InterfaceSizeVal =
3890         llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());
3891 
3892     llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal);
3893 
3894     // We don't necessarily build correct LLVM struct types for ObjC
3895     // interfaces, so we can't rely on GEP to do this scaling
3896     // correctly, so we need to cast to i8*.  FIXME: is this actually
3897     // true?  A lot of other things in the fragile ABI would break...
3898     llvm::Type *OrigBaseElemTy = Addr.getElementType();
3899     Addr = Builder.CreateElementBitCast(Addr, Int8Ty);
3900 
3901     // Do the GEP.
3902     CharUnits EltAlign =
3903       getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
3904     llvm::Value *EltPtr =
3905         emitArraySubscriptGEP(*this, Addr.getElementType(), Addr.getPointer(),
3906                               ScaledIdx, false, SignedIndices, E->getExprLoc());
3907     Addr = Address(EltPtr, Addr.getElementType(), EltAlign);
3908 
3909     // Cast back.
3910     Addr = Builder.CreateElementBitCast(Addr, OrigBaseElemTy);
3911   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
3912     // If this is A[i] where A is an array, the frontend will have decayed the
3913     // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
3914     // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
3915     // "gep x, i" here.  Emit one "gep A, 0, i".
3916     assert(Array->getType()->isArrayType() &&
3917            "Array to pointer decay must have array source type!");
3918     LValue ArrayLV;
3919     // For simple multidimensional array indexing, set the 'accessed' flag for
3920     // better bounds-checking of the base expression.
3921     if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
3922       ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
3923     else
3924       ArrayLV = EmitLValue(Array);
3925     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3926 
3927     // Propagate the alignment from the array itself to the result.
3928     QualType arrayType = Array->getType();
3929     Addr = emitArraySubscriptGEP(
3930         *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
3931         E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices,
3932         E->getExprLoc(), &arrayType, E->getBase());
3933     EltBaseInfo = ArrayLV.getBaseInfo();
3934     EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
3935   } else {
3936     // The base must be a pointer; emit it with an estimate of its alignment.
3937     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3938     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3939     QualType ptrType = E->getBase()->getType();
3940     Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
3941                                  !getLangOpts().isSignedOverflowDefined(),
3942                                  SignedIndices, E->getExprLoc(), &ptrType,
3943                                  E->getBase());
3944   }
3945 
3946   LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
3947 
3948   if (getLangOpts().ObjC &&
3949       getLangOpts().getGC() != LangOptions::NonGC) {
3950     LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
3951     setObjCGCLValueClass(getContext(), E, LV);
3952   }
3953   return LV;
3954 }
3955 
3956 LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
3957   assert(
3958       !E->isIncomplete() &&
3959       "incomplete matrix subscript expressions should be rejected during Sema");
3960   LValue Base = EmitLValue(E->getBase());
3961   llvm::Value *RowIdx = EmitScalarExpr(E->getRowIdx());
3962   llvm::Value *ColIdx = EmitScalarExpr(E->getColumnIdx());
3963   llvm::Value *NumRows = Builder.getIntN(
3964       RowIdx->getType()->getScalarSizeInBits(),
3965       E->getBase()->getType()->castAs<ConstantMatrixType>()->getNumRows());
3966   llvm::Value *FinalIdx =
3967       Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx);
3968   return LValue::MakeMatrixElt(
3969       MaybeConvertMatrixAddress(Base.getAddress(*this), *this), FinalIdx,
3970       E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo());
3971 }
3972 
3973 static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
3974                                        LValueBaseInfo &BaseInfo,
3975                                        TBAAAccessInfo &TBAAInfo,
3976                                        QualType BaseTy, QualType ElTy,
3977                                        bool IsLowerBound) {
3978   LValue BaseLVal;
3979   if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) {
3980     BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound);
3981     if (BaseTy->isArrayType()) {
3982       Address Addr = BaseLVal.getAddress(CGF);
3983       BaseInfo = BaseLVal.getBaseInfo();
3984 
3985       // If the array type was an incomplete type, we need to make sure
3986       // the decay ends up being the right type.
3987       llvm::Type *NewTy = CGF.ConvertType(BaseTy);
3988       Addr = CGF.Builder.CreateElementBitCast(Addr, NewTy);
3989 
3990       // Note that VLA pointers are always decayed, so we don't need to do
3991       // anything here.
3992       if (!BaseTy->isVariableArrayType()) {
3993         assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3994                "Expected pointer to array");
3995         Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3996       }
3997 
3998       return CGF.Builder.CreateElementBitCast(Addr,
3999                                               CGF.ConvertTypeForMem(ElTy));
4000     }
4001     LValueBaseInfo TypeBaseInfo;
4002     TBAAAccessInfo TypeTBAAInfo;
4003     CharUnits Align =
4004         CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo);
4005     BaseInfo.mergeForCast(TypeBaseInfo);
4006     TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo);
4007     return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)),
4008                    CGF.ConvertTypeForMem(ElTy), Align);
4009   }
4010   return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
4011 }
4012 
4013 LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
4014                                                 bool IsLowerBound) {
4015   QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(E->getBase());
4016   QualType ResultExprTy;
4017   if (auto *AT = getContext().getAsArrayType(BaseTy))
4018     ResultExprTy = AT->getElementType();
4019   else
4020     ResultExprTy = BaseTy->getPointeeType();
4021   llvm::Value *Idx = nullptr;
4022   if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
4023     // Requesting lower bound or upper bound, but without provided length and
4024     // without ':' symbol for the default length -> length = 1.
4025     // Idx = LowerBound ?: 0;
4026     if (auto *LowerBound = E->getLowerBound()) {
4027       Idx = Builder.CreateIntCast(
4028           EmitScalarExpr(LowerBound), IntPtrTy,
4029           LowerBound->getType()->hasSignedIntegerRepresentation());
4030     } else
4031       Idx = llvm::ConstantInt::getNullValue(IntPtrTy);
4032   } else {
4033     // Try to emit length or lower bound as constant. If this is possible, 1
4034     // is subtracted from constant length or lower bound. Otherwise, emit LLVM
4035     // IR (LB + Len) - 1.
4036     auto &C = CGM.getContext();
4037     auto *Length = E->getLength();
4038     llvm::APSInt ConstLength;
4039     if (Length) {
4040       // Idx = LowerBound + Length - 1;
4041       if (Optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) {
4042         ConstLength = CL->zextOrTrunc(PointerWidthInBits);
4043         Length = nullptr;
4044       }
4045       auto *LowerBound = E->getLowerBound();
4046       llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
4047       if (LowerBound) {
4048         if (Optional<llvm::APSInt> LB = LowerBound->getIntegerConstantExpr(C)) {
4049           ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits);
4050           LowerBound = nullptr;
4051         }
4052       }
4053       if (!Length)
4054         --ConstLength;
4055       else if (!LowerBound)
4056         --ConstLowerBound;
4057 
4058       if (Length || LowerBound) {
4059         auto *LowerBoundVal =
4060             LowerBound
4061                 ? Builder.CreateIntCast(
4062                       EmitScalarExpr(LowerBound), IntPtrTy,
4063                       LowerBound->getType()->hasSignedIntegerRepresentation())
4064                 : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound);
4065         auto *LengthVal =
4066             Length
4067                 ? Builder.CreateIntCast(
4068                       EmitScalarExpr(Length), IntPtrTy,
4069                       Length->getType()->hasSignedIntegerRepresentation())
4070                 : llvm::ConstantInt::get(IntPtrTy, ConstLength);
4071         Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",
4072                                 /*HasNUW=*/false,
4073                                 !getLangOpts().isSignedOverflowDefined());
4074         if (Length && LowerBound) {
4075           Idx = Builder.CreateSub(
4076               Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",
4077               /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4078         }
4079       } else
4080         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);
4081     } else {
4082       // Idx = ArraySize - 1;
4083       QualType ArrayTy = BaseTy->isPointerType()
4084                              ? E->getBase()->IgnoreParenImpCasts()->getType()
4085                              : BaseTy;
4086       if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) {
4087         Length = VAT->getSizeExpr();
4088         if (Optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) {
4089           ConstLength = *L;
4090           Length = nullptr;
4091         }
4092       } else {
4093         auto *CAT = C.getAsConstantArrayType(ArrayTy);
4094         ConstLength = CAT->getSize();
4095       }
4096       if (Length) {
4097         auto *LengthVal = Builder.CreateIntCast(
4098             EmitScalarExpr(Length), IntPtrTy,
4099             Length->getType()->hasSignedIntegerRepresentation());
4100         Idx = Builder.CreateSub(
4101             LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",
4102             /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4103       } else {
4104         ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
4105         --ConstLength;
4106         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength);
4107       }
4108     }
4109   }
4110   assert(Idx);
4111 
4112   Address EltPtr = Address::invalid();
4113   LValueBaseInfo BaseInfo;
4114   TBAAAccessInfo TBAAInfo;
4115   if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) {
4116     // The base must be a pointer, which is not an aggregate.  Emit
4117     // it.  It needs to be emitted first in case it's what captures
4118     // the VLA bounds.
4119     Address Base =
4120         emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo,
4121                                 BaseTy, VLA->getElementType(), IsLowerBound);
4122     // The element count here is the total number of non-VLA elements.
4123     llvm::Value *NumElements = getVLASize(VLA).NumElts;
4124 
4125     // Effectively, the multiply by the VLA size is part of the GEP.
4126     // GEP indexes are signed, and scaling an index isn't permitted to
4127     // signed-overflow, so we use the same semantics for our explicit
4128     // multiply.  We suppress this if overflow is not undefined behavior.
4129     if (getLangOpts().isSignedOverflowDefined())
4130       Idx = Builder.CreateMul(Idx, NumElements);
4131     else
4132       Idx = Builder.CreateNSWMul(Idx, NumElements);
4133     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),
4134                                    !getLangOpts().isSignedOverflowDefined(),
4135                                    /*signedIndices=*/false, E->getExprLoc());
4136   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
4137     // If this is A[i] where A is an array, the frontend will have decayed the
4138     // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
4139     // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4140     // "gep x, i" here.  Emit one "gep A, 0, i".
4141     assert(Array->getType()->isArrayType() &&
4142            "Array to pointer decay must have array source type!");
4143     LValue ArrayLV;
4144     // For simple multidimensional array indexing, set the 'accessed' flag for
4145     // better bounds-checking of the base expression.
4146     if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4147       ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
4148     else
4149       ArrayLV = EmitLValue(Array);
4150 
4151     // Propagate the alignment from the array itself to the result.
4152     EltPtr = emitArraySubscriptGEP(
4153         *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
4154         ResultExprTy, !getLangOpts().isSignedOverflowDefined(),
4155         /*signedIndices=*/false, E->getExprLoc());
4156     BaseInfo = ArrayLV.getBaseInfo();
4157     TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
4158   } else {
4159     Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo,
4160                                            TBAAInfo, BaseTy, ResultExprTy,
4161                                            IsLowerBound);
4162     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
4163                                    !getLangOpts().isSignedOverflowDefined(),
4164                                    /*signedIndices=*/false, E->getExprLoc());
4165   }
4166 
4167   return MakeAddrLValue(EltPtr, ResultExprTy, BaseInfo, TBAAInfo);
4168 }
4169 
4170 LValue CodeGenFunction::
4171 EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
4172   // Emit the base vector as an l-value.
4173   LValue Base;
4174 
4175   // ExtVectorElementExpr's base can either be a vector or pointer to vector.
4176   if (E->isArrow()) {
4177     // If it is a pointer to a vector, emit the address and form an lvalue with
4178     // it.
4179     LValueBaseInfo BaseInfo;
4180     TBAAAccessInfo TBAAInfo;
4181     Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo);
4182     const auto *PT = E->getBase()->getType()->castAs<PointerType>();
4183     Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo);
4184     Base.getQuals().removeObjCGCAttr();
4185   } else if (E->getBase()->isGLValue()) {
4186     // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
4187     // emit the base as an lvalue.
4188     assert(E->getBase()->getType()->isVectorType());
4189     Base = EmitLValue(E->getBase());
4190   } else {
4191     // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
4192     assert(E->getBase()->getType()->isVectorType() &&
4193            "Result must be a vector");
4194     llvm::Value *Vec = EmitScalarExpr(E->getBase());
4195 
4196     // Store the vector to memory (because LValue wants an address).
4197     Address VecMem = CreateMemTemp(E->getBase()->getType());
4198     Builder.CreateStore(Vec, VecMem);
4199     Base = MakeAddrLValue(VecMem, E->getBase()->getType(),
4200                           AlignmentSource::Decl);
4201   }
4202 
4203   QualType type =
4204     E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
4205 
4206   // Encode the element access list into a vector of unsigned indices.
4207   SmallVector<uint32_t, 4> Indices;
4208   E->getEncodedElementAccess(Indices);
4209 
4210   if (Base.isSimple()) {
4211     llvm::Constant *CV =
4212         llvm::ConstantDataVector::get(getLLVMContext(), Indices);
4213     return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type,
4214                                     Base.getBaseInfo(), TBAAAccessInfo());
4215   }
4216   assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
4217 
4218   llvm::Constant *BaseElts = Base.getExtVectorElts();
4219   SmallVector<llvm::Constant *, 4> CElts;
4220 
4221   for (unsigned i = 0, e = Indices.size(); i != e; ++i)
4222     CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
4223   llvm::Constant *CV = llvm::ConstantVector::get(CElts);
4224   return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,
4225                                   Base.getBaseInfo(), TBAAAccessInfo());
4226 }
4227 
4228 LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
4229   if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, E)) {
4230     EmitIgnoredExpr(E->getBase());
4231     return EmitDeclRefLValue(DRE);
4232   }
4233 
4234   Expr *BaseExpr = E->getBase();
4235   // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
4236   LValue BaseLV;
4237   if (E->isArrow()) {
4238     LValueBaseInfo BaseInfo;
4239     TBAAAccessInfo TBAAInfo;
4240     Address Addr = EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
4241     QualType PtrTy = BaseExpr->getType()->getPointeeType();
4242     SanitizerSet SkippedChecks;
4243     bool IsBaseCXXThis = IsWrappedCXXThis(BaseExpr);
4244     if (IsBaseCXXThis)
4245       SkippedChecks.set(SanitizerKind::Alignment, true);
4246     if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
4247       SkippedChecks.set(SanitizerKind::Null, true);
4248     EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy,
4249                   /*Alignment=*/CharUnits::Zero(), SkippedChecks);
4250     BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
4251   } else
4252     BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
4253 
4254   NamedDecl *ND = E->getMemberDecl();
4255   if (auto *Field = dyn_cast<FieldDecl>(ND)) {
4256     LValue LV = EmitLValueForField(BaseLV, Field);
4257     setObjCGCLValueClass(getContext(), E, LV);
4258     if (getLangOpts().OpenMP) {
4259       // If the member was explicitly marked as nontemporal, mark it as
4260       // nontemporal. If the base lvalue is marked as nontemporal, mark access
4261       // to children as nontemporal too.
4262       if ((IsWrappedCXXThis(BaseExpr) &&
4263            CGM.getOpenMPRuntime().isNontemporalDecl(Field)) ||
4264           BaseLV.isNontemporal())
4265         LV.setNontemporal(/*Value=*/true);
4266     }
4267     return LV;
4268   }
4269 
4270   if (const auto *FD = dyn_cast<FunctionDecl>(ND))
4271     return EmitFunctionDeclLValue(*this, E, FD);
4272 
4273   llvm_unreachable("Unhandled member declaration!");
4274 }
4275 
4276 /// Given that we are currently emitting a lambda, emit an l-value for
4277 /// one of its members.
4278 LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
4279   if (CurCodeDecl) {
4280     assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent()->isLambda());
4281     assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent() == Field->getParent());
4282   }
4283   QualType LambdaTagType =
4284     getContext().getTagDeclType(Field->getParent());
4285   LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue, LambdaTagType);
4286   return EmitLValueForField(LambdaLV, Field);
4287 }
4288 
4289 /// Get the field index in the debug info. The debug info structure/union
4290 /// will ignore the unnamed bitfields.
4291 unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
4292                                              unsigned FieldIndex) {
4293   unsigned I = 0, Skipped = 0;
4294 
4295   for (auto F : Rec->getDefinition()->fields()) {
4296     if (I == FieldIndex)
4297       break;
4298     if (F->isUnnamedBitfield())
4299       Skipped++;
4300     I++;
4301   }
4302 
4303   return FieldIndex - Skipped;
4304 }
4305 
4306 /// Get the address of a zero-sized field within a record. The resulting
4307 /// address doesn't necessarily have the right type.
4308 static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
4309                                        const FieldDecl *Field) {
4310   CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
4311       CGF.getContext().getFieldOffset(Field));
4312   if (Offset.isZero())
4313     return Base;
4314   Base = CGF.Builder.CreateElementBitCast(Base, CGF.Int8Ty);
4315   return CGF.Builder.CreateConstInBoundsByteGEP(Base, Offset);
4316 }
4317 
4318 /// Drill down to the storage of a field without walking into
4319 /// reference types.
4320 ///
4321 /// The resulting address doesn't necessarily have the right type.
4322 static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
4323                                       const FieldDecl *field) {
4324   if (field->isZeroSize(CGF.getContext()))
4325     return emitAddrOfZeroSizeField(CGF, base, field);
4326 
4327   const RecordDecl *rec = field->getParent();
4328 
4329   unsigned idx =
4330     CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4331 
4332   return CGF.Builder.CreateStructGEP(base, idx, field->getName());
4333 }
4334 
4335 static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
4336                                         Address addr, const FieldDecl *field) {
4337   const RecordDecl *rec = field->getParent();
4338   llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
4339       base.getType(), rec->getLocation());
4340 
4341   unsigned idx =
4342       CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4343 
4344   return CGF.Builder.CreatePreserveStructAccessIndex(
4345       addr, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo);
4346 }
4347 
4348 static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
4349   const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
4350   if (!RD)
4351     return false;
4352 
4353   if (RD->isDynamicClass())
4354     return true;
4355 
4356   for (const auto &Base : RD->bases())
4357     if (hasAnyVptr(Base.getType(), Context))
4358       return true;
4359 
4360   for (const FieldDecl *Field : RD->fields())
4361     if (hasAnyVptr(Field->getType(), Context))
4362       return true;
4363 
4364   return false;
4365 }
4366 
4367 LValue CodeGenFunction::EmitLValueForField(LValue base,
4368                                            const FieldDecl *field) {
4369   LValueBaseInfo BaseInfo = base.getBaseInfo();
4370 
4371   if (field->isBitField()) {
4372     const CGRecordLayout &RL =
4373         CGM.getTypes().getCGRecordLayout(field->getParent());
4374     const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
4375     const bool UseVolatile = isAAPCS(CGM.getTarget()) &&
4376                              CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
4377                              Info.VolatileStorageSize != 0 &&
4378                              field->getType()
4379                                  .withCVRQualifiers(base.getVRQualifiers())
4380                                  .isVolatileQualified();
4381     Address Addr = base.getAddress(*this);
4382     unsigned Idx = RL.getLLVMFieldNo(field);
4383     const RecordDecl *rec = field->getParent();
4384     if (!UseVolatile) {
4385       if (!IsInPreservedAIRegion &&
4386           (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4387         if (Idx != 0)
4388           // For structs, we GEP to the field that the record layout suggests.
4389           Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
4390       } else {
4391         llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
4392             getContext().getRecordType(rec), rec->getLocation());
4393         Addr = Builder.CreatePreserveStructAccessIndex(
4394             Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()),
4395             DbgInfo);
4396       }
4397     }
4398     const unsigned SS =
4399         UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
4400     // Get the access type.
4401     llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS);
4402     if (Addr.getElementType() != FieldIntTy)
4403       Addr = Builder.CreateElementBitCast(Addr, FieldIntTy);
4404     if (UseVolatile) {
4405       const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
4406       if (VolatileOffset)
4407         Addr = Builder.CreateConstInBoundsGEP(Addr, VolatileOffset);
4408     }
4409 
4410     QualType fieldType =
4411         field->getType().withCVRQualifiers(base.getVRQualifiers());
4412     // TODO: Support TBAA for bit fields.
4413     LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
4414     return LValue::MakeBitfield(Addr, Info, fieldType, FieldBaseInfo,
4415                                 TBAAAccessInfo());
4416   }
4417 
4418   // Fields of may-alias structures are may-alias themselves.
4419   // FIXME: this should get propagated down through anonymous structs
4420   // and unions.
4421   QualType FieldType = field->getType();
4422   const RecordDecl *rec = field->getParent();
4423   AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
4424   LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));
4425   TBAAAccessInfo FieldTBAAInfo;
4426   if (base.getTBAAInfo().isMayAlias() ||
4427           rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
4428     FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4429   } else if (rec->isUnion()) {
4430     // TODO: Support TBAA for unions.
4431     FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4432   } else {
4433     // If no base type been assigned for the base access, then try to generate
4434     // one for this base lvalue.
4435     FieldTBAAInfo = base.getTBAAInfo();
4436     if (!FieldTBAAInfo.BaseType) {
4437         FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(base.getType());
4438         assert(!FieldTBAAInfo.Offset &&
4439                "Nonzero offset for an access with no base type!");
4440     }
4441 
4442     // Adjust offset to be relative to the base type.
4443     const ASTRecordLayout &Layout =
4444         getContext().getASTRecordLayout(field->getParent());
4445     unsigned CharWidth = getContext().getCharWidth();
4446     if (FieldTBAAInfo.BaseType)
4447       FieldTBAAInfo.Offset +=
4448           Layout.getFieldOffset(field->getFieldIndex()) / CharWidth;
4449 
4450     // Update the final access type and size.
4451     FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(FieldType);
4452     FieldTBAAInfo.Size =
4453         getContext().getTypeSizeInChars(FieldType).getQuantity();
4454   }
4455 
4456   Address addr = base.getAddress(*this);
4457   if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) {
4458     if (CGM.getCodeGenOpts().StrictVTablePointers &&
4459         ClassDef->isDynamicClass()) {
4460       // Getting to any field of dynamic object requires stripping dynamic
4461       // information provided by invariant.group.  This is because accessing
4462       // fields may leak the real address of dynamic object, which could result
4463       // in miscompilation when leaked pointer would be compared.
4464       auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
4465       addr = Address(stripped, addr.getElementType(), addr.getAlignment());
4466     }
4467   }
4468 
4469   unsigned RecordCVR = base.getVRQualifiers();
4470   if (rec->isUnion()) {
4471     // For unions, there is no pointer adjustment.
4472     if (CGM.getCodeGenOpts().StrictVTablePointers &&
4473         hasAnyVptr(FieldType, getContext()))
4474       // Because unions can easily skip invariant.barriers, we need to add
4475       // a barrier every time CXXRecord field with vptr is referenced.
4476       addr = Builder.CreateLaunderInvariantGroup(addr);
4477 
4478     if (IsInPreservedAIRegion ||
4479         (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4480       // Remember the original union field index
4481       llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),
4482           rec->getLocation());
4483       addr = Address(
4484           Builder.CreatePreserveUnionAccessIndex(
4485               addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
4486           addr.getElementType(), addr.getAlignment());
4487     }
4488 
4489     if (FieldType->isReferenceType())
4490       addr = Builder.CreateElementBitCast(
4491           addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName());
4492   } else {
4493     if (!IsInPreservedAIRegion &&
4494         (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
4495       // For structs, we GEP to the field that the record layout suggests.
4496       addr = emitAddrOfFieldStorage(*this, addr, field);
4497     else
4498       // Remember the original struct field index
4499       addr = emitPreserveStructAccess(*this, base, addr, field);
4500   }
4501 
4502   // If this is a reference field, load the reference right now.
4503   if (FieldType->isReferenceType()) {
4504     LValue RefLVal =
4505         MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4506     if (RecordCVR & Qualifiers::Volatile)
4507       RefLVal.getQuals().addVolatile();
4508     addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo);
4509 
4510     // Qualifiers on the struct don't apply to the referencee.
4511     RecordCVR = 0;
4512     FieldType = FieldType->getPointeeType();
4513   }
4514 
4515   // Make sure that the address is pointing to the right type.  This is critical
4516   // for both unions and structs.  A union needs a bitcast, a struct element
4517   // will need a bitcast if the LLVM type laid out doesn't match the desired
4518   // type.
4519   addr = Builder.CreateElementBitCast(
4520       addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName());
4521 
4522   if (field->hasAttr<AnnotateAttr>())
4523     addr = EmitFieldAnnotations(field, addr);
4524 
4525   LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4526   LV.getQuals().addCVRQualifiers(RecordCVR);
4527 
4528   // __weak attribute on a field is ignored.
4529   if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
4530     LV.getQuals().removeObjCGCAttr();
4531 
4532   return LV;
4533 }
4534 
4535 LValue
4536 CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
4537                                                   const FieldDecl *Field) {
4538   QualType FieldType = Field->getType();
4539 
4540   if (!FieldType->isReferenceType())
4541     return EmitLValueForField(Base, Field);
4542 
4543   Address V = emitAddrOfFieldStorage(*this, Base.getAddress(*this), Field);
4544 
4545   // Make sure that the address is pointing to the right type.
4546   llvm::Type *llvmType = ConvertTypeForMem(FieldType);
4547   V = Builder.CreateElementBitCast(V, llvmType, Field->getName());
4548 
4549   // TODO: Generate TBAA information that describes this access as a structure
4550   // member access and not just an access to an object of the field's type. This
4551   // should be similar to what we do in EmitLValueForField().
4552   LValueBaseInfo BaseInfo = Base.getBaseInfo();
4553   AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
4554   LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(FieldAlignSource));
4555   return MakeAddrLValue(V, FieldType, FieldBaseInfo,
4556                         CGM.getTBAAInfoForSubobject(Base, FieldType));
4557 }
4558 
4559 LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
4560   if (E->isFileScope()) {
4561     ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
4562     return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl);
4563   }
4564   if (E->getType()->isVariablyModifiedType())
4565     // make sure to emit the VLA size.
4566     EmitVariablyModifiedType(E->getType());
4567 
4568   Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
4569   const Expr *InitExpr = E->getInitializer();
4570   LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl);
4571 
4572   EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
4573                    /*Init*/ true);
4574 
4575   // Block-scope compound literals are destroyed at the end of the enclosing
4576   // scope in C.
4577   if (!getLangOpts().CPlusPlus)
4578     if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
4579       pushLifetimeExtendedDestroy(getCleanupKind(DtorKind), DeclPtr,
4580                                   E->getType(), getDestroyer(DtorKind),
4581                                   DtorKind & EHCleanup);
4582 
4583   return Result;
4584 }
4585 
4586 LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
4587   if (!E->isGLValue())
4588     // Initializing an aggregate temporary in C++11: T{...}.
4589     return EmitAggExprToLValue(E);
4590 
4591   // An lvalue initializer list must be initializing a reference.
4592   assert(E->isTransparent() && "non-transparent glvalue init list");
4593   return EmitLValue(E->getInit(0));
4594 }
4595 
4596 /// Emit the operand of a glvalue conditional operator. This is either a glvalue
4597 /// or a (possibly-parenthesized) throw-expression. If this is a throw, no
4598 /// LValue is returned and the current block has been terminated.
4599 static Optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
4600                                                     const Expr *Operand) {
4601   if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Operand->IgnoreParens())) {
4602     CGF.EmitCXXThrowExpr(ThrowExpr, /*KeepInsertionPoint*/false);
4603     return None;
4604   }
4605 
4606   return CGF.EmitLValue(Operand);
4607 }
4608 
4609 namespace {
4610 // Handle the case where the condition is a constant evaluatable simple integer,
4611 // which means we don't have to separately handle the true/false blocks.
4612 llvm::Optional<LValue> HandleConditionalOperatorLValueSimpleCase(
4613     CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
4614   const Expr *condExpr = E->getCond();
4615   bool CondExprBool;
4616   if (CGF.ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
4617     const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
4618     if (!CondExprBool)
4619       std::swap(Live, Dead);
4620 
4621     if (!CGF.ContainsLabel(Dead)) {
4622       // If the true case is live, we need to track its region.
4623       if (CondExprBool)
4624         CGF.incrementProfileCounter(E);
4625       // If a throw expression we emit it and return an undefined lvalue
4626       // because it can't be used.
4627       if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Live->IgnoreParens())) {
4628         CGF.EmitCXXThrowExpr(ThrowExpr);
4629         llvm::Type *ElemTy = CGF.ConvertType(Dead->getType());
4630         llvm::Type *Ty = llvm::PointerType::getUnqual(ElemTy);
4631         return CGF.MakeAddrLValue(
4632             Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()),
4633             Dead->getType());
4634       }
4635       return CGF.EmitLValue(Live);
4636     }
4637   }
4638   return llvm::None;
4639 }
4640 struct ConditionalInfo {
4641   llvm::BasicBlock *lhsBlock, *rhsBlock;
4642   Optional<LValue> LHS, RHS;
4643 };
4644 
4645 // Create and generate the 3 blocks for a conditional operator.
4646 // Leaves the 'current block' in the continuation basic block.
4647 template<typename FuncTy>
4648 ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
4649                                       const AbstractConditionalOperator *E,
4650                                       const FuncTy &BranchGenFunc) {
4651   ConditionalInfo Info{CGF.createBasicBlock("cond.true"),
4652                        CGF.createBasicBlock("cond.false"), llvm::None,
4653                        llvm::None};
4654   llvm::BasicBlock *endBlock = CGF.createBasicBlock("cond.end");
4655 
4656   CodeGenFunction::ConditionalEvaluation eval(CGF);
4657   CGF.EmitBranchOnBoolExpr(E->getCond(), Info.lhsBlock, Info.rhsBlock,
4658                            CGF.getProfileCount(E));
4659 
4660   // Any temporaries created here are conditional.
4661   CGF.EmitBlock(Info.lhsBlock);
4662   CGF.incrementProfileCounter(E);
4663   eval.begin(CGF);
4664   Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
4665   eval.end(CGF);
4666   Info.lhsBlock = CGF.Builder.GetInsertBlock();
4667 
4668   if (Info.LHS)
4669     CGF.Builder.CreateBr(endBlock);
4670 
4671   // Any temporaries created here are conditional.
4672   CGF.EmitBlock(Info.rhsBlock);
4673   eval.begin(CGF);
4674   Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
4675   eval.end(CGF);
4676   Info.rhsBlock = CGF.Builder.GetInsertBlock();
4677   CGF.EmitBlock(endBlock);
4678 
4679   return Info;
4680 }
4681 } // namespace
4682 
4683 void CodeGenFunction::EmitIgnoredConditionalOperator(
4684     const AbstractConditionalOperator *E) {
4685   if (!E->isGLValue()) {
4686     // ?: here should be an aggregate.
4687     assert(hasAggregateEvaluationKind(E->getType()) &&
4688            "Unexpected conditional operator!");
4689     return (void)EmitAggExprToLValue(E);
4690   }
4691 
4692   OpaqueValueMapping binding(*this, E);
4693   if (HandleConditionalOperatorLValueSimpleCase(*this, E))
4694     return;
4695 
4696   EmitConditionalBlocks(*this, E, [](CodeGenFunction &CGF, const Expr *E) {
4697     CGF.EmitIgnoredExpr(E);
4698     return LValue{};
4699   });
4700 }
4701 LValue CodeGenFunction::EmitConditionalOperatorLValue(
4702     const AbstractConditionalOperator *expr) {
4703   if (!expr->isGLValue()) {
4704     // ?: here should be an aggregate.
4705     assert(hasAggregateEvaluationKind(expr->getType()) &&
4706            "Unexpected conditional operator!");
4707     return EmitAggExprToLValue(expr);
4708   }
4709 
4710   OpaqueValueMapping binding(*this, expr);
4711   if (llvm::Optional<LValue> Res =
4712           HandleConditionalOperatorLValueSimpleCase(*this, expr))
4713     return *Res;
4714 
4715   ConditionalInfo Info = EmitConditionalBlocks(
4716       *this, expr, [](CodeGenFunction &CGF, const Expr *E) {
4717         return EmitLValueOrThrowExpression(CGF, E);
4718       });
4719 
4720   if ((Info.LHS && !Info.LHS->isSimple()) ||
4721       (Info.RHS && !Info.RHS->isSimple()))
4722     return EmitUnsupportedLValue(expr, "conditional operator");
4723 
4724   if (Info.LHS && Info.RHS) {
4725     Address lhsAddr = Info.LHS->getAddress(*this);
4726     Address rhsAddr = Info.RHS->getAddress(*this);
4727     llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue");
4728     phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock);
4729     phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock);
4730     Address result(phi, lhsAddr.getElementType(),
4731                    std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
4732     AlignmentSource alignSource =
4733         std::max(Info.LHS->getBaseInfo().getAlignmentSource(),
4734                  Info.RHS->getBaseInfo().getAlignmentSource());
4735     TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
4736         Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo());
4737     return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),
4738                           TBAAInfo);
4739   } else {
4740     assert((Info.LHS || Info.RHS) &&
4741            "both operands of glvalue conditional are throw-expressions?");
4742     return Info.LHS ? *Info.LHS : *Info.RHS;
4743   }
4744 }
4745 
4746 /// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
4747 /// type. If the cast is to a reference, we can have the usual lvalue result,
4748 /// otherwise if a cast is needed by the code generator in an lvalue context,
4749 /// then it must mean that we need the address of an aggregate in order to
4750 /// access one of its members.  This can happen for all the reasons that casts
4751 /// are permitted with aggregate result, including noop aggregate casts, and
4752 /// cast from scalar to union.
4753 LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
4754   switch (E->getCastKind()) {
4755   case CK_ToVoid:
4756   case CK_BitCast:
4757   case CK_LValueToRValueBitCast:
4758   case CK_ArrayToPointerDecay:
4759   case CK_FunctionToPointerDecay:
4760   case CK_NullToMemberPointer:
4761   case CK_NullToPointer:
4762   case CK_IntegralToPointer:
4763   case CK_PointerToIntegral:
4764   case CK_PointerToBoolean:
4765   case CK_VectorSplat:
4766   case CK_IntegralCast:
4767   case CK_BooleanToSignedIntegral:
4768   case CK_IntegralToBoolean:
4769   case CK_IntegralToFloating:
4770   case CK_FloatingToIntegral:
4771   case CK_FloatingToBoolean:
4772   case CK_FloatingCast:
4773   case CK_FloatingRealToComplex:
4774   case CK_FloatingComplexToReal:
4775   case CK_FloatingComplexToBoolean:
4776   case CK_FloatingComplexCast:
4777   case CK_FloatingComplexToIntegralComplex:
4778   case CK_IntegralRealToComplex:
4779   case CK_IntegralComplexToReal:
4780   case CK_IntegralComplexToBoolean:
4781   case CK_IntegralComplexCast:
4782   case CK_IntegralComplexToFloatingComplex:
4783   case CK_DerivedToBaseMemberPointer:
4784   case CK_BaseToDerivedMemberPointer:
4785   case CK_MemberPointerToBoolean:
4786   case CK_ReinterpretMemberPointer:
4787   case CK_AnyPointerToBlockPointerCast:
4788   case CK_ARCProduceObject:
4789   case CK_ARCConsumeObject:
4790   case CK_ARCReclaimReturnedObject:
4791   case CK_ARCExtendBlockObject:
4792   case CK_CopyAndAutoreleaseBlockObject:
4793   case CK_IntToOCLSampler:
4794   case CK_FloatingToFixedPoint:
4795   case CK_FixedPointToFloating:
4796   case CK_FixedPointCast:
4797   case CK_FixedPointToBoolean:
4798   case CK_FixedPointToIntegral:
4799   case CK_IntegralToFixedPoint:
4800   case CK_MatrixCast:
4801     return EmitUnsupportedLValue(E, "unexpected cast lvalue");
4802 
4803   case CK_Dependent:
4804     llvm_unreachable("dependent cast kind in IR gen!");
4805 
4806   case CK_BuiltinFnToFnPtr:
4807     llvm_unreachable("builtin functions are handled elsewhere");
4808 
4809   // These are never l-values; just use the aggregate emission code.
4810   case CK_NonAtomicToAtomic:
4811   case CK_AtomicToNonAtomic:
4812     return EmitAggExprToLValue(E);
4813 
4814   case CK_Dynamic: {
4815     LValue LV = EmitLValue(E->getSubExpr());
4816     Address V = LV.getAddress(*this);
4817     const auto *DCE = cast<CXXDynamicCastExpr>(E);
4818     return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType());
4819   }
4820 
4821   case CK_ConstructorConversion:
4822   case CK_UserDefinedConversion:
4823   case CK_CPointerToObjCPointerCast:
4824   case CK_BlockPointerToObjCPointerCast:
4825   case CK_LValueToRValue:
4826     return EmitLValue(E->getSubExpr());
4827 
4828   case CK_NoOp: {
4829     // CK_NoOp can model a qualification conversion, which can remove an array
4830     // bound and change the IR type.
4831     // FIXME: Once pointee types are removed from IR, remove this.
4832     LValue LV = EmitLValue(E->getSubExpr());
4833     if (LV.isSimple()) {
4834       Address V = LV.getAddress(*this);
4835       if (V.isValid()) {
4836         llvm::Type *T = ConvertTypeForMem(E->getType());
4837         if (V.getElementType() != T)
4838           LV.setAddress(Builder.CreateElementBitCast(V, T));
4839       }
4840     }
4841     return LV;
4842   }
4843 
4844   case CK_UncheckedDerivedToBase:
4845   case CK_DerivedToBase: {
4846     const auto *DerivedClassTy =
4847         E->getSubExpr()->getType()->castAs<RecordType>();
4848     auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
4849 
4850     LValue LV = EmitLValue(E->getSubExpr());
4851     Address This = LV.getAddress(*this);
4852 
4853     // Perform the derived-to-base conversion
4854     Address Base = GetAddressOfBaseClass(
4855         This, DerivedClassDecl, E->path_begin(), E->path_end(),
4856         /*NullCheckValue=*/false, E->getExprLoc());
4857 
4858     // TODO: Support accesses to members of base classes in TBAA. For now, we
4859     // conservatively pretend that the complete object is of the base class
4860     // type.
4861     return MakeAddrLValue(Base, E->getType(), LV.getBaseInfo(),
4862                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4863   }
4864   case CK_ToUnion:
4865     return EmitAggExprToLValue(E);
4866   case CK_BaseToDerived: {
4867     const auto *DerivedClassTy = E->getType()->castAs<RecordType>();
4868     auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
4869 
4870     LValue LV = EmitLValue(E->getSubExpr());
4871 
4872     // Perform the base-to-derived conversion
4873     Address Derived = GetAddressOfDerivedClass(
4874         LV.getAddress(*this), DerivedClassDecl, E->path_begin(), E->path_end(),
4875         /*NullCheckValue=*/false);
4876 
4877     // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
4878     // performed and the object is not of the derived type.
4879     if (sanitizePerformTypeCheck())
4880       EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(),
4881                     Derived.getPointer(), E->getType());
4882 
4883     if (SanOpts.has(SanitizerKind::CFIDerivedCast))
4884       EmitVTablePtrCheckForCast(E->getType(), Derived,
4885                                 /*MayBeNull=*/false, CFITCK_DerivedCast,
4886                                 E->getBeginLoc());
4887 
4888     return MakeAddrLValue(Derived, E->getType(), LV.getBaseInfo(),
4889                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4890   }
4891   case CK_LValueBitCast: {
4892     // This must be a reinterpret_cast (or c-style equivalent).
4893     const auto *CE = cast<ExplicitCastExpr>(E);
4894 
4895     CGM.EmitExplicitCastExprType(CE, this);
4896     LValue LV = EmitLValue(E->getSubExpr());
4897     Address V = Builder.CreateElementBitCast(
4898         LV.getAddress(*this),
4899         ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType()));
4900 
4901     if (SanOpts.has(SanitizerKind::CFIUnrelatedCast))
4902       EmitVTablePtrCheckForCast(E->getType(), V,
4903                                 /*MayBeNull=*/false, CFITCK_UnrelatedCast,
4904                                 E->getBeginLoc());
4905 
4906     return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
4907                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4908   }
4909   case CK_AddressSpaceConversion: {
4910     LValue LV = EmitLValue(E->getSubExpr());
4911     QualType DestTy = getContext().getPointerType(E->getType());
4912     llvm::Value *V = getTargetHooks().performAddrSpaceCast(
4913         *this, LV.getPointer(*this),
4914         E->getSubExpr()->getType().getAddressSpace(),
4915         E->getType().getAddressSpace(), ConvertType(DestTy));
4916     return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()),
4917                                   LV.getAddress(*this).getAlignment()),
4918                           E->getType(), LV.getBaseInfo(), LV.getTBAAInfo());
4919   }
4920   case CK_ObjCObjectLValueCast: {
4921     LValue LV = EmitLValue(E->getSubExpr());
4922     Address V = Builder.CreateElementBitCast(LV.getAddress(*this),
4923                                              ConvertType(E->getType()));
4924     return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
4925                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4926   }
4927   case CK_ZeroToOCLOpaqueType:
4928     llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
4929   }
4930 
4931   llvm_unreachable("Unhandled lvalue cast kind?");
4932 }
4933 
4934 LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
4935   assert(OpaqueValueMappingData::shouldBindAsLValue(e));
4936   return getOrCreateOpaqueLValueMapping(e);
4937 }
4938 
4939 LValue
4940 CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
4941   assert(OpaqueValueMapping::shouldBindAsLValue(e));
4942 
4943   llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
4944       it = OpaqueLValues.find(e);
4945 
4946   if (it != OpaqueLValues.end())
4947     return it->second;
4948 
4949   assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
4950   return EmitLValue(e->getSourceExpr());
4951 }
4952 
4953 RValue
4954 CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
4955   assert(!OpaqueValueMapping::shouldBindAsLValue(e));
4956 
4957   llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
4958       it = OpaqueRValues.find(e);
4959 
4960   if (it != OpaqueRValues.end())
4961     return it->second;
4962 
4963   assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
4964   return EmitAnyExpr(e->getSourceExpr());
4965 }
4966 
4967 RValue CodeGenFunction::EmitRValueForField(LValue LV,
4968                                            const FieldDecl *FD,
4969                                            SourceLocation Loc) {
4970   QualType FT = FD->getType();
4971   LValue FieldLV = EmitLValueForField(LV, FD);
4972   switch (getEvaluationKind(FT)) {
4973   case TEK_Complex:
4974     return RValue::getComplex(EmitLoadOfComplex(FieldLV, Loc));
4975   case TEK_Aggregate:
4976     return FieldLV.asAggregateRValue(*this);
4977   case TEK_Scalar:
4978     // This routine is used to load fields one-by-one to perform a copy, so
4979     // don't load reference fields.
4980     if (FD->getType()->isReferenceType())
4981       return RValue::get(FieldLV.getPointer(*this));
4982     // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
4983     // primitive load.
4984     if (FieldLV.isBitField())
4985       return EmitLoadOfLValue(FieldLV, Loc);
4986     return RValue::get(EmitLoadOfScalar(FieldLV, Loc));
4987   }
4988   llvm_unreachable("bad evaluation kind");
4989 }
4990 
4991 //===--------------------------------------------------------------------===//
4992 //                             Expression Emission
4993 //===--------------------------------------------------------------------===//
4994 
4995 RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
4996                                      ReturnValueSlot ReturnValue) {
4997   // Builtins never have block type.
4998   if (E->getCallee()->getType()->isBlockPointerType())
4999     return EmitBlockCallExpr(E, ReturnValue);
5000 
5001   if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E))
5002     return EmitCXXMemberCallExpr(CE, ReturnValue);
5003 
5004   if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E))
5005     return EmitCUDAKernelCallExpr(CE, ReturnValue);
5006 
5007   if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E))
5008     if (const CXXMethodDecl *MD =
5009           dyn_cast_or_null<CXXMethodDecl>(CE->getCalleeDecl()))
5010       return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
5011 
5012   CGCallee callee = EmitCallee(E->getCallee());
5013 
5014   if (callee.isBuiltin()) {
5015     return EmitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(),
5016                            E, ReturnValue);
5017   }
5018 
5019   if (callee.isPseudoDestructor()) {
5020     return EmitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr());
5021   }
5022 
5023   return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue);
5024 }
5025 
5026 /// Emit a CallExpr without considering whether it might be a subclass.
5027 RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
5028                                            ReturnValueSlot ReturnValue) {
5029   CGCallee Callee = EmitCallee(E->getCallee());
5030   return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue);
5031 }
5032 
5033 // Detect the unusual situation where an inline version is shadowed by a
5034 // non-inline version. In that case we should pick the external one
5035 // everywhere. That's GCC behavior too.
5036 static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
5037   for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
5038     if (!PD->isInlineBuiltinDeclaration())
5039       return false;
5040   return true;
5041 }
5042 
5043 static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
5044   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
5045 
5046   if (auto builtinID = FD->getBuiltinID()) {
5047     std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
5048     std::string NoBuiltins = "no-builtins";
5049     std::string FDInlineName = (FD->getName() + ".inline").str();
5050 
5051     bool IsPredefinedLibFunction =
5052         CGF.getContext().BuiltinInfo.isPredefinedLibFunction(builtinID);
5053     bool HasAttributeNoBuiltin =
5054         CGF.CurFn->getAttributes().hasFnAttr(NoBuiltinFD) ||
5055         CGF.CurFn->getAttributes().hasFnAttr(NoBuiltins);
5056 
5057     // When directing calling an inline builtin, call it through it's mangled
5058     // name to make it clear it's not the actual builtin.
5059     if (CGF.CurFn->getName() != FDInlineName &&
5060         OnlyHasInlineBuiltinDeclaration(FD)) {
5061       llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
5062       llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr);
5063       llvm::Module *M = Fn->getParent();
5064       llvm::Function *Clone = M->getFunction(FDInlineName);
5065       if (!Clone) {
5066         Clone = llvm::Function::Create(Fn->getFunctionType(),
5067                                        llvm::GlobalValue::InternalLinkage,
5068                                        Fn->getAddressSpace(), FDInlineName, M);
5069         Clone->addFnAttr(llvm::Attribute::AlwaysInline);
5070       }
5071       return CGCallee::forDirect(Clone, GD);
5072     }
5073 
5074     // Replaceable builtins provide their own implementation of a builtin. If we
5075     // are in an inline builtin implementation, avoid trivial infinite
5076     // recursion. Honor __attribute__((no_builtin("foo"))) or
5077     // __attribute__((no_builtin)) on the current function unless foo is
5078     // not a predefined library function which means we must generate the
5079     // builtin no matter what.
5080     else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
5081       return CGCallee::forBuiltin(builtinID, FD);
5082   }
5083 
5084   llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
5085   if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
5086       FD->hasAttr<CUDAGlobalAttr>())
5087     CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
5088         cast<llvm::GlobalValue>(CalleePtr->stripPointerCasts()));
5089 
5090   return CGCallee::forDirect(CalleePtr, GD);
5091 }
5092 
5093 CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
5094   E = E->IgnoreParens();
5095 
5096   // Look through function-to-pointer decay.
5097   if (auto ICE = dyn_cast<ImplicitCastExpr>(E)) {
5098     if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
5099         ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
5100       return EmitCallee(ICE->getSubExpr());
5101     }
5102 
5103   // Resolve direct calls.
5104   } else if (auto DRE = dyn_cast<DeclRefExpr>(E)) {
5105     if (auto FD = dyn_cast<FunctionDecl>(DRE->getDecl())) {
5106       return EmitDirectCallee(*this, FD);
5107     }
5108   } else if (auto ME = dyn_cast<MemberExpr>(E)) {
5109     if (auto FD = dyn_cast<FunctionDecl>(ME->getMemberDecl())) {
5110       EmitIgnoredExpr(ME->getBase());
5111       return EmitDirectCallee(*this, FD);
5112     }
5113 
5114   // Look through template substitutions.
5115   } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {
5116     return EmitCallee(NTTP->getReplacement());
5117 
5118   // Treat pseudo-destructor calls differently.
5119   } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(E)) {
5120     return CGCallee::forPseudoDestructor(PDE);
5121   }
5122 
5123   // Otherwise, we have an indirect reference.
5124   llvm::Value *calleePtr;
5125   QualType functionType;
5126   if (auto ptrType = E->getType()->getAs<PointerType>()) {
5127     calleePtr = EmitScalarExpr(E);
5128     functionType = ptrType->getPointeeType();
5129   } else {
5130     functionType = E->getType();
5131     calleePtr = EmitLValue(E).getPointer(*this);
5132   }
5133   assert(functionType->isFunctionType());
5134 
5135   GlobalDecl GD;
5136   if (const auto *VD =
5137           dyn_cast_or_null<VarDecl>(E->getReferencedDeclOfCallee()))
5138     GD = GlobalDecl(VD);
5139 
5140   CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD);
5141   CGCallee callee(calleeInfo, calleePtr);
5142   return callee;
5143 }
5144 
5145 LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
5146   // Comma expressions just emit their LHS then their RHS as an l-value.
5147   if (E->getOpcode() == BO_Comma) {
5148     EmitIgnoredExpr(E->getLHS());
5149     EnsureInsertPoint();
5150     return EmitLValue(E->getRHS());
5151   }
5152 
5153   if (E->getOpcode() == BO_PtrMemD ||
5154       E->getOpcode() == BO_PtrMemI)
5155     return EmitPointerToDataMemberBinaryExpr(E);
5156 
5157   assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
5158 
5159   // Note that in all of these cases, __block variables need the RHS
5160   // evaluated first just in case the variable gets moved by the RHS.
5161 
5162   switch (getEvaluationKind(E->getType())) {
5163   case TEK_Scalar: {
5164     switch (E->getLHS()->getType().getObjCLifetime()) {
5165     case Qualifiers::OCL_Strong:
5166       return EmitARCStoreStrong(E, /*ignored*/ false).first;
5167 
5168     case Qualifiers::OCL_Autoreleasing:
5169       return EmitARCStoreAutoreleasing(E).first;
5170 
5171     // No reason to do any of these differently.
5172     case Qualifiers::OCL_None:
5173     case Qualifiers::OCL_ExplicitNone:
5174     case Qualifiers::OCL_Weak:
5175       break;
5176     }
5177 
5178     RValue RV = EmitAnyExpr(E->getRHS());
5179     LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);
5180     if (RV.isScalar())
5181       EmitNullabilityCheck(LV, RV.getScalarVal(), E->getExprLoc());
5182     EmitStoreThroughLValue(RV, LV);
5183     if (getLangOpts().OpenMP)
5184       CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
5185                                                                 E->getLHS());
5186     return LV;
5187   }
5188 
5189   case TEK_Complex:
5190     return EmitComplexAssignmentLValue(E);
5191 
5192   case TEK_Aggregate:
5193     return EmitAggExprToLValue(E);
5194   }
5195   llvm_unreachable("bad evaluation kind");
5196 }
5197 
5198 LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
5199   RValue RV = EmitCallExpr(E);
5200 
5201   if (!RV.isScalar())
5202     return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5203                           AlignmentSource::Decl);
5204 
5205   assert(E->getCallReturnType(getContext())->isReferenceType() &&
5206          "Can't have a scalar return unless the return type is a "
5207          "reference type!");
5208 
5209   return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
5210 }
5211 
5212 LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
5213   // FIXME: This shouldn't require another copy.
5214   return EmitAggExprToLValue(E);
5215 }
5216 
5217 LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
5218   assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
5219          && "binding l-value to type which needs a temporary");
5220   AggValueSlot Slot = CreateAggTemp(E->getType());
5221   EmitCXXConstructExpr(E, Slot);
5222   return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5223 }
5224 
5225 LValue
5226 CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
5227   return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType());
5228 }
5229 
5230 Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
5231   return Builder.CreateElementBitCast(CGM.GetAddrOfMSGuidDecl(E->getGuidDecl()),
5232                                       ConvertType(E->getType()));
5233 }
5234 
5235 LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
5236   return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(),
5237                         AlignmentSource::Decl);
5238 }
5239 
5240 LValue
5241 CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
5242   AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
5243   Slot.setExternallyDestructed();
5244   EmitAggExpr(E->getSubExpr(), Slot);
5245   EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddress());
5246   return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5247 }
5248 
5249 LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
5250   RValue RV = EmitObjCMessageExpr(E);
5251 
5252   if (!RV.isScalar())
5253     return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5254                           AlignmentSource::Decl);
5255 
5256   assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
5257          "Can't have a scalar return unless the return type is a "
5258          "reference type!");
5259 
5260   return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
5261 }
5262 
5263 LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
5264   Address V =
5265     CGM.getObjCRuntime().GetAddrOfSelector(*this, E->getSelector());
5266   return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl);
5267 }
5268 
5269 llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
5270                                              const ObjCIvarDecl *Ivar) {
5271   return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);
5272 }
5273 
5274 LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
5275                                           llvm::Value *BaseValue,
5276                                           const ObjCIvarDecl *Ivar,
5277                                           unsigned CVRQualifiers) {
5278   return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,
5279                                                    Ivar, CVRQualifiers);
5280 }
5281 
5282 LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
5283   // FIXME: A lot of the code below could be shared with EmitMemberExpr.
5284   llvm::Value *BaseValue = nullptr;
5285   const Expr *BaseExpr = E->getBase();
5286   Qualifiers BaseQuals;
5287   QualType ObjectTy;
5288   if (E->isArrow()) {
5289     BaseValue = EmitScalarExpr(BaseExpr);
5290     ObjectTy = BaseExpr->getType()->getPointeeType();
5291     BaseQuals = ObjectTy.getQualifiers();
5292   } else {
5293     LValue BaseLV = EmitLValue(BaseExpr);
5294     BaseValue = BaseLV.getPointer(*this);
5295     ObjectTy = BaseExpr->getType();
5296     BaseQuals = ObjectTy.getQualifiers();
5297   }
5298 
5299   LValue LV =
5300     EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),
5301                       BaseQuals.getCVRQualifiers());
5302   setObjCGCLValueClass(getContext(), E, LV);
5303   return LV;
5304 }
5305 
5306 LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
5307   // Can only get l-value for message expression returning aggregate type
5308   RValue RV = EmitAnyExprToTemp(E);
5309   return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5310                         AlignmentSource::Decl);
5311 }
5312 
5313 RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee,
5314                                  const CallExpr *E, ReturnValueSlot ReturnValue,
5315                                  llvm::Value *Chain) {
5316   // Get the actual function type. The callee type will always be a pointer to
5317   // function type or a block pointer type.
5318   assert(CalleeType->isFunctionPointerType() &&
5319          "Call must have function pointer type!");
5320 
5321   const Decl *TargetDecl =
5322       OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
5323 
5324   CalleeType = getContext().getCanonicalType(CalleeType);
5325 
5326   auto PointeeType = cast<PointerType>(CalleeType)->getPointeeType();
5327 
5328   CGCallee Callee = OrigCallee;
5329 
5330   if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function) &&
5331       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5332     if (llvm::Constant *PrefixSig =
5333             CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
5334       SanitizerScope SanScope(this);
5335       // Remove any (C++17) exception specifications, to allow calling e.g. a
5336       // noexcept function through a non-noexcept pointer.
5337       auto ProtoTy =
5338         getContext().getFunctionTypeWithExceptionSpec(PointeeType, EST_None);
5339       llvm::Constant *FTRTTIConst =
5340           CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true);
5341       llvm::Type *PrefixSigType = PrefixSig->getType();
5342       llvm::StructType *PrefixStructTy = llvm::StructType::get(
5343           CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true);
5344 
5345       llvm::Value *CalleePtr = Callee.getFunctionPointer();
5346 
5347       llvm::Value *CalleePrefixStruct = Builder.CreateBitCast(
5348           CalleePtr, llvm::PointerType::getUnqual(PrefixStructTy));
5349       llvm::Value *CalleeSigPtr =
5350           Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0);
5351       llvm::Value *CalleeSig =
5352           Builder.CreateAlignedLoad(PrefixSigType, CalleeSigPtr, getIntAlign());
5353       llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig);
5354 
5355       llvm::BasicBlock *Cont = createBasicBlock("cont");
5356       llvm::BasicBlock *TypeCheck = createBasicBlock("typecheck");
5357       Builder.CreateCondBr(CalleeSigMatch, TypeCheck, Cont);
5358 
5359       EmitBlock(TypeCheck);
5360       llvm::Value *CalleeRTTIPtr =
5361           Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1);
5362       llvm::Value *CalleeRTTIEncoded =
5363           Builder.CreateAlignedLoad(Int32Ty, CalleeRTTIPtr, getPointerAlign());
5364       llvm::Value *CalleeRTTI =
5365           DecodeAddrUsedInPrologue(CalleePtr, CalleeRTTIEncoded);
5366       llvm::Value *CalleeRTTIMatch =
5367           Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst);
5368       llvm::Constant *StaticData[] = {EmitCheckSourceLocation(E->getBeginLoc()),
5369                                       EmitCheckTypeDescriptor(CalleeType)};
5370       EmitCheck(std::make_pair(CalleeRTTIMatch, SanitizerKind::Function),
5371                 SanitizerHandler::FunctionTypeMismatch, StaticData,
5372                 {CalleePtr, CalleeRTTI, FTRTTIConst});
5373 
5374       Builder.CreateBr(Cont);
5375       EmitBlock(Cont);
5376     }
5377   }
5378 
5379   const auto *FnType = cast<FunctionType>(PointeeType);
5380 
5381   // If we are checking indirect calls and this call is indirect, check that the
5382   // function pointer is a member of the bit set for the function type.
5383   if (SanOpts.has(SanitizerKind::CFIICall) &&
5384       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5385     SanitizerScope SanScope(this);
5386     EmitSanitizerStatReport(llvm::SanStat_CFI_ICall);
5387 
5388     llvm::Metadata *MD;
5389     if (CGM.getCodeGenOpts().SanitizeCfiICallGeneralizePointers)
5390       MD = CGM.CreateMetadataIdentifierGeneralized(QualType(FnType, 0));
5391     else
5392       MD = CGM.CreateMetadataIdentifierForType(QualType(FnType, 0));
5393 
5394     llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
5395 
5396     llvm::Value *CalleePtr = Callee.getFunctionPointer();
5397     llvm::Value *CastedCallee = Builder.CreateBitCast(CalleePtr, Int8PtrTy);
5398     llvm::Value *TypeTest = Builder.CreateCall(
5399         CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedCallee, TypeId});
5400 
5401     auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
5402     llvm::Constant *StaticData[] = {
5403         llvm::ConstantInt::get(Int8Ty, CFITCK_ICall),
5404         EmitCheckSourceLocation(E->getBeginLoc()),
5405         EmitCheckTypeDescriptor(QualType(FnType, 0)),
5406     };
5407     if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
5408       EmitCfiSlowPathCheck(SanitizerKind::CFIICall, TypeTest, CrossDsoTypeId,
5409                            CastedCallee, StaticData);
5410     } else {
5411       EmitCheck(std::make_pair(TypeTest, SanitizerKind::CFIICall),
5412                 SanitizerHandler::CFICheckFail, StaticData,
5413                 {CastedCallee, llvm::UndefValue::get(IntPtrTy)});
5414     }
5415   }
5416 
5417   CallArgList Args;
5418   if (Chain)
5419     Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)),
5420              CGM.getContext().VoidPtrTy);
5421 
5422   // C++17 requires that we evaluate arguments to a call using assignment syntax
5423   // right-to-left, and that we evaluate arguments to certain other operators
5424   // left-to-right. Note that we allow this to override the order dictated by
5425   // the calling convention on the MS ABI, which means that parameter
5426   // destruction order is not necessarily reverse construction order.
5427   // FIXME: Revisit this based on C++ committee response to unimplementability.
5428   EvaluationOrder Order = EvaluationOrder::Default;
5429   if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) {
5430     if (OCE->isAssignmentOp())
5431       Order = EvaluationOrder::ForceRightToLeft;
5432     else {
5433       switch (OCE->getOperator()) {
5434       case OO_LessLess:
5435       case OO_GreaterGreater:
5436       case OO_AmpAmp:
5437       case OO_PipePipe:
5438       case OO_Comma:
5439       case OO_ArrowStar:
5440         Order = EvaluationOrder::ForceLeftToRight;
5441         break;
5442       default:
5443         break;
5444       }
5445     }
5446   }
5447 
5448   EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(),
5449                E->getDirectCallee(), /*ParamsToSkip*/ 0, Order);
5450 
5451   const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
5452       Args, FnType, /*ChainCall=*/Chain);
5453 
5454   // C99 6.5.2.2p6:
5455   //   If the expression that denotes the called function has a type
5456   //   that does not include a prototype, [the default argument
5457   //   promotions are performed]. If the number of arguments does not
5458   //   equal the number of parameters, the behavior is undefined. If
5459   //   the function is defined with a type that includes a prototype,
5460   //   and either the prototype ends with an ellipsis (, ...) or the
5461   //   types of the arguments after promotion are not compatible with
5462   //   the types of the parameters, the behavior is undefined. If the
5463   //   function is defined with a type that does not include a
5464   //   prototype, and the types of the arguments after promotion are
5465   //   not compatible with those of the parameters after promotion,
5466   //   the behavior is undefined [except in some trivial cases].
5467   // That is, in the general case, we should assume that a call
5468   // through an unprototyped function type works like a *non-variadic*
5469   // call.  The way we make this work is to cast to the exact type
5470   // of the promoted arguments.
5471   //
5472   // Chain calls use this same code path to add the invisible chain parameter
5473   // to the function type.
5474   if (isa<FunctionNoProtoType>(FnType) || Chain) {
5475     llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
5476     int AS = Callee.getFunctionPointer()->getType()->getPointerAddressSpace();
5477     CalleeTy = CalleeTy->getPointerTo(AS);
5478 
5479     llvm::Value *CalleePtr = Callee.getFunctionPointer();
5480     CalleePtr = Builder.CreateBitCast(CalleePtr, CalleeTy, "callee.knr.cast");
5481     Callee.setFunctionPointer(CalleePtr);
5482   }
5483 
5484   // HIP function pointer contains kernel handle when it is used in triple
5485   // chevron. The kernel stub needs to be loaded from kernel handle and used
5486   // as callee.
5487   if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
5488       isa<CUDAKernelCallExpr>(E) &&
5489       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5490     llvm::Value *Handle = Callee.getFunctionPointer();
5491     auto *Cast =
5492         Builder.CreateBitCast(Handle, Handle->getType()->getPointerTo());
5493     auto *Stub = Builder.CreateLoad(
5494         Address(Cast, Handle->getType(), CGM.getPointerAlign()));
5495     Callee.setFunctionPointer(Stub);
5496   }
5497   llvm::CallBase *CallOrInvoke = nullptr;
5498   RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &CallOrInvoke,
5499                          E == MustTailCall, E->getExprLoc());
5500 
5501   // Generate function declaration DISuprogram in order to be used
5502   // in debug info about call sites.
5503   if (CGDebugInfo *DI = getDebugInfo()) {
5504     if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
5505       FunctionArgList Args;
5506       QualType ResTy = BuildFunctionArgList(CalleeDecl, Args);
5507       DI->EmitFuncDeclForCallSite(CallOrInvoke,
5508                                   DI->getFunctionType(CalleeDecl, ResTy, Args),
5509                                   CalleeDecl);
5510     }
5511   }
5512 
5513   return Call;
5514 }
5515 
5516 LValue CodeGenFunction::
5517 EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
5518   Address BaseAddr = Address::invalid();
5519   if (E->getOpcode() == BO_PtrMemI) {
5520     BaseAddr = EmitPointerWithAlignment(E->getLHS());
5521   } else {
5522     BaseAddr = EmitLValue(E->getLHS()).getAddress(*this);
5523   }
5524 
5525   llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
5526   const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
5527 
5528   LValueBaseInfo BaseInfo;
5529   TBAAAccessInfo TBAAInfo;
5530   Address MemberAddr =
5531     EmitCXXMemberDataPointerAddress(E, BaseAddr, OffsetV, MPT, &BaseInfo,
5532                                     &TBAAInfo);
5533 
5534   return MakeAddrLValue(MemberAddr, MPT->getPointeeType(), BaseInfo, TBAAInfo);
5535 }
5536 
5537 /// Given the address of a temporary variable, produce an r-value of
5538 /// its type.
5539 RValue CodeGenFunction::convertTempToRValue(Address addr,
5540                                             QualType type,
5541                                             SourceLocation loc) {
5542   LValue lvalue = MakeAddrLValue(addr, type, AlignmentSource::Decl);
5543   switch (getEvaluationKind(type)) {
5544   case TEK_Complex:
5545     return RValue::getComplex(EmitLoadOfComplex(lvalue, loc));
5546   case TEK_Aggregate:
5547     return lvalue.asAggregateRValue(*this);
5548   case TEK_Scalar:
5549     return RValue::get(EmitLoadOfScalar(lvalue, loc));
5550   }
5551   llvm_unreachable("bad evaluation kind");
5552 }
5553 
5554 void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
5555   assert(Val->getType()->isFPOrFPVectorTy());
5556   if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
5557     return;
5558 
5559   llvm::MDBuilder MDHelper(getLLVMContext());
5560   llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
5561 
5562   cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node);
5563 }
5564 
5565 namespace {
5566   struct LValueOrRValue {
5567     LValue LV;
5568     RValue RV;
5569   };
5570 }
5571 
5572 static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
5573                                            const PseudoObjectExpr *E,
5574                                            bool forLValue,
5575                                            AggValueSlot slot) {
5576   SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
5577 
5578   // Find the result expression, if any.
5579   const Expr *resultExpr = E->getResultExpr();
5580   LValueOrRValue result;
5581 
5582   for (PseudoObjectExpr::const_semantics_iterator
5583          i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
5584     const Expr *semantic = *i;
5585 
5586     // If this semantic expression is an opaque value, bind it
5587     // to the result of its source expression.
5588     if (const auto *ov = dyn_cast<OpaqueValueExpr>(semantic)) {
5589       // Skip unique OVEs.
5590       if (ov->isUnique()) {
5591         assert(ov != resultExpr &&
5592                "A unique OVE cannot be used as the result expression");
5593         continue;
5594       }
5595 
5596       // If this is the result expression, we may need to evaluate
5597       // directly into the slot.
5598       typedef CodeGenFunction::OpaqueValueMappingData OVMA;
5599       OVMA opaqueData;
5600       if (ov == resultExpr && ov->isPRValue() && !forLValue &&
5601           CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) {
5602         CGF.EmitAggExpr(ov->getSourceExpr(), slot);
5603         LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(),
5604                                        AlignmentSource::Decl);
5605         opaqueData = OVMA::bind(CGF, ov, LV);
5606         result.RV = slot.asRValue();
5607 
5608       // Otherwise, emit as normal.
5609       } else {
5610         opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr());
5611 
5612         // If this is the result, also evaluate the result now.
5613         if (ov == resultExpr) {
5614           if (forLValue)
5615             result.LV = CGF.EmitLValue(ov);
5616           else
5617             result.RV = CGF.EmitAnyExpr(ov, slot);
5618         }
5619       }
5620 
5621       opaques.push_back(opaqueData);
5622 
5623     // Otherwise, if the expression is the result, evaluate it
5624     // and remember the result.
5625     } else if (semantic == resultExpr) {
5626       if (forLValue)
5627         result.LV = CGF.EmitLValue(semantic);
5628       else
5629         result.RV = CGF.EmitAnyExpr(semantic, slot);
5630 
5631     // Otherwise, evaluate the expression in an ignored context.
5632     } else {
5633       CGF.EmitIgnoredExpr(semantic);
5634     }
5635   }
5636 
5637   // Unbind all the opaques now.
5638   for (unsigned i = 0, e = opaques.size(); i != e; ++i)
5639     opaques[i].unbind(CGF);
5640 
5641   return result;
5642 }
5643 
5644 RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
5645                                                AggValueSlot slot) {
5646   return emitPseudoObjectExpr(*this, E, false, slot).RV;
5647 }
5648 
5649 LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
5650   return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;
5651 }
5652