xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGClass.cpp (revision ec0ea6efa1ad229d75c394c1a9b9cac33af2b1d3)
1 //===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code dealing with C++ code generation of classes
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGBlocks.h"
14 #include "CGCXXABI.h"
15 #include "CGDebugInfo.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenFunction.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Attr.h"
20 #include "clang/AST/CXXInheritance.h"
21 #include "clang/AST/CharUnits.h"
22 #include "clang/AST/DeclTemplate.h"
23 #include "clang/AST/EvaluatedExprVisitor.h"
24 #include "clang/AST/RecordLayout.h"
25 #include "clang/AST/StmtCXX.h"
26 #include "clang/Basic/CodeGenOptions.h"
27 #include "clang/Basic/TargetBuiltins.h"
28 #include "clang/CodeGen/CGFunctionInfo.h"
29 #include "llvm/IR/Intrinsics.h"
30 #include "llvm/IR/Metadata.h"
31 #include "llvm/Transforms/Utils/SanitizerStats.h"
32 
33 using namespace clang;
34 using namespace CodeGen;
35 
36 /// Return the best known alignment for an unknown pointer to a
37 /// particular class.
38 CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
39   if (!RD->hasDefinition())
40     return CharUnits::One(); // Hopefully won't be used anywhere.
41 
42   auto &layout = getContext().getASTRecordLayout(RD);
43 
44   // If the class is final, then we know that the pointer points to an
45   // object of that type and can use the full alignment.
46   if (RD->isEffectivelyFinal())
47     return layout.getAlignment();
48 
49   // Otherwise, we have to assume it could be a subclass.
50   return layout.getNonVirtualAlignment();
51 }
52 
53 /// Return the smallest possible amount of storage that might be allocated
54 /// starting from the beginning of an object of a particular class.
55 ///
56 /// This may be smaller than sizeof(RD) if RD has virtual base classes.
57 CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
58   if (!RD->hasDefinition())
59     return CharUnits::One();
60 
61   auto &layout = getContext().getASTRecordLayout(RD);
62 
63   // If the class is final, then we know that the pointer points to an
64   // object of that type and can use the full alignment.
65   if (RD->isEffectivelyFinal())
66     return layout.getSize();
67 
68   // Otherwise, we have to assume it could be a subclass.
69   return std::max(layout.getNonVirtualSize(), CharUnits::One());
70 }
71 
72 /// Return the best known alignment for a pointer to a virtual base,
73 /// given the alignment of a pointer to the derived class.
74 CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
75                                            const CXXRecordDecl *derivedClass,
76                                            const CXXRecordDecl *vbaseClass) {
77   // The basic idea here is that an underaligned derived pointer might
78   // indicate an underaligned base pointer.
79 
80   assert(vbaseClass->isCompleteDefinition());
81   auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
82   CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
83 
84   return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
85                                    expectedVBaseAlign);
86 }
87 
88 CharUnits
89 CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
90                                          const CXXRecordDecl *baseDecl,
91                                          CharUnits expectedTargetAlign) {
92   // If the base is an incomplete type (which is, alas, possible with
93   // member pointers), be pessimistic.
94   if (!baseDecl->isCompleteDefinition())
95     return std::min(actualBaseAlign, expectedTargetAlign);
96 
97   auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
98   CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
99 
100   // If the class is properly aligned, assume the target offset is, too.
101   //
102   // This actually isn't necessarily the right thing to do --- if the
103   // class is a complete object, but it's only properly aligned for a
104   // base subobject, then the alignments of things relative to it are
105   // probably off as well.  (Note that this requires the alignment of
106   // the target to be greater than the NV alignment of the derived
107   // class.)
108   //
109   // However, our approach to this kind of under-alignment can only
110   // ever be best effort; after all, we're never going to propagate
111   // alignments through variables or parameters.  Note, in particular,
112   // that constructing a polymorphic type in an address that's less
113   // than pointer-aligned will generally trap in the constructor,
114   // unless we someday add some sort of attribute to change the
115   // assumed alignment of 'this'.  So our goal here is pretty much
116   // just to allow the user to explicitly say that a pointer is
117   // under-aligned and then safely access its fields and vtables.
118   if (actualBaseAlign >= expectedBaseAlign) {
119     return expectedTargetAlign;
120   }
121 
122   // Otherwise, we might be offset by an arbitrary multiple of the
123   // actual alignment.  The correct adjustment is to take the min of
124   // the two alignments.
125   return std::min(actualBaseAlign, expectedTargetAlign);
126 }
127 
128 Address CodeGenFunction::LoadCXXThisAddress() {
129   assert(CurFuncDecl && "loading 'this' without a func declaration?");
130   assert(isa<CXXMethodDecl>(CurFuncDecl));
131 
132   // Lazily compute CXXThisAlignment.
133   if (CXXThisAlignment.isZero()) {
134     // Just use the best known alignment for the parent.
135     // TODO: if we're currently emitting a complete-object ctor/dtor,
136     // we can always use the complete-object alignment.
137     auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent();
138     CXXThisAlignment = CGM.getClassPointerAlignment(RD);
139   }
140 
141   return Address(LoadCXXThis(), CXXThisAlignment);
142 }
143 
144 /// Emit the address of a field using a member data pointer.
145 ///
146 /// \param E Only used for emergency diagnostics
147 Address
148 CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
149                                                  llvm::Value *memberPtr,
150                                       const MemberPointerType *memberPtrType,
151                                                  LValueBaseInfo *BaseInfo,
152                                                  TBAAAccessInfo *TBAAInfo) {
153   // Ask the ABI to compute the actual address.
154   llvm::Value *ptr =
155     CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
156                                                  memberPtr, memberPtrType);
157 
158   QualType memberType = memberPtrType->getPointeeType();
159   CharUnits memberAlign =
160       CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo);
161   memberAlign =
162     CGM.getDynamicOffsetAlignment(base.getAlignment(),
163                             memberPtrType->getClass()->getAsCXXRecordDecl(),
164                                   memberAlign);
165   return Address(ptr, memberAlign);
166 }
167 
168 CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
169     const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
170     CastExpr::path_const_iterator End) {
171   CharUnits Offset = CharUnits::Zero();
172 
173   const ASTContext &Context = getContext();
174   const CXXRecordDecl *RD = DerivedClass;
175 
176   for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
177     const CXXBaseSpecifier *Base = *I;
178     assert(!Base->isVirtual() && "Should not see virtual bases here!");
179 
180     // Get the layout.
181     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
182 
183     const auto *BaseDecl =
184         cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
185 
186     // Add the offset.
187     Offset += Layout.getBaseClassOffset(BaseDecl);
188 
189     RD = BaseDecl;
190   }
191 
192   return Offset;
193 }
194 
195 llvm::Constant *
196 CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
197                                    CastExpr::path_const_iterator PathBegin,
198                                    CastExpr::path_const_iterator PathEnd) {
199   assert(PathBegin != PathEnd && "Base path should not be empty!");
200 
201   CharUnits Offset =
202       computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
203   if (Offset.isZero())
204     return nullptr;
205 
206   llvm::Type *PtrDiffTy =
207   Types.ConvertType(getContext().getPointerDiffType());
208 
209   return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
210 }
211 
212 /// Gets the address of a direct base class within a complete object.
213 /// This should only be used for (1) non-virtual bases or (2) virtual bases
214 /// when the type is known to be complete (e.g. in complete destructors).
215 ///
216 /// The object pointed to by 'This' is assumed to be non-null.
217 Address
218 CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
219                                                    const CXXRecordDecl *Derived,
220                                                    const CXXRecordDecl *Base,
221                                                    bool BaseIsVirtual) {
222   // 'this' must be a pointer (in some address space) to Derived.
223   assert(This.getElementType() == ConvertType(Derived));
224 
225   // Compute the offset of the virtual base.
226   CharUnits Offset;
227   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
228   if (BaseIsVirtual)
229     Offset = Layout.getVBaseClassOffset(Base);
230   else
231     Offset = Layout.getBaseClassOffset(Base);
232 
233   // Shift and cast down to the base type.
234   // TODO: for complete types, this should be possible with a GEP.
235   Address V = This;
236   if (!Offset.isZero()) {
237     V = Builder.CreateElementBitCast(V, Int8Ty);
238     V = Builder.CreateConstInBoundsByteGEP(V, Offset);
239   }
240   V = Builder.CreateElementBitCast(V, ConvertType(Base));
241 
242   return V;
243 }
244 
245 static Address
246 ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
247                                 CharUnits nonVirtualOffset,
248                                 llvm::Value *virtualOffset,
249                                 const CXXRecordDecl *derivedClass,
250                                 const CXXRecordDecl *nearestVBase) {
251   // Assert that we have something to do.
252   assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
253 
254   // Compute the offset from the static and dynamic components.
255   llvm::Value *baseOffset;
256   if (!nonVirtualOffset.isZero()) {
257     llvm::Type *OffsetType =
258         (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
259          CGF.CGM.getItaniumVTableContext().isRelativeLayout())
260             ? CGF.Int32Ty
261             : CGF.PtrDiffTy;
262     baseOffset =
263         llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity());
264     if (virtualOffset) {
265       baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);
266     }
267   } else {
268     baseOffset = virtualOffset;
269   }
270 
271   // Apply the base offset.
272   llvm::Value *ptr = addr.getPointer();
273   unsigned AddrSpace = ptr->getType()->getPointerAddressSpace();
274   ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8Ty->getPointerTo(AddrSpace));
275   ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr");
276 
277   // If we have a virtual component, the alignment of the result will
278   // be relative only to the known alignment of that vbase.
279   CharUnits alignment;
280   if (virtualOffset) {
281     assert(nearestVBase && "virtual offset without vbase?");
282     alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
283                                           derivedClass, nearestVBase);
284   } else {
285     alignment = addr.getAlignment();
286   }
287   alignment = alignment.alignmentAtOffset(nonVirtualOffset);
288 
289   return Address(ptr, alignment);
290 }
291 
292 Address CodeGenFunction::GetAddressOfBaseClass(
293     Address Value, const CXXRecordDecl *Derived,
294     CastExpr::path_const_iterator PathBegin,
295     CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
296     SourceLocation Loc) {
297   assert(PathBegin != PathEnd && "Base path should not be empty!");
298 
299   CastExpr::path_const_iterator Start = PathBegin;
300   const CXXRecordDecl *VBase = nullptr;
301 
302   // Sema has done some convenient canonicalization here: if the
303   // access path involved any virtual steps, the conversion path will
304   // *start* with a step down to the correct virtual base subobject,
305   // and hence will not require any further steps.
306   if ((*Start)->isVirtual()) {
307     VBase = cast<CXXRecordDecl>(
308         (*Start)->getType()->castAs<RecordType>()->getDecl());
309     ++Start;
310   }
311 
312   // Compute the static offset of the ultimate destination within its
313   // allocating subobject (the virtual base, if there is one, or else
314   // the "complete" object that we see).
315   CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
316       VBase ? VBase : Derived, Start, PathEnd);
317 
318   // If there's a virtual step, we can sometimes "devirtualize" it.
319   // For now, that's limited to when the derived type is final.
320   // TODO: "devirtualize" this for accesses to known-complete objects.
321   if (VBase && Derived->hasAttr<FinalAttr>()) {
322     const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
323     CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
324     NonVirtualOffset += vBaseOffset;
325     VBase = nullptr; // we no longer have a virtual step
326   }
327 
328   // Get the base pointer type.
329   llvm::Type *BasePtrTy =
330       ConvertType((PathEnd[-1])->getType())
331           ->getPointerTo(Value.getType()->getPointerAddressSpace());
332 
333   QualType DerivedTy = getContext().getRecordType(Derived);
334   CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
335 
336   // If the static offset is zero and we don't have a virtual step,
337   // just do a bitcast; null checks are unnecessary.
338   if (NonVirtualOffset.isZero() && !VBase) {
339     if (sanitizePerformTypeCheck()) {
340       SanitizerSet SkippedChecks;
341       SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
342       EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
343                     DerivedTy, DerivedAlign, SkippedChecks);
344     }
345     return Builder.CreateBitCast(Value, BasePtrTy);
346   }
347 
348   llvm::BasicBlock *origBB = nullptr;
349   llvm::BasicBlock *endBB = nullptr;
350 
351   // Skip over the offset (and the vtable load) if we're supposed to
352   // null-check the pointer.
353   if (NullCheckValue) {
354     origBB = Builder.GetInsertBlock();
355     llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
356     endBB = createBasicBlock("cast.end");
357 
358     llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
359     Builder.CreateCondBr(isNull, endBB, notNullBB);
360     EmitBlock(notNullBB);
361   }
362 
363   if (sanitizePerformTypeCheck()) {
364     SanitizerSet SkippedChecks;
365     SkippedChecks.set(SanitizerKind::Null, true);
366     EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
367                   Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
368   }
369 
370   // Compute the virtual offset.
371   llvm::Value *VirtualOffset = nullptr;
372   if (VBase) {
373     VirtualOffset =
374       CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
375   }
376 
377   // Apply both offsets.
378   Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
379                                           VirtualOffset, Derived, VBase);
380 
381   // Cast to the destination type.
382   Value = Builder.CreateBitCast(Value, BasePtrTy);
383 
384   // Build a phi if we needed a null check.
385   if (NullCheckValue) {
386     llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
387     Builder.CreateBr(endBB);
388     EmitBlock(endBB);
389 
390     llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
391     PHI->addIncoming(Value.getPointer(), notNullBB);
392     PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
393     Value = Address(PHI, Value.getAlignment());
394   }
395 
396   return Value;
397 }
398 
399 Address
400 CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
401                                           const CXXRecordDecl *Derived,
402                                         CastExpr::path_const_iterator PathBegin,
403                                           CastExpr::path_const_iterator PathEnd,
404                                           bool NullCheckValue) {
405   assert(PathBegin != PathEnd && "Base path should not be empty!");
406 
407   QualType DerivedTy =
408     getContext().getCanonicalType(getContext().getTagDeclType(Derived));
409   unsigned AddrSpace =
410     BaseAddr.getPointer()->getType()->getPointerAddressSpace();
411   llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo(AddrSpace);
412 
413   llvm::Value *NonVirtualOffset =
414     CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
415 
416   if (!NonVirtualOffset) {
417     // No offset, we can just cast back.
418     return Builder.CreateBitCast(BaseAddr, DerivedPtrTy);
419   }
420 
421   llvm::BasicBlock *CastNull = nullptr;
422   llvm::BasicBlock *CastNotNull = nullptr;
423   llvm::BasicBlock *CastEnd = nullptr;
424 
425   if (NullCheckValue) {
426     CastNull = createBasicBlock("cast.null");
427     CastNotNull = createBasicBlock("cast.notnull");
428     CastEnd = createBasicBlock("cast.end");
429 
430     llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
431     Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
432     EmitBlock(CastNotNull);
433   }
434 
435   // Apply the offset.
436   llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
437   Value = Builder.CreateInBoundsGEP(
438       Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr");
439 
440   // Just cast.
441   Value = Builder.CreateBitCast(Value, DerivedPtrTy);
442 
443   // Produce a PHI if we had a null-check.
444   if (NullCheckValue) {
445     Builder.CreateBr(CastEnd);
446     EmitBlock(CastNull);
447     Builder.CreateBr(CastEnd);
448     EmitBlock(CastEnd);
449 
450     llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
451     PHI->addIncoming(Value, CastNotNull);
452     PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
453     Value = PHI;
454   }
455 
456   return Address(Value, CGM.getClassPointerAlignment(Derived));
457 }
458 
459 llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
460                                               bool ForVirtualBase,
461                                               bool Delegating) {
462   if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
463     // This constructor/destructor does not need a VTT parameter.
464     return nullptr;
465   }
466 
467   const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
468   const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
469 
470   uint64_t SubVTTIndex;
471 
472   if (Delegating) {
473     // If this is a delegating constructor call, just load the VTT.
474     return LoadCXXVTT();
475   } else if (RD == Base) {
476     // If the record matches the base, this is the complete ctor/dtor
477     // variant calling the base variant in a class with virtual bases.
478     assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
479            "doing no-op VTT offset in base dtor/ctor?");
480     assert(!ForVirtualBase && "Can't have same class as virtual base!");
481     SubVTTIndex = 0;
482   } else {
483     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
484     CharUnits BaseOffset = ForVirtualBase ?
485       Layout.getVBaseClassOffset(Base) :
486       Layout.getBaseClassOffset(Base);
487 
488     SubVTTIndex =
489       CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
490     assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
491   }
492 
493   if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
494     // A VTT parameter was passed to the constructor, use it.
495     llvm::Value *VTT = LoadCXXVTT();
496     return Builder.CreateConstInBoundsGEP1_64(VoidPtrTy, VTT, SubVTTIndex);
497   } else {
498     // We're the complete constructor, so get the VTT by name.
499     llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD);
500     return Builder.CreateConstInBoundsGEP2_64(
501         VTT->getValueType(), VTT, 0, SubVTTIndex);
502   }
503 }
504 
505 namespace {
506   /// Call the destructor for a direct base class.
507   struct CallBaseDtor final : EHScopeStack::Cleanup {
508     const CXXRecordDecl *BaseClass;
509     bool BaseIsVirtual;
510     CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
511       : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
512 
513     void Emit(CodeGenFunction &CGF, Flags flags) override {
514       const CXXRecordDecl *DerivedClass =
515         cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
516 
517       const CXXDestructorDecl *D = BaseClass->getDestructor();
518       // We are already inside a destructor, so presumably the object being
519       // destroyed should have the expected type.
520       QualType ThisTy = D->getThisObjectType();
521       Address Addr =
522         CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
523                                                   DerivedClass, BaseClass,
524                                                   BaseIsVirtual);
525       CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
526                                 /*Delegating=*/false, Addr, ThisTy);
527     }
528   };
529 
530   /// A visitor which checks whether an initializer uses 'this' in a
531   /// way which requires the vtable to be properly set.
532   struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
533     typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
534 
535     bool UsesThis;
536 
537     DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
538 
539     // Black-list all explicit and implicit references to 'this'.
540     //
541     // Do we need to worry about external references to 'this' derived
542     // from arbitrary code?  If so, then anything which runs arbitrary
543     // external code might potentially access the vtable.
544     void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
545   };
546 } // end anonymous namespace
547 
548 static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
549   DynamicThisUseChecker Checker(C);
550   Checker.Visit(Init);
551   return Checker.UsesThis;
552 }
553 
554 static void EmitBaseInitializer(CodeGenFunction &CGF,
555                                 const CXXRecordDecl *ClassDecl,
556                                 CXXCtorInitializer *BaseInit) {
557   assert(BaseInit->isBaseInitializer() &&
558          "Must have base initializer!");
559 
560   Address ThisPtr = CGF.LoadCXXThisAddress();
561 
562   const Type *BaseType = BaseInit->getBaseClass();
563   const auto *BaseClassDecl =
564       cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
565 
566   bool isBaseVirtual = BaseInit->isBaseVirtual();
567 
568   // If the initializer for the base (other than the constructor
569   // itself) accesses 'this' in any way, we need to initialize the
570   // vtables.
571   if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
572     CGF.InitializeVTablePointers(ClassDecl);
573 
574   // We can pretend to be a complete class because it only matters for
575   // virtual bases, and we only do virtual bases for complete ctors.
576   Address V =
577     CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
578                                               BaseClassDecl,
579                                               isBaseVirtual);
580   AggValueSlot AggSlot =
581       AggValueSlot::forAddr(
582           V, Qualifiers(),
583           AggValueSlot::IsDestructed,
584           AggValueSlot::DoesNotNeedGCBarriers,
585           AggValueSlot::IsNotAliased,
586           CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));
587 
588   CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
589 
590   if (CGF.CGM.getLangOpts().Exceptions &&
591       !BaseClassDecl->hasTrivialDestructor())
592     CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
593                                           isBaseVirtual);
594 }
595 
596 static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
597   auto *CD = dyn_cast<CXXConstructorDecl>(D);
598   if (!(CD && CD->isCopyOrMoveConstructor()) &&
599       !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator())
600     return false;
601 
602   // We can emit a memcpy for a trivial copy or move constructor/assignment.
603   if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding())
604     return true;
605 
606   // We *must* emit a memcpy for a defaulted union copy or move op.
607   if (D->getParent()->isUnion() && D->isDefaulted())
608     return true;
609 
610   return false;
611 }
612 
613 static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
614                                                 CXXCtorInitializer *MemberInit,
615                                                 LValue &LHS) {
616   FieldDecl *Field = MemberInit->getAnyMember();
617   if (MemberInit->isIndirectMemberInitializer()) {
618     // If we are initializing an anonymous union field, drill down to the field.
619     IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
620     for (const auto *I : IndirectField->chain())
621       LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
622   } else {
623     LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
624   }
625 }
626 
627 static void EmitMemberInitializer(CodeGenFunction &CGF,
628                                   const CXXRecordDecl *ClassDecl,
629                                   CXXCtorInitializer *MemberInit,
630                                   const CXXConstructorDecl *Constructor,
631                                   FunctionArgList &Args) {
632   ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
633   assert(MemberInit->isAnyMemberInitializer() &&
634          "Must have member initializer!");
635   assert(MemberInit->getInit() && "Must have initializer!");
636 
637   // non-static data member initializers.
638   FieldDecl *Field = MemberInit->getAnyMember();
639   QualType FieldType = Field->getType();
640 
641   llvm::Value *ThisPtr = CGF.LoadCXXThis();
642   QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
643   LValue LHS;
644 
645   // If a base constructor is being emitted, create an LValue that has the
646   // non-virtual alignment.
647   if (CGF.CurGD.getCtorType() == Ctor_Base)
648     LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);
649   else
650     LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
651 
652   EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
653 
654   // Special case: if we are in a copy or move constructor, and we are copying
655   // an array of PODs or classes with trivial copy constructors, ignore the
656   // AST and perform the copy we know is equivalent.
657   // FIXME: This is hacky at best... if we had a bit more explicit information
658   // in the AST, we could generalize it more easily.
659   const ConstantArrayType *Array
660     = CGF.getContext().getAsConstantArrayType(FieldType);
661   if (Array && Constructor->isDefaulted() &&
662       Constructor->isCopyOrMoveConstructor()) {
663     QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
664     CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
665     if (BaseElementTy.isPODType(CGF.getContext()) ||
666         (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) {
667       unsigned SrcArgIndex =
668           CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
669       llvm::Value *SrcPtr
670         = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
671       LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
672       LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
673 
674       // Copy the aggregate.
675       CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),
676                             LHS.isVolatileQualified());
677       // Ensure that we destroy the objects if an exception is thrown later in
678       // the constructor.
679       QualType::DestructionKind dtorKind = FieldType.isDestructedType();
680       if (CGF.needsEHCleanup(dtorKind))
681         CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType);
682       return;
683     }
684   }
685 
686   CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());
687 }
688 
689 void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
690                                               Expr *Init) {
691   QualType FieldType = Field->getType();
692   switch (getEvaluationKind(FieldType)) {
693   case TEK_Scalar:
694     if (LHS.isSimple()) {
695       EmitExprAsInit(Init, Field, LHS, false);
696     } else {
697       RValue RHS = RValue::get(EmitScalarExpr(Init));
698       EmitStoreThroughLValue(RHS, LHS);
699     }
700     break;
701   case TEK_Complex:
702     EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
703     break;
704   case TEK_Aggregate: {
705     AggValueSlot Slot = AggValueSlot::forLValue(
706         LHS, *this, AggValueSlot::IsDestructed,
707         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
708         getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed,
709         // Checks are made by the code that calls constructor.
710         AggValueSlot::IsSanitizerChecked);
711     EmitAggExpr(Init, Slot);
712     break;
713   }
714   }
715 
716   // Ensure that we destroy this object if an exception is thrown
717   // later in the constructor.
718   QualType::DestructionKind dtorKind = FieldType.isDestructedType();
719   if (needsEHCleanup(dtorKind))
720     pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType);
721 }
722 
723 /// Checks whether the given constructor is a valid subject for the
724 /// complete-to-base constructor delegation optimization, i.e.
725 /// emitting the complete constructor as a simple call to the base
726 /// constructor.
727 bool CodeGenFunction::IsConstructorDelegationValid(
728     const CXXConstructorDecl *Ctor) {
729 
730   // Currently we disable the optimization for classes with virtual
731   // bases because (1) the addresses of parameter variables need to be
732   // consistent across all initializers but (2) the delegate function
733   // call necessarily creates a second copy of the parameter variable.
734   //
735   // The limiting example (purely theoretical AFAIK):
736   //   struct A { A(int &c) { c++; } };
737   //   struct B : virtual A {
738   //     B(int count) : A(count) { printf("%d\n", count); }
739   //   };
740   // ...although even this example could in principle be emitted as a
741   // delegation since the address of the parameter doesn't escape.
742   if (Ctor->getParent()->getNumVBases()) {
743     // TODO: white-list trivial vbase initializers.  This case wouldn't
744     // be subject to the restrictions below.
745 
746     // TODO: white-list cases where:
747     //  - there are no non-reference parameters to the constructor
748     //  - the initializers don't access any non-reference parameters
749     //  - the initializers don't take the address of non-reference
750     //    parameters
751     //  - etc.
752     // If we ever add any of the above cases, remember that:
753     //  - function-try-blocks will always exclude this optimization
754     //  - we need to perform the constructor prologue and cleanup in
755     //    EmitConstructorBody.
756 
757     return false;
758   }
759 
760   // We also disable the optimization for variadic functions because
761   // it's impossible to "re-pass" varargs.
762   if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
763     return false;
764 
765   // FIXME: Decide if we can do a delegation of a delegating constructor.
766   if (Ctor->isDelegatingConstructor())
767     return false;
768 
769   return true;
770 }
771 
772 // Emit code in ctor (Prologue==true) or dtor (Prologue==false)
773 // to poison the extra field paddings inserted under
774 // -fsanitize-address-field-padding=1|2.
775 void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
776   ASTContext &Context = getContext();
777   const CXXRecordDecl *ClassDecl =
778       Prologue ? cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()
779                : cast<CXXDestructorDecl>(CurGD.getDecl())->getParent();
780   if (!ClassDecl->mayInsertExtraPadding()) return;
781 
782   struct SizeAndOffset {
783     uint64_t Size;
784     uint64_t Offset;
785   };
786 
787   unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
788   const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
789 
790   // Populate sizes and offsets of fields.
791   SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
792   for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)
793     SSV[i].Offset =
794         Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();
795 
796   size_t NumFields = 0;
797   for (const auto *Field : ClassDecl->fields()) {
798     const FieldDecl *D = Field;
799     auto FieldInfo = Context.getTypeInfoInChars(D->getType());
800     CharUnits FieldSize = FieldInfo.Width;
801     assert(NumFields < SSV.size());
802     SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();
803     NumFields++;
804   }
805   assert(NumFields == SSV.size());
806   if (SSV.size() <= 1) return;
807 
808   // We will insert calls to __asan_* run-time functions.
809   // LLVM AddressSanitizer pass may decide to inline them later.
810   llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
811   llvm::FunctionType *FTy =
812       llvm::FunctionType::get(CGM.VoidTy, Args, false);
813   llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
814       FTy, Prologue ? "__asan_poison_intra_object_redzone"
815                     : "__asan_unpoison_intra_object_redzone");
816 
817   llvm::Value *ThisPtr = LoadCXXThis();
818   ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);
819   uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
820   // For each field check if it has sufficient padding,
821   // if so (un)poison it with a call.
822   for (size_t i = 0; i < SSV.size(); i++) {
823     uint64_t AsanAlignment = 8;
824     uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;
825     uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
826     uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
827     if (PoisonSize < AsanAlignment || !SSV[i].Size ||
828         (NextField % AsanAlignment) != 0)
829       continue;
830     Builder.CreateCall(
831         F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
832             Builder.getIntN(PtrSize, PoisonSize)});
833   }
834 }
835 
836 /// EmitConstructorBody - Emits the body of the current constructor.
837 void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
838   EmitAsanPrologueOrEpilogue(true);
839   const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
840   CXXCtorType CtorType = CurGD.getCtorType();
841 
842   assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
843           CtorType == Ctor_Complete) &&
844          "can only generate complete ctor for this ABI");
845 
846   // Before we go any further, try the complete->base constructor
847   // delegation optimization.
848   if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&
849       CGM.getTarget().getCXXABI().hasConstructorVariants()) {
850     EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());
851     return;
852   }
853 
854   const FunctionDecl *Definition = nullptr;
855   Stmt *Body = Ctor->getBody(Definition);
856   assert(Definition == Ctor && "emitting wrong constructor body");
857 
858   // Enter the function-try-block before the constructor prologue if
859   // applicable.
860   bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
861   if (IsTryBody)
862     EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
863 
864   incrementProfileCounter(Body);
865 
866   RunCleanupsScope RunCleanups(*this);
867 
868   // TODO: in restricted cases, we can emit the vbase initializers of
869   // a complete ctor and then delegate to the base ctor.
870 
871   // Emit the constructor prologue, i.e. the base and member
872   // initializers.
873   EmitCtorPrologue(Ctor, CtorType, Args);
874 
875   // Emit the body of the statement.
876   if (IsTryBody)
877     EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
878   else if (Body)
879     EmitStmt(Body);
880 
881   // Emit any cleanup blocks associated with the member or base
882   // initializers, which includes (along the exceptional path) the
883   // destructors for those members and bases that were fully
884   // constructed.
885   RunCleanups.ForceCleanup();
886 
887   if (IsTryBody)
888     ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
889 }
890 
891 namespace {
892   /// RAII object to indicate that codegen is copying the value representation
893   /// instead of the object representation. Useful when copying a struct or
894   /// class which has uninitialized members and we're only performing
895   /// lvalue-to-rvalue conversion on the object but not its members.
896   class CopyingValueRepresentation {
897   public:
898     explicit CopyingValueRepresentation(CodeGenFunction &CGF)
899         : CGF(CGF), OldSanOpts(CGF.SanOpts) {
900       CGF.SanOpts.set(SanitizerKind::Bool, false);
901       CGF.SanOpts.set(SanitizerKind::Enum, false);
902     }
903     ~CopyingValueRepresentation() {
904       CGF.SanOpts = OldSanOpts;
905     }
906   private:
907     CodeGenFunction &CGF;
908     SanitizerSet OldSanOpts;
909   };
910 } // end anonymous namespace
911 
912 namespace {
913   class FieldMemcpyizer {
914   public:
915     FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
916                     const VarDecl *SrcRec)
917       : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
918         RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
919         FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
920         LastFieldOffset(0), LastAddedFieldIndex(0) {}
921 
922     bool isMemcpyableField(FieldDecl *F) const {
923       // Never memcpy fields when we are adding poisoned paddings.
924       if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
925         return false;
926       Qualifiers Qual = F->getType().getQualifiers();
927       if (Qual.hasVolatile() || Qual.hasObjCLifetime())
928         return false;
929       return true;
930     }
931 
932     void addMemcpyableField(FieldDecl *F) {
933       if (F->isZeroSize(CGF.getContext()))
934         return;
935       if (!FirstField)
936         addInitialField(F);
937       else
938         addNextField(F);
939     }
940 
941     CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
942       ASTContext &Ctx = CGF.getContext();
943       unsigned LastFieldSize =
944           LastField->isBitField()
945               ? LastField->getBitWidthValue(Ctx)
946               : Ctx.toBits(
947                     Ctx.getTypeInfoDataSizeInChars(LastField->getType()).Width);
948       uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
949                                 FirstByteOffset + Ctx.getCharWidth() - 1;
950       CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);
951       return MemcpySize;
952     }
953 
954     void emitMemcpy() {
955       // Give the subclass a chance to bail out if it feels the memcpy isn't
956       // worth it (e.g. Hasn't aggregated enough data).
957       if (!FirstField) {
958         return;
959       }
960 
961       uint64_t FirstByteOffset;
962       if (FirstField->isBitField()) {
963         const CGRecordLayout &RL =
964           CGF.getTypes().getCGRecordLayout(FirstField->getParent());
965         const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
966         // FirstFieldOffset is not appropriate for bitfields,
967         // we need to use the storage offset instead.
968         FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
969       } else {
970         FirstByteOffset = FirstFieldOffset;
971       }
972 
973       CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
974       QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
975       Address ThisPtr = CGF.LoadCXXThisAddress();
976       LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
977       LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
978       llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
979       LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
980       LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
981 
982       emitMemcpyIR(
983           Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(CGF),
984           Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(CGF),
985           MemcpySize);
986       reset();
987     }
988 
989     void reset() {
990       FirstField = nullptr;
991     }
992 
993   protected:
994     CodeGenFunction &CGF;
995     const CXXRecordDecl *ClassDecl;
996 
997   private:
998     void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
999       llvm::PointerType *DPT = DestPtr.getType();
1000       llvm::Type *DBP =
1001         llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace());
1002       DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP);
1003 
1004       llvm::PointerType *SPT = SrcPtr.getType();
1005       llvm::Type *SBP =
1006         llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace());
1007       SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP);
1008 
1009       CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
1010     }
1011 
1012     void addInitialField(FieldDecl *F) {
1013       FirstField = F;
1014       LastField = F;
1015       FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1016       LastFieldOffset = FirstFieldOffset;
1017       LastAddedFieldIndex = F->getFieldIndex();
1018     }
1019 
1020     void addNextField(FieldDecl *F) {
1021       // For the most part, the following invariant will hold:
1022       //   F->getFieldIndex() == LastAddedFieldIndex + 1
1023       // The one exception is that Sema won't add a copy-initializer for an
1024       // unnamed bitfield, which will show up here as a gap in the sequence.
1025       assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1026              "Cannot aggregate fields out of order.");
1027       LastAddedFieldIndex = F->getFieldIndex();
1028 
1029       // The 'first' and 'last' fields are chosen by offset, rather than field
1030       // index. This allows the code to support bitfields, as well as regular
1031       // fields.
1032       uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1033       if (FOffset < FirstFieldOffset) {
1034         FirstField = F;
1035         FirstFieldOffset = FOffset;
1036       } else if (FOffset >= LastFieldOffset) {
1037         LastField = F;
1038         LastFieldOffset = FOffset;
1039       }
1040     }
1041 
1042     const VarDecl *SrcRec;
1043     const ASTRecordLayout &RecLayout;
1044     FieldDecl *FirstField;
1045     FieldDecl *LastField;
1046     uint64_t FirstFieldOffset, LastFieldOffset;
1047     unsigned LastAddedFieldIndex;
1048   };
1049 
1050   class ConstructorMemcpyizer : public FieldMemcpyizer {
1051   private:
1052     /// Get source argument for copy constructor. Returns null if not a copy
1053     /// constructor.
1054     static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1055                                                const CXXConstructorDecl *CD,
1056                                                FunctionArgList &Args) {
1057       if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())
1058         return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1059       return nullptr;
1060     }
1061 
1062     // Returns true if a CXXCtorInitializer represents a member initialization
1063     // that can be rolled into a memcpy.
1064     bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1065       if (!MemcpyableCtor)
1066         return false;
1067       FieldDecl *Field = MemberInit->getMember();
1068       assert(Field && "No field for member init.");
1069       QualType FieldType = Field->getType();
1070       CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
1071 
1072       // Bail out on non-memcpyable, not-trivially-copyable members.
1073       if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) &&
1074           !(FieldType.isTriviallyCopyableType(CGF.getContext()) ||
1075             FieldType->isReferenceType()))
1076         return false;
1077 
1078       // Bail out on volatile fields.
1079       if (!isMemcpyableField(Field))
1080         return false;
1081 
1082       // Otherwise we're good.
1083       return true;
1084     }
1085 
1086   public:
1087     ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1088                           FunctionArgList &Args)
1089       : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1090         ConstructorDecl(CD),
1091         MemcpyableCtor(CD->isDefaulted() &&
1092                        CD->isCopyOrMoveConstructor() &&
1093                        CGF.getLangOpts().getGC() == LangOptions::NonGC),
1094         Args(Args) { }
1095 
1096     void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1097       if (isMemberInitMemcpyable(MemberInit)) {
1098         AggregatedInits.push_back(MemberInit);
1099         addMemcpyableField(MemberInit->getMember());
1100       } else {
1101         emitAggregatedInits();
1102         EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1103                               ConstructorDecl, Args);
1104       }
1105     }
1106 
1107     void emitAggregatedInits() {
1108       if (AggregatedInits.size() <= 1) {
1109         // This memcpy is too small to be worthwhile. Fall back on default
1110         // codegen.
1111         if (!AggregatedInits.empty()) {
1112           CopyingValueRepresentation CVR(CGF);
1113           EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1114                                 AggregatedInits[0], ConstructorDecl, Args);
1115           AggregatedInits.clear();
1116         }
1117         reset();
1118         return;
1119       }
1120 
1121       pushEHDestructors();
1122       emitMemcpy();
1123       AggregatedInits.clear();
1124     }
1125 
1126     void pushEHDestructors() {
1127       Address ThisPtr = CGF.LoadCXXThisAddress();
1128       QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1129       LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
1130 
1131       for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
1132         CXXCtorInitializer *MemberInit = AggregatedInits[i];
1133         QualType FieldType = MemberInit->getAnyMember()->getType();
1134         QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1135         if (!CGF.needsEHCleanup(dtorKind))
1136           continue;
1137         LValue FieldLHS = LHS;
1138         EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
1139         CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType);
1140       }
1141     }
1142 
1143     void finish() {
1144       emitAggregatedInits();
1145     }
1146 
1147   private:
1148     const CXXConstructorDecl *ConstructorDecl;
1149     bool MemcpyableCtor;
1150     FunctionArgList &Args;
1151     SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1152   };
1153 
1154   class AssignmentMemcpyizer : public FieldMemcpyizer {
1155   private:
1156     // Returns the memcpyable field copied by the given statement, if one
1157     // exists. Otherwise returns null.
1158     FieldDecl *getMemcpyableField(Stmt *S) {
1159       if (!AssignmentsMemcpyable)
1160         return nullptr;
1161       if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
1162         // Recognise trivial assignments.
1163         if (BO->getOpcode() != BO_Assign)
1164           return nullptr;
1165         MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());
1166         if (!ME)
1167           return nullptr;
1168         FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1169         if (!Field || !isMemcpyableField(Field))
1170           return nullptr;
1171         Stmt *RHS = BO->getRHS();
1172         if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))
1173           RHS = EC->getSubExpr();
1174         if (!RHS)
1175           return nullptr;
1176         if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {
1177           if (ME2->getMemberDecl() == Field)
1178             return Field;
1179         }
1180         return nullptr;
1181       } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
1182         CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1183         if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
1184           return nullptr;
1185         MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
1186         if (!IOA)
1187           return nullptr;
1188         FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());
1189         if (!Field || !isMemcpyableField(Field))
1190           return nullptr;
1191         MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1192         if (!Arg0 || Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl()))
1193           return nullptr;
1194         return Field;
1195       } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
1196         FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());
1197         if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1198           return nullptr;
1199         Expr *DstPtr = CE->getArg(0);
1200         if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))
1201           DstPtr = DC->getSubExpr();
1202         UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);
1203         if (!DUO || DUO->getOpcode() != UO_AddrOf)
1204           return nullptr;
1205         MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());
1206         if (!ME)
1207           return nullptr;
1208         FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1209         if (!Field || !isMemcpyableField(Field))
1210           return nullptr;
1211         Expr *SrcPtr = CE->getArg(1);
1212         if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))
1213           SrcPtr = SC->getSubExpr();
1214         UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);
1215         if (!SUO || SUO->getOpcode() != UO_AddrOf)
1216           return nullptr;
1217         MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());
1218         if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))
1219           return nullptr;
1220         return Field;
1221       }
1222 
1223       return nullptr;
1224     }
1225 
1226     bool AssignmentsMemcpyable;
1227     SmallVector<Stmt*, 16> AggregatedStmts;
1228 
1229   public:
1230     AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1231                          FunctionArgList &Args)
1232       : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1233         AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1234       assert(Args.size() == 2);
1235     }
1236 
1237     void emitAssignment(Stmt *S) {
1238       FieldDecl *F = getMemcpyableField(S);
1239       if (F) {
1240         addMemcpyableField(F);
1241         AggregatedStmts.push_back(S);
1242       } else {
1243         emitAggregatedStmts();
1244         CGF.EmitStmt(S);
1245       }
1246     }
1247 
1248     void emitAggregatedStmts() {
1249       if (AggregatedStmts.size() <= 1) {
1250         if (!AggregatedStmts.empty()) {
1251           CopyingValueRepresentation CVR(CGF);
1252           CGF.EmitStmt(AggregatedStmts[0]);
1253         }
1254         reset();
1255       }
1256 
1257       emitMemcpy();
1258       AggregatedStmts.clear();
1259     }
1260 
1261     void finish() {
1262       emitAggregatedStmts();
1263     }
1264   };
1265 } // end anonymous namespace
1266 
1267 static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1268   const Type *BaseType = BaseInit->getBaseClass();
1269   const auto *BaseClassDecl =
1270       cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
1271   return BaseClassDecl->isDynamicClass();
1272 }
1273 
1274 /// EmitCtorPrologue - This routine generates necessary code to initialize
1275 /// base classes and non-static data members belonging to this constructor.
1276 void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1277                                        CXXCtorType CtorType,
1278                                        FunctionArgList &Args) {
1279   if (CD->isDelegatingConstructor())
1280     return EmitDelegatingCXXConstructorCall(CD, Args);
1281 
1282   const CXXRecordDecl *ClassDecl = CD->getParent();
1283 
1284   CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1285                                           E = CD->init_end();
1286 
1287   // Virtual base initializers first, if any. They aren't needed if:
1288   // - This is a base ctor variant
1289   // - There are no vbases
1290   // - The class is abstract, so a complete object of it cannot be constructed
1291   //
1292   // The check for an abstract class is necessary because sema may not have
1293   // marked virtual base destructors referenced.
1294   bool ConstructVBases = CtorType != Ctor_Base &&
1295                          ClassDecl->getNumVBases() != 0 &&
1296                          !ClassDecl->isAbstract();
1297 
1298   // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1299   // constructor of a class with virtual bases takes an additional parameter to
1300   // conditionally construct the virtual bases. Emit that check here.
1301   llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1302   if (ConstructVBases &&
1303       !CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1304     BaseCtorContinueBB =
1305         CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);
1306     assert(BaseCtorContinueBB);
1307   }
1308 
1309   llvm::Value *const OldThis = CXXThisValue;
1310   for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) {
1311     if (!ConstructVBases)
1312       continue;
1313     if (CGM.getCodeGenOpts().StrictVTablePointers &&
1314         CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1315         isInitializerOfDynamicClass(*B))
1316       CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1317     EmitBaseInitializer(*this, ClassDecl, *B);
1318   }
1319 
1320   if (BaseCtorContinueBB) {
1321     // Complete object handler should continue to the remaining initializers.
1322     Builder.CreateBr(BaseCtorContinueBB);
1323     EmitBlock(BaseCtorContinueBB);
1324   }
1325 
1326   // Then, non-virtual base initializers.
1327   for (; B != E && (*B)->isBaseInitializer(); B++) {
1328     assert(!(*B)->isBaseVirtual());
1329 
1330     if (CGM.getCodeGenOpts().StrictVTablePointers &&
1331         CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1332         isInitializerOfDynamicClass(*B))
1333       CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1334     EmitBaseInitializer(*this, ClassDecl, *B);
1335   }
1336 
1337   CXXThisValue = OldThis;
1338 
1339   InitializeVTablePointers(ClassDecl);
1340 
1341   // And finally, initialize class members.
1342   FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1343   ConstructorMemcpyizer CM(*this, CD, Args);
1344   for (; B != E; B++) {
1345     CXXCtorInitializer *Member = (*B);
1346     assert(!Member->isBaseInitializer());
1347     assert(Member->isAnyMemberInitializer() &&
1348            "Delegating initializer on non-delegating constructor");
1349     CM.addMemberInitializer(Member);
1350   }
1351   CM.finish();
1352 }
1353 
1354 static bool
1355 FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1356 
1357 static bool
1358 HasTrivialDestructorBody(ASTContext &Context,
1359                          const CXXRecordDecl *BaseClassDecl,
1360                          const CXXRecordDecl *MostDerivedClassDecl)
1361 {
1362   // If the destructor is trivial we don't have to check anything else.
1363   if (BaseClassDecl->hasTrivialDestructor())
1364     return true;
1365 
1366   if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1367     return false;
1368 
1369   // Check fields.
1370   for (const auto *Field : BaseClassDecl->fields())
1371     if (!FieldHasTrivialDestructorBody(Context, Field))
1372       return false;
1373 
1374   // Check non-virtual bases.
1375   for (const auto &I : BaseClassDecl->bases()) {
1376     if (I.isVirtual())
1377       continue;
1378 
1379     const CXXRecordDecl *NonVirtualBase =
1380       cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1381     if (!HasTrivialDestructorBody(Context, NonVirtualBase,
1382                                   MostDerivedClassDecl))
1383       return false;
1384   }
1385 
1386   if (BaseClassDecl == MostDerivedClassDecl) {
1387     // Check virtual bases.
1388     for (const auto &I : BaseClassDecl->vbases()) {
1389       const CXXRecordDecl *VirtualBase =
1390         cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1391       if (!HasTrivialDestructorBody(Context, VirtualBase,
1392                                     MostDerivedClassDecl))
1393         return false;
1394     }
1395   }
1396 
1397   return true;
1398 }
1399 
1400 static bool
1401 FieldHasTrivialDestructorBody(ASTContext &Context,
1402                                           const FieldDecl *Field)
1403 {
1404   QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1405 
1406   const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1407   if (!RT)
1408     return true;
1409 
1410   CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
1411 
1412   // The destructor for an implicit anonymous union member is never invoked.
1413   if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
1414     return false;
1415 
1416   return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
1417 }
1418 
1419 /// CanSkipVTablePointerInitialization - Check whether we need to initialize
1420 /// any vtable pointers before calling this destructor.
1421 static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1422                                                const CXXDestructorDecl *Dtor) {
1423   const CXXRecordDecl *ClassDecl = Dtor->getParent();
1424   if (!ClassDecl->isDynamicClass())
1425     return true;
1426 
1427   if (!Dtor->hasTrivialBody())
1428     return false;
1429 
1430   // Check the fields.
1431   for (const auto *Field : ClassDecl->fields())
1432     if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1433       return false;
1434 
1435   return true;
1436 }
1437 
1438 /// EmitDestructorBody - Emits the body of the current destructor.
1439 void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1440   const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
1441   CXXDtorType DtorType = CurGD.getDtorType();
1442 
1443   // For an abstract class, non-base destructors are never used (and can't
1444   // be emitted in general, because vbase dtors may not have been validated
1445   // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1446   // in fact emit references to them from other compilations, so emit them
1447   // as functions containing a trap instruction.
1448   if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {
1449     llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1450     TrapCall->setDoesNotReturn();
1451     TrapCall->setDoesNotThrow();
1452     Builder.CreateUnreachable();
1453     Builder.ClearInsertionPoint();
1454     return;
1455   }
1456 
1457   Stmt *Body = Dtor->getBody();
1458   if (Body)
1459     incrementProfileCounter(Body);
1460 
1461   // The call to operator delete in a deleting destructor happens
1462   // outside of the function-try-block, which means it's always
1463   // possible to delegate the destructor body to the complete
1464   // destructor.  Do so.
1465   if (DtorType == Dtor_Deleting) {
1466     RunCleanupsScope DtorEpilogue(*this);
1467     EnterDtorCleanups(Dtor, Dtor_Deleting);
1468     if (HaveInsertPoint()) {
1469       QualType ThisTy = Dtor->getThisObjectType();
1470       EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
1471                             /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1472     }
1473     return;
1474   }
1475 
1476   // If the body is a function-try-block, enter the try before
1477   // anything else.
1478   bool isTryBody = (Body && isa<CXXTryStmt>(Body));
1479   if (isTryBody)
1480     EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1481   EmitAsanPrologueOrEpilogue(false);
1482 
1483   // Enter the epilogue cleanups.
1484   RunCleanupsScope DtorEpilogue(*this);
1485 
1486   // If this is the complete variant, just invoke the base variant;
1487   // the epilogue will destruct the virtual bases.  But we can't do
1488   // this optimization if the body is a function-try-block, because
1489   // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
1490   // always delegate because we might not have a definition in this TU.
1491   switch (DtorType) {
1492   case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1493   case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1494 
1495   case Dtor_Complete:
1496     assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1497            "can't emit a dtor without a body for non-Microsoft ABIs");
1498 
1499     // Enter the cleanup scopes for virtual bases.
1500     EnterDtorCleanups(Dtor, Dtor_Complete);
1501 
1502     if (!isTryBody) {
1503       QualType ThisTy = Dtor->getThisObjectType();
1504       EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
1505                             /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1506       break;
1507     }
1508 
1509     // Fallthrough: act like we're in the base variant.
1510     LLVM_FALLTHROUGH;
1511 
1512   case Dtor_Base:
1513     assert(Body);
1514 
1515     // Enter the cleanup scopes for fields and non-virtual bases.
1516     EnterDtorCleanups(Dtor, Dtor_Base);
1517 
1518     // Initialize the vtable pointers before entering the body.
1519     if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
1520       // Insert the llvm.launder.invariant.group intrinsic before initializing
1521       // the vptrs to cancel any previous assumptions we might have made.
1522       if (CGM.getCodeGenOpts().StrictVTablePointers &&
1523           CGM.getCodeGenOpts().OptimizationLevel > 0)
1524         CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1525       InitializeVTablePointers(Dtor->getParent());
1526     }
1527 
1528     if (isTryBody)
1529       EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
1530     else if (Body)
1531       EmitStmt(Body);
1532     else {
1533       assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1534       // nothing to do besides what's in the epilogue
1535     }
1536     // -fapple-kext must inline any call to this dtor into
1537     // the caller's body.
1538     if (getLangOpts().AppleKext)
1539       CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1540 
1541     break;
1542   }
1543 
1544   // Jump out through the epilogue cleanups.
1545   DtorEpilogue.ForceCleanup();
1546 
1547   // Exit the try if applicable.
1548   if (isTryBody)
1549     ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1550 }
1551 
1552 void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1553   const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());
1554   const Stmt *RootS = AssignOp->getBody();
1555   assert(isa<CompoundStmt>(RootS) &&
1556          "Body of an implicit assignment operator should be compound stmt.");
1557   const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);
1558 
1559   LexicalScope Scope(*this, RootCS->getSourceRange());
1560 
1561   incrementProfileCounter(RootCS);
1562   AssignmentMemcpyizer AM(*this, AssignOp, Args);
1563   for (auto *I : RootCS->body())
1564     AM.emitAssignment(I);
1565   AM.finish();
1566 }
1567 
1568 namespace {
1569   llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1570                                      const CXXDestructorDecl *DD) {
1571     if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1572       return CGF.EmitScalarExpr(ThisArg);
1573     return CGF.LoadCXXThis();
1574   }
1575 
1576   /// Call the operator delete associated with the current destructor.
1577   struct CallDtorDelete final : EHScopeStack::Cleanup {
1578     CallDtorDelete() {}
1579 
1580     void Emit(CodeGenFunction &CGF, Flags flags) override {
1581       const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1582       const CXXRecordDecl *ClassDecl = Dtor->getParent();
1583       CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1584                          LoadThisForDtorDelete(CGF, Dtor),
1585                          CGF.getContext().getTagDeclType(ClassDecl));
1586     }
1587   };
1588 
1589   void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1590                                      llvm::Value *ShouldDeleteCondition,
1591                                      bool ReturnAfterDelete) {
1592     llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");
1593     llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");
1594     llvm::Value *ShouldCallDelete
1595       = CGF.Builder.CreateIsNull(ShouldDeleteCondition);
1596     CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);
1597 
1598     CGF.EmitBlock(callDeleteBB);
1599     const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1600     const CXXRecordDecl *ClassDecl = Dtor->getParent();
1601     CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1602                        LoadThisForDtorDelete(CGF, Dtor),
1603                        CGF.getContext().getTagDeclType(ClassDecl));
1604     assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1605                ReturnAfterDelete &&
1606            "unexpected value for ReturnAfterDelete");
1607     if (ReturnAfterDelete)
1608       CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
1609     else
1610       CGF.Builder.CreateBr(continueBB);
1611 
1612     CGF.EmitBlock(continueBB);
1613   }
1614 
1615   struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1616     llvm::Value *ShouldDeleteCondition;
1617 
1618   public:
1619     CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1620         : ShouldDeleteCondition(ShouldDeleteCondition) {
1621       assert(ShouldDeleteCondition != nullptr);
1622     }
1623 
1624     void Emit(CodeGenFunction &CGF, Flags flags) override {
1625       EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1626                                     /*ReturnAfterDelete*/false);
1627     }
1628   };
1629 
1630   class DestroyField  final : public EHScopeStack::Cleanup {
1631     const FieldDecl *field;
1632     CodeGenFunction::Destroyer *destroyer;
1633     bool useEHCleanupForArray;
1634 
1635   public:
1636     DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1637                  bool useEHCleanupForArray)
1638         : field(field), destroyer(destroyer),
1639           useEHCleanupForArray(useEHCleanupForArray) {}
1640 
1641     void Emit(CodeGenFunction &CGF, Flags flags) override {
1642       // Find the address of the field.
1643       Address thisValue = CGF.LoadCXXThisAddress();
1644       QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1645       LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
1646       LValue LV = CGF.EmitLValueForField(ThisLV, field);
1647       assert(LV.isSimple());
1648 
1649       CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer,
1650                       flags.isForNormalCleanup() && useEHCleanupForArray);
1651     }
1652   };
1653 
1654  static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1655              CharUnits::QuantityType PoisonSize) {
1656    CodeGenFunction::SanitizerScope SanScope(&CGF);
1657    // Pass in void pointer and size of region as arguments to runtime
1658    // function
1659    llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
1660                           llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
1661 
1662    llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
1663 
1664    llvm::FunctionType *FnType =
1665        llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
1666    llvm::FunctionCallee Fn =
1667        CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
1668    CGF.EmitNounwindRuntimeCall(Fn, Args);
1669  }
1670 
1671   class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
1672     const CXXDestructorDecl *Dtor;
1673 
1674   public:
1675     SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1676 
1677     // Generate function call for handling object poisoning.
1678     // Disables tail call elimination, to prevent the current stack frame
1679     // from disappearing from the stack trace.
1680     void Emit(CodeGenFunction &CGF, Flags flags) override {
1681       const ASTRecordLayout &Layout =
1682           CGF.getContext().getASTRecordLayout(Dtor->getParent());
1683 
1684       // Nothing to poison.
1685       if (Layout.getFieldCount() == 0)
1686         return;
1687 
1688       // Prevent the current stack frame from disappearing from the stack trace.
1689       CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1690 
1691       // Construct pointer to region to begin poisoning, and calculate poison
1692       // size, so that only members declared in this class are poisoned.
1693       ASTContext &Context = CGF.getContext();
1694 
1695       const RecordDecl *Decl = Dtor->getParent();
1696       auto Fields = Decl->fields();
1697       auto IsTrivial = [&](const FieldDecl *F) {
1698         return FieldHasTrivialDestructorBody(Context, F);
1699       };
1700 
1701       auto IsZeroSize = [&](const FieldDecl *F) {
1702         return F->isZeroSize(Context);
1703       };
1704 
1705       // Poison blocks of fields with trivial destructors making sure that block
1706       // begin and end do not point to zero-sized fields. They don't have
1707       // correct offsets so can't be used to calculate poisoning range.
1708       for (auto It = Fields.begin(); It != Fields.end();) {
1709         It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
1710           return IsTrivial(F) && !IsZeroSize(F);
1711         });
1712         if (It == Fields.end())
1713           break;
1714         auto Start = It++;
1715         It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
1716           return !IsTrivial(F) && !IsZeroSize(F);
1717         });
1718 
1719         PoisonMembers(CGF, (*Start)->getFieldIndex(),
1720                       It == Fields.end() ? -1 : (*It)->getFieldIndex());
1721       }
1722     }
1723 
1724   private:
1725     /// \param layoutStartOffset index of the ASTRecordLayout field to
1726     ///     start poisoning (inclusive)
1727     /// \param layoutEndOffset index of the ASTRecordLayout field to
1728     ///     end poisoning (exclusive)
1729     void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
1730                        unsigned layoutEndOffset) {
1731       ASTContext &Context = CGF.getContext();
1732       const ASTRecordLayout &Layout =
1733           Context.getASTRecordLayout(Dtor->getParent());
1734 
1735       // It's a first trivia field so it should be at the begining of char,
1736       // still round up start offset just in case.
1737       CharUnits PoisonStart =
1738           Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset) +
1739                                       Context.getCharWidth() - 1);
1740       llvm::ConstantInt *OffsetSizePtr =
1741           llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity());
1742 
1743       llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
1744           CGF.Int8Ty,
1745           CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
1746           OffsetSizePtr);
1747 
1748       CharUnits PoisonEnd;
1749       if (layoutEndOffset >= Layout.getFieldCount()) {
1750         PoisonEnd = Layout.getNonVirtualSize();
1751       } else {
1752         PoisonEnd =
1753             Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutEndOffset));
1754       }
1755       CharUnits PoisonSize = PoisonEnd - PoisonStart;
1756       if (!PoisonSize.isPositive())
1757         return;
1758 
1759       EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize.getQuantity());
1760     }
1761   };
1762 
1763  class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1764     const CXXDestructorDecl *Dtor;
1765 
1766   public:
1767     SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1768 
1769     // Generate function call for handling vtable pointer poisoning.
1770     void Emit(CodeGenFunction &CGF, Flags flags) override {
1771       assert(Dtor->getParent()->isDynamicClass());
1772       (void)Dtor;
1773       ASTContext &Context = CGF.getContext();
1774       // Poison vtable and vtable ptr if they exist for this class.
1775       llvm::Value *VTablePtr = CGF.LoadCXXThis();
1776 
1777       CharUnits::QuantityType PoisonSize =
1778           Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
1779       // Pass in void pointer and size of region as arguments to runtime
1780       // function
1781       EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
1782     }
1783  };
1784 } // end anonymous namespace
1785 
1786 /// Emit all code that comes at the end of class's
1787 /// destructor. This is to call destructors on members and base classes
1788 /// in reverse order of their construction.
1789 ///
1790 /// For a deleting destructor, this also handles the case where a destroying
1791 /// operator delete completely overrides the definition.
1792 void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1793                                         CXXDtorType DtorType) {
1794   assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1795          "Should not emit dtor epilogue for non-exported trivial dtor!");
1796 
1797   // The deleting-destructor phase just needs to call the appropriate
1798   // operator delete that Sema picked up.
1799   if (DtorType == Dtor_Deleting) {
1800     assert(DD->getOperatorDelete() &&
1801            "operator delete missing - EnterDtorCleanups");
1802     if (CXXStructorImplicitParamValue) {
1803       // If there is an implicit param to the deleting dtor, it's a boolean
1804       // telling whether this is a deleting destructor.
1805       if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1806         EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,
1807                                       /*ReturnAfterDelete*/true);
1808       else
1809         EHStack.pushCleanup<CallDtorDeleteConditional>(
1810             NormalAndEHCleanup, CXXStructorImplicitParamValue);
1811     } else {
1812       if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1813         const CXXRecordDecl *ClassDecl = DD->getParent();
1814         EmitDeleteCall(DD->getOperatorDelete(),
1815                        LoadThisForDtorDelete(*this, DD),
1816                        getContext().getTagDeclType(ClassDecl));
1817         EmitBranchThroughCleanup(ReturnBlock);
1818       } else {
1819         EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
1820       }
1821     }
1822     return;
1823   }
1824 
1825   const CXXRecordDecl *ClassDecl = DD->getParent();
1826 
1827   // Unions have no bases and do not call field destructors.
1828   if (ClassDecl->isUnion())
1829     return;
1830 
1831   // The complete-destructor phase just destructs all the virtual bases.
1832   if (DtorType == Dtor_Complete) {
1833     // Poison the vtable pointer such that access after the base
1834     // and member destructors are invoked is invalid.
1835     if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1836         SanOpts.has(SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
1837         ClassDecl->isPolymorphic())
1838       EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1839 
1840     // We push them in the forward order so that they'll be popped in
1841     // the reverse order.
1842     for (const auto &Base : ClassDecl->vbases()) {
1843       auto *BaseClassDecl =
1844           cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1845 
1846       // Ignore trivial destructors.
1847       if (BaseClassDecl->hasTrivialDestructor())
1848         continue;
1849 
1850       EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1851                                         BaseClassDecl,
1852                                         /*BaseIsVirtual*/ true);
1853     }
1854 
1855     return;
1856   }
1857 
1858   assert(DtorType == Dtor_Base);
1859   // Poison the vtable pointer if it has no virtual bases, but inherits
1860   // virtual functions.
1861   if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1862       SanOpts.has(SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
1863       ClassDecl->isPolymorphic())
1864     EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1865 
1866   // Destroy non-virtual bases.
1867   for (const auto &Base : ClassDecl->bases()) {
1868     // Ignore virtual bases.
1869     if (Base.isVirtual())
1870       continue;
1871 
1872     CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1873 
1874     // Ignore trivial destructors.
1875     if (BaseClassDecl->hasTrivialDestructor())
1876       continue;
1877 
1878     EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1879                                       BaseClassDecl,
1880                                       /*BaseIsVirtual*/ false);
1881   }
1882 
1883   // Poison fields such that access after their destructors are
1884   // invoked, and before the base class destructor runs, is invalid.
1885   if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1886       SanOpts.has(SanitizerKind::Memory))
1887     EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
1888 
1889   // Destroy direct fields.
1890   for (const auto *Field : ClassDecl->fields()) {
1891     QualType type = Field->getType();
1892     QualType::DestructionKind dtorKind = type.isDestructedType();
1893     if (!dtorKind) continue;
1894 
1895     // Anonymous union members do not have their destructors called.
1896     const RecordType *RT = type->getAsUnionType();
1897     if (RT && RT->getDecl()->isAnonymousStructOrUnion()) continue;
1898 
1899     CleanupKind cleanupKind = getCleanupKind(dtorKind);
1900     EHStack.pushCleanup<DestroyField>(cleanupKind, Field,
1901                                       getDestroyer(dtorKind),
1902                                       cleanupKind & EHCleanup);
1903   }
1904 }
1905 
1906 /// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1907 /// constructor for each of several members of an array.
1908 ///
1909 /// \param ctor the constructor to call for each element
1910 /// \param arrayType the type of the array to initialize
1911 /// \param arrayBegin an arrayType*
1912 /// \param zeroInitialize true if each element should be
1913 ///   zero-initialized before it is constructed
1914 void CodeGenFunction::EmitCXXAggrConstructorCall(
1915     const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1916     Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1917     bool zeroInitialize) {
1918   QualType elementType;
1919   llvm::Value *numElements =
1920     emitArrayLength(arrayType, elementType, arrayBegin);
1921 
1922   EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,
1923                              NewPointerIsChecked, zeroInitialize);
1924 }
1925 
1926 /// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1927 /// constructor for each of several members of an array.
1928 ///
1929 /// \param ctor the constructor to call for each element
1930 /// \param numElements the number of elements in the array;
1931 ///   may be zero
1932 /// \param arrayBase a T*, where T is the type constructed by ctor
1933 /// \param zeroInitialize true if each element should be
1934 ///   zero-initialized before it is constructed
1935 void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1936                                                  llvm::Value *numElements,
1937                                                  Address arrayBase,
1938                                                  const CXXConstructExpr *E,
1939                                                  bool NewPointerIsChecked,
1940                                                  bool zeroInitialize) {
1941   // It's legal for numElements to be zero.  This can happen both
1942   // dynamically, because x can be zero in 'new A[x]', and statically,
1943   // because of GCC extensions that permit zero-length arrays.  There
1944   // are probably legitimate places where we could assume that this
1945   // doesn't happen, but it's not clear that it's worth it.
1946   llvm::BranchInst *zeroCheckBranch = nullptr;
1947 
1948   // Optimize for a constant count.
1949   llvm::ConstantInt *constantCount
1950     = dyn_cast<llvm::ConstantInt>(numElements);
1951   if (constantCount) {
1952     // Just skip out if the constant count is zero.
1953     if (constantCount->isZero()) return;
1954 
1955   // Otherwise, emit the check.
1956   } else {
1957     llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");
1958     llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");
1959     zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
1960     EmitBlock(loopBB);
1961   }
1962 
1963   // Find the end of the array.
1964   llvm::Type *elementType = arrayBase.getElementType();
1965   llvm::Value *arrayBegin = arrayBase.getPointer();
1966   llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
1967       elementType, arrayBegin, numElements, "arrayctor.end");
1968 
1969   // Enter the loop, setting up a phi for the current location to initialize.
1970   llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1971   llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");
1972   EmitBlock(loopBB);
1973   llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,
1974                                          "arrayctor.cur");
1975   cur->addIncoming(arrayBegin, entryBB);
1976 
1977   // Inside the loop body, emit the constructor call on the array element.
1978 
1979   // The alignment of the base, adjusted by the size of a single element,
1980   // provides a conservative estimate of the alignment of every element.
1981   // (This assumes we never start tracking offsetted alignments.)
1982   //
1983   // Note that these are complete objects and so we don't need to
1984   // use the non-virtual size or alignment.
1985   QualType type = getContext().getTypeDeclType(ctor->getParent());
1986   CharUnits eltAlignment =
1987     arrayBase.getAlignment()
1988              .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
1989   Address curAddr = Address(cur, eltAlignment);
1990 
1991   // Zero initialize the storage, if requested.
1992   if (zeroInitialize)
1993     EmitNullInitialization(curAddr, type);
1994 
1995   // C++ [class.temporary]p4:
1996   // There are two contexts in which temporaries are destroyed at a different
1997   // point than the end of the full-expression. The first context is when a
1998   // default constructor is called to initialize an element of an array.
1999   // If the constructor has one or more default arguments, the destruction of
2000   // every temporary created in a default argument expression is sequenced
2001   // before the construction of the next array element, if any.
2002 
2003   {
2004     RunCleanupsScope Scope(*this);
2005 
2006     // Evaluate the constructor and its arguments in a regular
2007     // partial-destroy cleanup.
2008     if (getLangOpts().Exceptions &&
2009         !ctor->getParent()->hasTrivialDestructor()) {
2010       Destroyer *destroyer = destroyCXXObject;
2011       pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
2012                                      *destroyer);
2013     }
2014     auto currAVS = AggValueSlot::forAddr(
2015         curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,
2016         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2017         AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,
2018         NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2019                             : AggValueSlot::IsNotSanitizerChecked);
2020     EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2021                            /*Delegating=*/false, currAVS, E);
2022   }
2023 
2024   // Go to the next element.
2025   llvm::Value *next = Builder.CreateInBoundsGEP(
2026       elementType, cur, llvm::ConstantInt::get(SizeTy, 1), "arrayctor.next");
2027   cur->addIncoming(next, Builder.GetInsertBlock());
2028 
2029   // Check whether that's the end of the loop.
2030   llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");
2031   llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");
2032   Builder.CreateCondBr(done, contBB, loopBB);
2033 
2034   // Patch the earlier check to skip over the loop.
2035   if (zeroCheckBranch) zeroCheckBranch->setSuccessor(0, contBB);
2036 
2037   EmitBlock(contBB);
2038 }
2039 
2040 void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2041                                        Address addr,
2042                                        QualType type) {
2043   const RecordType *rtype = type->castAs<RecordType>();
2044   const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
2045   const CXXDestructorDecl *dtor = record->getDestructor();
2046   assert(!dtor->isTrivial());
2047   CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
2048                             /*Delegating=*/false, addr, type);
2049 }
2050 
2051 void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2052                                              CXXCtorType Type,
2053                                              bool ForVirtualBase,
2054                                              bool Delegating,
2055                                              AggValueSlot ThisAVS,
2056                                              const CXXConstructExpr *E) {
2057   CallArgList Args;
2058   Address This = ThisAVS.getAddress();
2059   LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2060   QualType ThisType = D->getThisType();
2061   LangAS ThisAS = ThisType.getTypePtr()->getPointeeType().getAddressSpace();
2062   llvm::Value *ThisPtr = This.getPointer();
2063 
2064   if (SlotAS != ThisAS) {
2065     unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
2066     llvm::Type *NewType =
2067         ThisPtr->getType()->getPointerElementType()->getPointerTo(TargetThisAS);
2068     ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
2069                                                     ThisAS, SlotAS, NewType);
2070   }
2071 
2072   // Push the this ptr.
2073   Args.add(RValue::get(ThisPtr), D->getThisType());
2074 
2075   // If this is a trivial constructor, emit a memcpy now before we lose
2076   // the alignment information on the argument.
2077   // FIXME: It would be better to preserve alignment information into CallArg.
2078   if (isMemcpyEquivalentSpecialMember(D)) {
2079     assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2080 
2081     const Expr *Arg = E->getArg(0);
2082     LValue Src = EmitLValue(Arg);
2083     QualType DestTy = getContext().getTypeDeclType(D->getParent());
2084     LValue Dest = MakeAddrLValue(This, DestTy);
2085     EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
2086     return;
2087   }
2088 
2089   // Add the rest of the user-supplied arguments.
2090   const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2091   EvaluationOrder Order = E->isListInitialization()
2092                               ? EvaluationOrder::ForceLeftToRight
2093                               : EvaluationOrder::Default;
2094   EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),
2095                /*ParamsToSkip*/ 0, Order);
2096 
2097   EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2098                          ThisAVS.mayOverlap(), E->getExprLoc(),
2099                          ThisAVS.isSanitizerChecked());
2100 }
2101 
2102 static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2103                                     const CXXConstructorDecl *Ctor,
2104                                     CXXCtorType Type, CallArgList &Args) {
2105   // We can't forward a variadic call.
2106   if (Ctor->isVariadic())
2107     return false;
2108 
2109   if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2110     // If the parameters are callee-cleanup, it's not safe to forward.
2111     for (auto *P : Ctor->parameters())
2112       if (P->needsDestruction(CGF.getContext()))
2113         return false;
2114 
2115     // Likewise if they're inalloca.
2116     const CGFunctionInfo &Info =
2117         CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);
2118     if (Info.usesInAlloca())
2119       return false;
2120   }
2121 
2122   // Anything else should be OK.
2123   return true;
2124 }
2125 
2126 void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2127                                              CXXCtorType Type,
2128                                              bool ForVirtualBase,
2129                                              bool Delegating,
2130                                              Address This,
2131                                              CallArgList &Args,
2132                                              AggValueSlot::Overlap_t Overlap,
2133                                              SourceLocation Loc,
2134                                              bool NewPointerIsChecked) {
2135   const CXXRecordDecl *ClassDecl = D->getParent();
2136 
2137   if (!NewPointerIsChecked)
2138     EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
2139                   getContext().getRecordType(ClassDecl), CharUnits::Zero());
2140 
2141   if (D->isTrivial() && D->isDefaultConstructor()) {
2142     assert(Args.size() == 1 && "trivial default ctor with args");
2143     return;
2144   }
2145 
2146   // If this is a trivial constructor, just emit what's needed. If this is a
2147   // union copy constructor, we must emit a memcpy, because the AST does not
2148   // model that copy.
2149   if (isMemcpyEquivalentSpecialMember(D)) {
2150     assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2151 
2152     QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2153     Address Src(Args[1].getRValue(*this).getScalarVal(),
2154                 CGM.getNaturalTypeAlignment(SrcTy));
2155     LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
2156     QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2157     LValue DestLVal = MakeAddrLValue(This, DestTy);
2158     EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
2159     return;
2160   }
2161 
2162   bool PassPrototypeArgs = true;
2163   // Check whether we can actually emit the constructor before trying to do so.
2164   if (auto Inherited = D->getInheritedConstructor()) {
2165     PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2166     if (PassPrototypeArgs && !canEmitDelegateCallArgs(*this, D, Type, Args)) {
2167       EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,
2168                                               Delegating, Args);
2169       return;
2170     }
2171   }
2172 
2173   // Insert any ABI-specific implicit constructor arguments.
2174   CGCXXABI::AddedStructorArgCounts ExtraArgs =
2175       CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,
2176                                                  Delegating, Args);
2177 
2178   // Emit the call.
2179   llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));
2180   const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2181       Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
2182   CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2183   EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, false, Loc);
2184 
2185   // Generate vtable assumptions if we're constructing a complete object
2186   // with a vtable.  We don't do this for base subobjects for two reasons:
2187   // first, it's incorrect for classes with virtual bases, and second, we're
2188   // about to overwrite the vptrs anyway.
2189   // We also have to make sure if we can refer to vtable:
2190   // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2191   // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2192   // sure that definition of vtable is not hidden,
2193   // then we are always safe to refer to it.
2194   // FIXME: It looks like InstCombine is very inefficient on dealing with
2195   // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2196   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2197       ClassDecl->isDynamicClass() && Type != Ctor_Base &&
2198       CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) &&
2199       CGM.getCodeGenOpts().StrictVTablePointers)
2200     EmitVTableAssumptionLoads(ClassDecl, This);
2201 }
2202 
2203 void CodeGenFunction::EmitInheritedCXXConstructorCall(
2204     const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2205     bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2206   CallArgList Args;
2207   CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
2208 
2209   // Forward the parameters.
2210   if (InheritedFromVBase &&
2211       CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2212     // Nothing to do; this construction is not responsible for constructing
2213     // the base class containing the inherited constructor.
2214     // FIXME: Can we just pass undef's for the remaining arguments if we don't
2215     // have constructor variants?
2216     Args.push_back(ThisArg);
2217   } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2218     // The inheriting constructor was inlined; just inject its arguments.
2219     assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2220            "wrong number of parameters for inherited constructor call");
2221     Args = CXXInheritedCtorInitExprArgs;
2222     Args[0] = ThisArg;
2223   } else {
2224     // The inheriting constructor was not inlined. Emit delegating arguments.
2225     Args.push_back(ThisArg);
2226     const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);
2227     assert(OuterCtor->getNumParams() == D->getNumParams());
2228     assert(!OuterCtor->isVariadic() && "should have been inlined");
2229 
2230     for (const auto *Param : OuterCtor->parameters()) {
2231       assert(getContext().hasSameUnqualifiedType(
2232           OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2233           Param->getType()));
2234       EmitDelegateCallArg(Args, Param, E->getLocation());
2235 
2236       // Forward __attribute__(pass_object_size).
2237       if (Param->hasAttr<PassObjectSizeAttr>()) {
2238         auto *POSParam = SizeArguments[Param];
2239         assert(POSParam && "missing pass_object_size value for forwarding");
2240         EmitDelegateCallArg(Args, POSParam, E->getLocation());
2241       }
2242     }
2243   }
2244 
2245   EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,
2246                          This, Args, AggValueSlot::MayOverlap,
2247                          E->getLocation(), /*NewPointerIsChecked*/true);
2248 }
2249 
2250 void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2251     const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2252     bool Delegating, CallArgList &Args) {
2253   GlobalDecl GD(Ctor, CtorType);
2254   InlinedInheritingConstructorScope Scope(*this, GD);
2255   ApplyInlineDebugLocation DebugScope(*this, GD);
2256   RunCleanupsScope RunCleanups(*this);
2257 
2258   // Save the arguments to be passed to the inherited constructor.
2259   CXXInheritedCtorInitExprArgs = Args;
2260 
2261   FunctionArgList Params;
2262   QualType RetType = BuildFunctionArgList(CurGD, Params);
2263   FnRetTy = RetType;
2264 
2265   // Insert any ABI-specific implicit constructor arguments.
2266   CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,
2267                                              ForVirtualBase, Delegating, Args);
2268 
2269   // Emit a simplified prolog. We only need to emit the implicit params.
2270   assert(Args.size() >= Params.size() && "too few arguments for call");
2271   for (unsigned I = 0, N = Args.size(); I != N; ++I) {
2272     if (I < Params.size() && isa<ImplicitParamDecl>(Params[I])) {
2273       const RValue &RV = Args[I].getRValue(*this);
2274       assert(!RV.isComplex() && "complex indirect params not supported");
2275       ParamValue Val = RV.isScalar()
2276                            ? ParamValue::forDirect(RV.getScalarVal())
2277                            : ParamValue::forIndirect(RV.getAggregateAddress());
2278       EmitParmDecl(*Params[I], Val, I + 1);
2279     }
2280   }
2281 
2282   // Create a return value slot if the ABI implementation wants one.
2283   // FIXME: This is dumb, we should ask the ABI not to try to set the return
2284   // value instead.
2285   if (!RetType->isVoidType())
2286     ReturnValue = CreateIRTemp(RetType, "retval.inhctor");
2287 
2288   CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
2289   CXXThisValue = CXXABIThisValue;
2290 
2291   // Directly emit the constructor initializers.
2292   EmitCtorPrologue(Ctor, CtorType, Params);
2293 }
2294 
2295 void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2296   llvm::Value *VTableGlobal =
2297       CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
2298   if (!VTableGlobal)
2299     return;
2300 
2301   // We can just use the base offset in the complete class.
2302   CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2303 
2304   if (!NonVirtualOffset.isZero())
2305     This =
2306         ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
2307                                         Vptr.VTableClass, Vptr.NearestVBase);
2308 
2309   llvm::Value *VPtrValue =
2310       GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
2311   llvm::Value *Cmp =
2312       Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
2313   Builder.CreateAssumption(Cmp);
2314 }
2315 
2316 void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2317                                                 Address This) {
2318   if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
2319     for (const VPtr &Vptr : getVTablePointers(ClassDecl))
2320       EmitVTableAssumptionLoad(Vptr, This);
2321 }
2322 
2323 void
2324 CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2325                                                 Address This, Address Src,
2326                                                 const CXXConstructExpr *E) {
2327   const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2328 
2329   CallArgList Args;
2330 
2331   // Push the this ptr.
2332   Args.add(RValue::get(This.getPointer()), D->getThisType());
2333 
2334   // Push the src ptr.
2335   QualType QT = *(FPT->param_type_begin());
2336   llvm::Type *t = CGM.getTypes().ConvertType(QT);
2337   Src = Builder.CreateBitCast(Src, t);
2338   Args.add(RValue::get(Src.getPointer()), QT);
2339 
2340   // Skip over first argument (Src).
2341   EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
2342                /*ParamsToSkip*/ 1);
2343 
2344   EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2345                          /*Delegating*/false, This, Args,
2346                          AggValueSlot::MayOverlap, E->getExprLoc(),
2347                          /*NewPointerIsChecked*/false);
2348 }
2349 
2350 void
2351 CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2352                                                 CXXCtorType CtorType,
2353                                                 const FunctionArgList &Args,
2354                                                 SourceLocation Loc) {
2355   CallArgList DelegateArgs;
2356 
2357   FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2358   assert(I != E && "no parameters to constructor");
2359 
2360   // this
2361   Address This = LoadCXXThisAddress();
2362   DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
2363   ++I;
2364 
2365   // FIXME: The location of the VTT parameter in the parameter list is
2366   // specific to the Itanium ABI and shouldn't be hardcoded here.
2367   if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
2368     assert(I != E && "cannot skip vtt parameter, already done with args");
2369     assert((*I)->getType()->isPointerType() &&
2370            "skipping parameter not of vtt type");
2371     ++I;
2372   }
2373 
2374   // Explicit arguments.
2375   for (; I != E; ++I) {
2376     const VarDecl *param = *I;
2377     // FIXME: per-argument source location
2378     EmitDelegateCallArg(DelegateArgs, param, Loc);
2379   }
2380 
2381   EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,
2382                          /*Delegating=*/true, This, DelegateArgs,
2383                          AggValueSlot::MayOverlap, Loc,
2384                          /*NewPointerIsChecked=*/true);
2385 }
2386 
2387 namespace {
2388   struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2389     const CXXDestructorDecl *Dtor;
2390     Address Addr;
2391     CXXDtorType Type;
2392 
2393     CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2394                            CXXDtorType Type)
2395       : Dtor(D), Addr(Addr), Type(Type) {}
2396 
2397     void Emit(CodeGenFunction &CGF, Flags flags) override {
2398       // We are calling the destructor from within the constructor.
2399       // Therefore, "this" should have the expected type.
2400       QualType ThisTy = Dtor->getThisObjectType();
2401       CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,
2402                                 /*Delegating=*/true, Addr, ThisTy);
2403     }
2404   };
2405 } // end anonymous namespace
2406 
2407 void
2408 CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2409                                                   const FunctionArgList &Args) {
2410   assert(Ctor->isDelegatingConstructor());
2411 
2412   Address ThisPtr = LoadCXXThisAddress();
2413 
2414   AggValueSlot AggSlot =
2415     AggValueSlot::forAddr(ThisPtr, Qualifiers(),
2416                           AggValueSlot::IsDestructed,
2417                           AggValueSlot::DoesNotNeedGCBarriers,
2418                           AggValueSlot::IsNotAliased,
2419                           AggValueSlot::MayOverlap,
2420                           AggValueSlot::IsNotZeroed,
2421                           // Checks are made by the code that calls constructor.
2422                           AggValueSlot::IsSanitizerChecked);
2423 
2424   EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
2425 
2426   const CXXRecordDecl *ClassDecl = Ctor->getParent();
2427   if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {
2428     CXXDtorType Type =
2429       CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;
2430 
2431     EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,
2432                                                 ClassDecl->getDestructor(),
2433                                                 ThisPtr, Type);
2434   }
2435 }
2436 
2437 void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2438                                             CXXDtorType Type,
2439                                             bool ForVirtualBase,
2440                                             bool Delegating, Address This,
2441                                             QualType ThisTy) {
2442   CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
2443                                      Delegating, This, ThisTy);
2444 }
2445 
2446 namespace {
2447   struct CallLocalDtor final : EHScopeStack::Cleanup {
2448     const CXXDestructorDecl *Dtor;
2449     Address Addr;
2450     QualType Ty;
2451 
2452     CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2453         : Dtor(D), Addr(Addr), Ty(Ty) {}
2454 
2455     void Emit(CodeGenFunction &CGF, Flags flags) override {
2456       CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2457                                 /*ForVirtualBase=*/false,
2458                                 /*Delegating=*/false, Addr, Ty);
2459     }
2460   };
2461 } // end anonymous namespace
2462 
2463 void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2464                                             QualType T, Address Addr) {
2465   EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);
2466 }
2467 
2468 void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2469   CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2470   if (!ClassDecl) return;
2471   if (ClassDecl->hasTrivialDestructor()) return;
2472 
2473   const CXXDestructorDecl *D = ClassDecl->getDestructor();
2474   assert(D && D->isUsed() && "destructor not marked as used!");
2475   PushDestructorCleanup(D, T, Addr);
2476 }
2477 
2478 void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2479   // Compute the address point.
2480   llvm::Value *VTableAddressPoint =
2481       CGM.getCXXABI().getVTableAddressPointInStructor(
2482           *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
2483 
2484   if (!VTableAddressPoint)
2485     return;
2486 
2487   // Compute where to store the address point.
2488   llvm::Value *VirtualOffset = nullptr;
2489   CharUnits NonVirtualOffset = CharUnits::Zero();
2490 
2491   if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
2492     // We need to use the virtual base offset offset because the virtual base
2493     // might have a different offset in the most derived class.
2494 
2495     VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2496         *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
2497     NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2498   } else {
2499     // We can just use the base offset in the complete class.
2500     NonVirtualOffset = Vptr.Base.getBaseOffset();
2501   }
2502 
2503   // Apply the offsets.
2504   Address VTableField = LoadCXXThisAddress();
2505 
2506   if (!NonVirtualOffset.isZero() || VirtualOffset)
2507     VTableField = ApplyNonVirtualAndVirtualOffset(
2508         *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
2509         Vptr.NearestVBase);
2510 
2511   // Finally, store the address point. Use the same LLVM types as the field to
2512   // support optimization.
2513   unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2514   unsigned ProgAS = CGM.getDataLayout().getProgramAddressSpace();
2515   llvm::Type *VTablePtrTy =
2516       llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
2517           ->getPointerTo(ProgAS)
2518           ->getPointerTo(GlobalsAS);
2519   // vtable field is is derived from `this` pointer, therefore it should be in
2520   // default address space.
2521   VTableField = Builder.CreatePointerBitCastOrAddrSpaceCast(
2522       VTableField, VTablePtrTy->getPointerTo());
2523   VTableAddressPoint = Builder.CreatePointerBitCastOrAddrSpaceCast(
2524       VTableAddressPoint, VTablePtrTy);
2525 
2526   llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
2527   TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrTy);
2528   CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2529   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2530       CGM.getCodeGenOpts().StrictVTablePointers)
2531     CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
2532 }
2533 
2534 CodeGenFunction::VPtrsVector
2535 CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2536   CodeGenFunction::VPtrsVector VPtrsResult;
2537   VisitedVirtualBasesSetTy VBases;
2538   getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
2539                     /*NearestVBase=*/nullptr,
2540                     /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2541                     /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2542                     VPtrsResult);
2543   return VPtrsResult;
2544 }
2545 
2546 void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2547                                         const CXXRecordDecl *NearestVBase,
2548                                         CharUnits OffsetFromNearestVBase,
2549                                         bool BaseIsNonVirtualPrimaryBase,
2550                                         const CXXRecordDecl *VTableClass,
2551                                         VisitedVirtualBasesSetTy &VBases,
2552                                         VPtrsVector &Vptrs) {
2553   // If this base is a non-virtual primary base the address point has already
2554   // been set.
2555   if (!BaseIsNonVirtualPrimaryBase) {
2556     // Initialize the vtable pointer for this base.
2557     VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
2558     Vptrs.push_back(Vptr);
2559   }
2560 
2561   const CXXRecordDecl *RD = Base.getBase();
2562 
2563   // Traverse bases.
2564   for (const auto &I : RD->bases()) {
2565     auto *BaseDecl =
2566         cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
2567 
2568     // Ignore classes without a vtable.
2569     if (!BaseDecl->isDynamicClass())
2570       continue;
2571 
2572     CharUnits BaseOffset;
2573     CharUnits BaseOffsetFromNearestVBase;
2574     bool BaseDeclIsNonVirtualPrimaryBase;
2575 
2576     if (I.isVirtual()) {
2577       // Check if we've visited this virtual base before.
2578       if (!VBases.insert(BaseDecl).second)
2579         continue;
2580 
2581       const ASTRecordLayout &Layout =
2582         getContext().getASTRecordLayout(VTableClass);
2583 
2584       BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
2585       BaseOffsetFromNearestVBase = CharUnits::Zero();
2586       BaseDeclIsNonVirtualPrimaryBase = false;
2587     } else {
2588       const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2589 
2590       BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
2591       BaseOffsetFromNearestVBase =
2592         OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
2593       BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2594     }
2595 
2596     getVTablePointers(
2597         BaseSubobject(BaseDecl, BaseOffset),
2598         I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase,
2599         BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2600   }
2601 }
2602 
2603 void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2604   // Ignore classes without a vtable.
2605   if (!RD->isDynamicClass())
2606     return;
2607 
2608   // Initialize the vtable pointers for this class and all of its bases.
2609   if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
2610     for (const VPtr &Vptr : getVTablePointers(RD))
2611       InitializeVTablePointer(Vptr);
2612 
2613   if (RD->getNumVBases())
2614     CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
2615 }
2616 
2617 llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2618                                            llvm::Type *VTableTy,
2619                                            const CXXRecordDecl *RD) {
2620   Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
2621   llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
2622   TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);
2623   CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);
2624 
2625   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2626       CGM.getCodeGenOpts().StrictVTablePointers)
2627     CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
2628 
2629   return VTable;
2630 }
2631 
2632 // If a class has a single non-virtual base and does not introduce or override
2633 // virtual member functions or fields, it will have the same layout as its base.
2634 // This function returns the least derived such class.
2635 //
2636 // Casting an instance of a base class to such a derived class is technically
2637 // undefined behavior, but it is a relatively common hack for introducing member
2638 // functions on class instances with specific properties (e.g. llvm::Operator)
2639 // that works under most compilers and should not have security implications, so
2640 // we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2641 static const CXXRecordDecl *
2642 LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2643   if (!RD->field_empty())
2644     return RD;
2645 
2646   if (RD->getNumVBases() != 0)
2647     return RD;
2648 
2649   if (RD->getNumBases() != 1)
2650     return RD;
2651 
2652   for (const CXXMethodDecl *MD : RD->methods()) {
2653     if (MD->isVirtual()) {
2654       // Virtual member functions are only ok if they are implicit destructors
2655       // because the implicit destructor will have the same semantics as the
2656       // base class's destructor if no fields are added.
2657       if (isa<CXXDestructorDecl>(MD) && MD->isImplicit())
2658         continue;
2659       return RD;
2660     }
2661   }
2662 
2663   return LeastDerivedClassWithSameLayout(
2664       RD->bases_begin()->getType()->getAsCXXRecordDecl());
2665 }
2666 
2667 void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2668                                                    llvm::Value *VTable,
2669                                                    SourceLocation Loc) {
2670   if (SanOpts.has(SanitizerKind::CFIVCall))
2671     EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
2672   else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2673            // Don't insert type test assumes if we are forcing public std
2674            // visibility.
2675            !CGM.HasLTOVisibilityPublicStd(RD)) {
2676     llvm::Metadata *MD =
2677         CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2678     llvm::Value *TypeId =
2679         llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2680 
2681     llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2682     llvm::Value *TypeTest =
2683         Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
2684                            {CastedVTable, TypeId});
2685     Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2686   }
2687 }
2688 
2689 void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2690                                                 llvm::Value *VTable,
2691                                                 CFITypeCheckKind TCK,
2692                                                 SourceLocation Loc) {
2693   if (!SanOpts.has(SanitizerKind::CFICastStrict))
2694     RD = LeastDerivedClassWithSameLayout(RD);
2695 
2696   EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2697 }
2698 
2699 void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
2700                                                 llvm::Value *Derived,
2701                                                 bool MayBeNull,
2702                                                 CFITypeCheckKind TCK,
2703                                                 SourceLocation Loc) {
2704   if (!getLangOpts().CPlusPlus)
2705     return;
2706 
2707   auto *ClassTy = T->getAs<RecordType>();
2708   if (!ClassTy)
2709     return;
2710 
2711   const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
2712 
2713   if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2714     return;
2715 
2716   if (!SanOpts.has(SanitizerKind::CFICastStrict))
2717     ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
2718 
2719   llvm::BasicBlock *ContBlock = nullptr;
2720 
2721   if (MayBeNull) {
2722     llvm::Value *DerivedNotNull =
2723         Builder.CreateIsNotNull(Derived, "cast.nonnull");
2724 
2725     llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
2726     ContBlock = createBasicBlock("cast.cont");
2727 
2728     Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
2729 
2730     EmitBlock(CheckBlock);
2731   }
2732 
2733   llvm::Value *VTable;
2734   std::tie(VTable, ClassDecl) = CGM.getCXXABI().LoadVTablePtr(
2735       *this, Address(Derived, getPointerAlign()), ClassDecl);
2736 
2737   EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
2738 
2739   if (MayBeNull) {
2740     Builder.CreateBr(ContBlock);
2741     EmitBlock(ContBlock);
2742   }
2743 }
2744 
2745 void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2746                                          llvm::Value *VTable,
2747                                          CFITypeCheckKind TCK,
2748                                          SourceLocation Loc) {
2749   if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2750       !CGM.HasHiddenLTOVisibility(RD))
2751     return;
2752 
2753   SanitizerMask M;
2754   llvm::SanitizerStatKind SSK;
2755   switch (TCK) {
2756   case CFITCK_VCall:
2757     M = SanitizerKind::CFIVCall;
2758     SSK = llvm::SanStat_CFI_VCall;
2759     break;
2760   case CFITCK_NVCall:
2761     M = SanitizerKind::CFINVCall;
2762     SSK = llvm::SanStat_CFI_NVCall;
2763     break;
2764   case CFITCK_DerivedCast:
2765     M = SanitizerKind::CFIDerivedCast;
2766     SSK = llvm::SanStat_CFI_DerivedCast;
2767     break;
2768   case CFITCK_UnrelatedCast:
2769     M = SanitizerKind::CFIUnrelatedCast;
2770     SSK = llvm::SanStat_CFI_UnrelatedCast;
2771     break;
2772   case CFITCK_ICall:
2773   case CFITCK_NVMFCall:
2774   case CFITCK_VMFCall:
2775     llvm_unreachable("unexpected sanitizer kind");
2776   }
2777 
2778   std::string TypeName = RD->getQualifiedNameAsString();
2779   if (getContext().getNoSanitizeList().containsType(M, TypeName))
2780     return;
2781 
2782   SanitizerScope SanScope(this);
2783   EmitSanitizerStatReport(SSK);
2784 
2785   llvm::Metadata *MD =
2786       CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2787   llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
2788 
2789   llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2790   llvm::Value *TypeTest = Builder.CreateCall(
2791       CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, TypeId});
2792 
2793   llvm::Constant *StaticData[] = {
2794       llvm::ConstantInt::get(Int8Ty, TCK),
2795       EmitCheckSourceLocation(Loc),
2796       EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
2797   };
2798 
2799   auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2800   if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
2801     EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, CastedVTable, StaticData);
2802     return;
2803   }
2804 
2805   if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
2806     EmitTrapCheck(TypeTest, SanitizerHandler::CFICheckFail);
2807     return;
2808   }
2809 
2810   llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2811       CGM.getLLVMContext(),
2812       llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
2813   llvm::Value *ValidVtable = Builder.CreateCall(
2814       CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables});
2815   EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,
2816             StaticData, {CastedVTable, ValidVtable});
2817 }
2818 
2819 bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2820   if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2821       !CGM.HasHiddenLTOVisibility(RD))
2822     return false;
2823 
2824   if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2825     return true;
2826 
2827   if (!SanOpts.has(SanitizerKind::CFIVCall) ||
2828       !CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall))
2829     return false;
2830 
2831   std::string TypeName = RD->getQualifiedNameAsString();
2832   return !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
2833                                                         TypeName);
2834 }
2835 
2836 llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2837     const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) {
2838   SanitizerScope SanScope(this);
2839 
2840   EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
2841 
2842   llvm::Metadata *MD =
2843       CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2844   llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2845 
2846   llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2847   llvm::Value *CheckedLoad = Builder.CreateCall(
2848       CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2849       {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset),
2850        TypeId});
2851   llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
2852 
2853   std::string TypeName = RD->getQualifiedNameAsString();
2854   if (SanOpts.has(SanitizerKind::CFIVCall) &&
2855       !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
2856                                                      TypeName)) {
2857     EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall),
2858               SanitizerHandler::CFICheckFail, {}, {});
2859   }
2860 
2861   return Builder.CreateBitCast(
2862       Builder.CreateExtractValue(CheckedLoad, 0),
2863       cast<llvm::PointerType>(VTable->getType())->getElementType());
2864 }
2865 
2866 void CodeGenFunction::EmitForwardingCallToLambda(
2867                                       const CXXMethodDecl *callOperator,
2868                                       CallArgList &callArgs) {
2869   // Get the address of the call operator.
2870   const CGFunctionInfo &calleeFnInfo =
2871     CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);
2872   llvm::Constant *calleePtr =
2873     CGM.GetAddrOfFunction(GlobalDecl(callOperator),
2874                           CGM.getTypes().GetFunctionType(calleeFnInfo));
2875 
2876   // Prepare the return slot.
2877   const FunctionProtoType *FPT =
2878     callOperator->getType()->castAs<FunctionProtoType>();
2879   QualType resultType = FPT->getReturnType();
2880   ReturnValueSlot returnSlot;
2881   if (!resultType->isVoidType() &&
2882       calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect &&
2883       !hasScalarEvaluationKind(calleeFnInfo.getReturnType()))
2884     returnSlot =
2885         ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2886                         /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2887 
2888   // We don't need to separately arrange the call arguments because
2889   // the call can't be variadic anyway --- it's impossible to forward
2890   // variadic arguments.
2891 
2892   // Now emit our call.
2893   auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));
2894   RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs);
2895 
2896   // If necessary, copy the returned value into the slot.
2897   if (!resultType->isVoidType() && returnSlot.isNull()) {
2898     if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {
2899       RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));
2900     }
2901     EmitReturnOfRValue(RV, resultType);
2902   } else
2903     EmitBranchThroughCleanup(ReturnBlock);
2904 }
2905 
2906 void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2907   const BlockDecl *BD = BlockInfo->getBlockDecl();
2908   const VarDecl *variable = BD->capture_begin()->getVariable();
2909   const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2910   const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2911 
2912   if (CallOp->isVariadic()) {
2913     // FIXME: Making this work correctly is nasty because it requires either
2914     // cloning the body of the call operator or making the call operator
2915     // forward.
2916     CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");
2917     return;
2918   }
2919 
2920   // Start building arguments for forwarding call
2921   CallArgList CallArgs;
2922 
2923   QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2924   Address ThisPtr = GetAddrOfBlockDecl(variable);
2925   CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
2926 
2927   // Add the rest of the parameters.
2928   for (auto param : BD->parameters())
2929     EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());
2930 
2931   assert(!Lambda->isGenericLambda() &&
2932             "generic lambda interconversion to block not implemented");
2933   EmitForwardingCallToLambda(CallOp, CallArgs);
2934 }
2935 
2936 void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
2937   const CXXRecordDecl *Lambda = MD->getParent();
2938 
2939   // Start building arguments for forwarding call
2940   CallArgList CallArgs;
2941 
2942   QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2943   llvm::Value *ThisPtr = llvm::UndefValue::get(getTypes().ConvertType(ThisType));
2944   CallArgs.add(RValue::get(ThisPtr), ThisType);
2945 
2946   // Add the rest of the parameters.
2947   for (auto Param : MD->parameters())
2948     EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
2949 
2950   const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2951   // For a generic lambda, find the corresponding call operator specialization
2952   // to which the call to the static-invoker shall be forwarded.
2953   if (Lambda->isGenericLambda()) {
2954     assert(MD->isFunctionTemplateSpecialization());
2955     const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
2956     FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
2957     void *InsertPos = nullptr;
2958     FunctionDecl *CorrespondingCallOpSpecialization =
2959         CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
2960     assert(CorrespondingCallOpSpecialization);
2961     CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
2962   }
2963   EmitForwardingCallToLambda(CallOp, CallArgs);
2964 }
2965 
2966 void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2967   if (MD->isVariadic()) {
2968     // FIXME: Making this work correctly is nasty because it requires either
2969     // cloning the body of the call operator or making the call operator forward.
2970     CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2971     return;
2972   }
2973 
2974   EmitLambdaDelegatingInvokeBody(MD);
2975 }
2976