xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGClass.cpp (revision 85868e8a1daeaae7a0e48effb2ea2310ae3b02c6)
1 //===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code dealing with C++ code generation of classes
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGBlocks.h"
14 #include "CGCXXABI.h"
15 #include "CGDebugInfo.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenFunction.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/CXXInheritance.h"
20 #include "clang/AST/DeclTemplate.h"
21 #include "clang/AST/EvaluatedExprVisitor.h"
22 #include "clang/AST/RecordLayout.h"
23 #include "clang/AST/StmtCXX.h"
24 #include "clang/Basic/CodeGenOptions.h"
25 #include "clang/Basic/TargetBuiltins.h"
26 #include "clang/CodeGen/CGFunctionInfo.h"
27 #include "llvm/IR/Intrinsics.h"
28 #include "llvm/IR/Metadata.h"
29 #include "llvm/Transforms/Utils/SanitizerStats.h"
30 
31 using namespace clang;
32 using namespace CodeGen;
33 
34 /// Return the best known alignment for an unknown pointer to a
35 /// particular class.
36 CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
37   if (!RD->isCompleteDefinition())
38     return CharUnits::One(); // Hopefully won't be used anywhere.
39 
40   auto &layout = getContext().getASTRecordLayout(RD);
41 
42   // If the class is final, then we know that the pointer points to an
43   // object of that type and can use the full alignment.
44   if (RD->hasAttr<FinalAttr>()) {
45     return layout.getAlignment();
46 
47   // Otherwise, we have to assume it could be a subclass.
48   } else {
49     return layout.getNonVirtualAlignment();
50   }
51 }
52 
53 /// Return the best known alignment for a pointer to a virtual base,
54 /// given the alignment of a pointer to the derived class.
55 CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
56                                            const CXXRecordDecl *derivedClass,
57                                            const CXXRecordDecl *vbaseClass) {
58   // The basic idea here is that an underaligned derived pointer might
59   // indicate an underaligned base pointer.
60 
61   assert(vbaseClass->isCompleteDefinition());
62   auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
63   CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
64 
65   return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
66                                    expectedVBaseAlign);
67 }
68 
69 CharUnits
70 CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
71                                          const CXXRecordDecl *baseDecl,
72                                          CharUnits expectedTargetAlign) {
73   // If the base is an incomplete type (which is, alas, possible with
74   // member pointers), be pessimistic.
75   if (!baseDecl->isCompleteDefinition())
76     return std::min(actualBaseAlign, expectedTargetAlign);
77 
78   auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
79   CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
80 
81   // If the class is properly aligned, assume the target offset is, too.
82   //
83   // This actually isn't necessarily the right thing to do --- if the
84   // class is a complete object, but it's only properly aligned for a
85   // base subobject, then the alignments of things relative to it are
86   // probably off as well.  (Note that this requires the alignment of
87   // the target to be greater than the NV alignment of the derived
88   // class.)
89   //
90   // However, our approach to this kind of under-alignment can only
91   // ever be best effort; after all, we're never going to propagate
92   // alignments through variables or parameters.  Note, in particular,
93   // that constructing a polymorphic type in an address that's less
94   // than pointer-aligned will generally trap in the constructor,
95   // unless we someday add some sort of attribute to change the
96   // assumed alignment of 'this'.  So our goal here is pretty much
97   // just to allow the user to explicitly say that a pointer is
98   // under-aligned and then safely access its fields and vtables.
99   if (actualBaseAlign >= expectedBaseAlign) {
100     return expectedTargetAlign;
101   }
102 
103   // Otherwise, we might be offset by an arbitrary multiple of the
104   // actual alignment.  The correct adjustment is to take the min of
105   // the two alignments.
106   return std::min(actualBaseAlign, expectedTargetAlign);
107 }
108 
109 Address CodeGenFunction::LoadCXXThisAddress() {
110   assert(CurFuncDecl && "loading 'this' without a func declaration?");
111   assert(isa<CXXMethodDecl>(CurFuncDecl));
112 
113   // Lazily compute CXXThisAlignment.
114   if (CXXThisAlignment.isZero()) {
115     // Just use the best known alignment for the parent.
116     // TODO: if we're currently emitting a complete-object ctor/dtor,
117     // we can always use the complete-object alignment.
118     auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent();
119     CXXThisAlignment = CGM.getClassPointerAlignment(RD);
120   }
121 
122   return Address(LoadCXXThis(), CXXThisAlignment);
123 }
124 
125 /// Emit the address of a field using a member data pointer.
126 ///
127 /// \param E Only used for emergency diagnostics
128 Address
129 CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
130                                                  llvm::Value *memberPtr,
131                                       const MemberPointerType *memberPtrType,
132                                                  LValueBaseInfo *BaseInfo,
133                                                  TBAAAccessInfo *TBAAInfo) {
134   // Ask the ABI to compute the actual address.
135   llvm::Value *ptr =
136     CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
137                                                  memberPtr, memberPtrType);
138 
139   QualType memberType = memberPtrType->getPointeeType();
140   CharUnits memberAlign = getNaturalTypeAlignment(memberType, BaseInfo,
141                                                   TBAAInfo);
142   memberAlign =
143     CGM.getDynamicOffsetAlignment(base.getAlignment(),
144                             memberPtrType->getClass()->getAsCXXRecordDecl(),
145                                   memberAlign);
146   return Address(ptr, memberAlign);
147 }
148 
149 CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
150     const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
151     CastExpr::path_const_iterator End) {
152   CharUnits Offset = CharUnits::Zero();
153 
154   const ASTContext &Context = getContext();
155   const CXXRecordDecl *RD = DerivedClass;
156 
157   for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
158     const CXXBaseSpecifier *Base = *I;
159     assert(!Base->isVirtual() && "Should not see virtual bases here!");
160 
161     // Get the layout.
162     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
163 
164     const auto *BaseDecl =
165         cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
166 
167     // Add the offset.
168     Offset += Layout.getBaseClassOffset(BaseDecl);
169 
170     RD = BaseDecl;
171   }
172 
173   return Offset;
174 }
175 
176 llvm::Constant *
177 CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
178                                    CastExpr::path_const_iterator PathBegin,
179                                    CastExpr::path_const_iterator PathEnd) {
180   assert(PathBegin != PathEnd && "Base path should not be empty!");
181 
182   CharUnits Offset =
183       computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
184   if (Offset.isZero())
185     return nullptr;
186 
187   llvm::Type *PtrDiffTy =
188   Types.ConvertType(getContext().getPointerDiffType());
189 
190   return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
191 }
192 
193 /// Gets the address of a direct base class within a complete object.
194 /// This should only be used for (1) non-virtual bases or (2) virtual bases
195 /// when the type is known to be complete (e.g. in complete destructors).
196 ///
197 /// The object pointed to by 'This' is assumed to be non-null.
198 Address
199 CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
200                                                    const CXXRecordDecl *Derived,
201                                                    const CXXRecordDecl *Base,
202                                                    bool BaseIsVirtual) {
203   // 'this' must be a pointer (in some address space) to Derived.
204   assert(This.getElementType() == ConvertType(Derived));
205 
206   // Compute the offset of the virtual base.
207   CharUnits Offset;
208   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
209   if (BaseIsVirtual)
210     Offset = Layout.getVBaseClassOffset(Base);
211   else
212     Offset = Layout.getBaseClassOffset(Base);
213 
214   // Shift and cast down to the base type.
215   // TODO: for complete types, this should be possible with a GEP.
216   Address V = This;
217   if (!Offset.isZero()) {
218     V = Builder.CreateElementBitCast(V, Int8Ty);
219     V = Builder.CreateConstInBoundsByteGEP(V, Offset);
220   }
221   V = Builder.CreateElementBitCast(V, ConvertType(Base));
222 
223   return V;
224 }
225 
226 static Address
227 ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
228                                 CharUnits nonVirtualOffset,
229                                 llvm::Value *virtualOffset,
230                                 const CXXRecordDecl *derivedClass,
231                                 const CXXRecordDecl *nearestVBase) {
232   // Assert that we have something to do.
233   assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
234 
235   // Compute the offset from the static and dynamic components.
236   llvm::Value *baseOffset;
237   if (!nonVirtualOffset.isZero()) {
238     baseOffset = llvm::ConstantInt::get(CGF.PtrDiffTy,
239                                         nonVirtualOffset.getQuantity());
240     if (virtualOffset) {
241       baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);
242     }
243   } else {
244     baseOffset = virtualOffset;
245   }
246 
247   // Apply the base offset.
248   llvm::Value *ptr = addr.getPointer();
249   unsigned AddrSpace = ptr->getType()->getPointerAddressSpace();
250   ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8Ty->getPointerTo(AddrSpace));
251   ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr");
252 
253   // If we have a virtual component, the alignment of the result will
254   // be relative only to the known alignment of that vbase.
255   CharUnits alignment;
256   if (virtualOffset) {
257     assert(nearestVBase && "virtual offset without vbase?");
258     alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
259                                           derivedClass, nearestVBase);
260   } else {
261     alignment = addr.getAlignment();
262   }
263   alignment = alignment.alignmentAtOffset(nonVirtualOffset);
264 
265   return Address(ptr, alignment);
266 }
267 
268 Address CodeGenFunction::GetAddressOfBaseClass(
269     Address Value, const CXXRecordDecl *Derived,
270     CastExpr::path_const_iterator PathBegin,
271     CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
272     SourceLocation Loc) {
273   assert(PathBegin != PathEnd && "Base path should not be empty!");
274 
275   CastExpr::path_const_iterator Start = PathBegin;
276   const CXXRecordDecl *VBase = nullptr;
277 
278   // Sema has done some convenient canonicalization here: if the
279   // access path involved any virtual steps, the conversion path will
280   // *start* with a step down to the correct virtual base subobject,
281   // and hence will not require any further steps.
282   if ((*Start)->isVirtual()) {
283     VBase = cast<CXXRecordDecl>(
284         (*Start)->getType()->castAs<RecordType>()->getDecl());
285     ++Start;
286   }
287 
288   // Compute the static offset of the ultimate destination within its
289   // allocating subobject (the virtual base, if there is one, or else
290   // the "complete" object that we see).
291   CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
292       VBase ? VBase : Derived, Start, PathEnd);
293 
294   // If there's a virtual step, we can sometimes "devirtualize" it.
295   // For now, that's limited to when the derived type is final.
296   // TODO: "devirtualize" this for accesses to known-complete objects.
297   if (VBase && Derived->hasAttr<FinalAttr>()) {
298     const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
299     CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
300     NonVirtualOffset += vBaseOffset;
301     VBase = nullptr; // we no longer have a virtual step
302   }
303 
304   // Get the base pointer type.
305   llvm::Type *BasePtrTy =
306       ConvertType((PathEnd[-1])->getType())
307           ->getPointerTo(Value.getType()->getPointerAddressSpace());
308 
309   QualType DerivedTy = getContext().getRecordType(Derived);
310   CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
311 
312   // If the static offset is zero and we don't have a virtual step,
313   // just do a bitcast; null checks are unnecessary.
314   if (NonVirtualOffset.isZero() && !VBase) {
315     if (sanitizePerformTypeCheck()) {
316       SanitizerSet SkippedChecks;
317       SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
318       EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
319                     DerivedTy, DerivedAlign, SkippedChecks);
320     }
321     return Builder.CreateBitCast(Value, BasePtrTy);
322   }
323 
324   llvm::BasicBlock *origBB = nullptr;
325   llvm::BasicBlock *endBB = nullptr;
326 
327   // Skip over the offset (and the vtable load) if we're supposed to
328   // null-check the pointer.
329   if (NullCheckValue) {
330     origBB = Builder.GetInsertBlock();
331     llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
332     endBB = createBasicBlock("cast.end");
333 
334     llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
335     Builder.CreateCondBr(isNull, endBB, notNullBB);
336     EmitBlock(notNullBB);
337   }
338 
339   if (sanitizePerformTypeCheck()) {
340     SanitizerSet SkippedChecks;
341     SkippedChecks.set(SanitizerKind::Null, true);
342     EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
343                   Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
344   }
345 
346   // Compute the virtual offset.
347   llvm::Value *VirtualOffset = nullptr;
348   if (VBase) {
349     VirtualOffset =
350       CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
351   }
352 
353   // Apply both offsets.
354   Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
355                                           VirtualOffset, Derived, VBase);
356 
357   // Cast to the destination type.
358   Value = Builder.CreateBitCast(Value, BasePtrTy);
359 
360   // Build a phi if we needed a null check.
361   if (NullCheckValue) {
362     llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
363     Builder.CreateBr(endBB);
364     EmitBlock(endBB);
365 
366     llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
367     PHI->addIncoming(Value.getPointer(), notNullBB);
368     PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
369     Value = Address(PHI, Value.getAlignment());
370   }
371 
372   return Value;
373 }
374 
375 Address
376 CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
377                                           const CXXRecordDecl *Derived,
378                                         CastExpr::path_const_iterator PathBegin,
379                                           CastExpr::path_const_iterator PathEnd,
380                                           bool NullCheckValue) {
381   assert(PathBegin != PathEnd && "Base path should not be empty!");
382 
383   QualType DerivedTy =
384     getContext().getCanonicalType(getContext().getTagDeclType(Derived));
385   unsigned AddrSpace =
386     BaseAddr.getPointer()->getType()->getPointerAddressSpace();
387   llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo(AddrSpace);
388 
389   llvm::Value *NonVirtualOffset =
390     CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
391 
392   if (!NonVirtualOffset) {
393     // No offset, we can just cast back.
394     return Builder.CreateBitCast(BaseAddr, DerivedPtrTy);
395   }
396 
397   llvm::BasicBlock *CastNull = nullptr;
398   llvm::BasicBlock *CastNotNull = nullptr;
399   llvm::BasicBlock *CastEnd = nullptr;
400 
401   if (NullCheckValue) {
402     CastNull = createBasicBlock("cast.null");
403     CastNotNull = createBasicBlock("cast.notnull");
404     CastEnd = createBasicBlock("cast.end");
405 
406     llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
407     Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
408     EmitBlock(CastNotNull);
409   }
410 
411   // Apply the offset.
412   llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
413   Value = Builder.CreateInBoundsGEP(Value, Builder.CreateNeg(NonVirtualOffset),
414                                     "sub.ptr");
415 
416   // Just cast.
417   Value = Builder.CreateBitCast(Value, DerivedPtrTy);
418 
419   // Produce a PHI if we had a null-check.
420   if (NullCheckValue) {
421     Builder.CreateBr(CastEnd);
422     EmitBlock(CastNull);
423     Builder.CreateBr(CastEnd);
424     EmitBlock(CastEnd);
425 
426     llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
427     PHI->addIncoming(Value, CastNotNull);
428     PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
429     Value = PHI;
430   }
431 
432   return Address(Value, CGM.getClassPointerAlignment(Derived));
433 }
434 
435 llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
436                                               bool ForVirtualBase,
437                                               bool Delegating) {
438   if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
439     // This constructor/destructor does not need a VTT parameter.
440     return nullptr;
441   }
442 
443   const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
444   const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
445 
446   llvm::Value *VTT;
447 
448   uint64_t SubVTTIndex;
449 
450   if (Delegating) {
451     // If this is a delegating constructor call, just load the VTT.
452     return LoadCXXVTT();
453   } else if (RD == Base) {
454     // If the record matches the base, this is the complete ctor/dtor
455     // variant calling the base variant in a class with virtual bases.
456     assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
457            "doing no-op VTT offset in base dtor/ctor?");
458     assert(!ForVirtualBase && "Can't have same class as virtual base!");
459     SubVTTIndex = 0;
460   } else {
461     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
462     CharUnits BaseOffset = ForVirtualBase ?
463       Layout.getVBaseClassOffset(Base) :
464       Layout.getBaseClassOffset(Base);
465 
466     SubVTTIndex =
467       CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
468     assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
469   }
470 
471   if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
472     // A VTT parameter was passed to the constructor, use it.
473     VTT = LoadCXXVTT();
474     VTT = Builder.CreateConstInBoundsGEP1_64(VTT, SubVTTIndex);
475   } else {
476     // We're the complete constructor, so get the VTT by name.
477     VTT = CGM.getVTables().GetAddrOfVTT(RD);
478     VTT = Builder.CreateConstInBoundsGEP2_64(VTT, 0, SubVTTIndex);
479   }
480 
481   return VTT;
482 }
483 
484 namespace {
485   /// Call the destructor for a direct base class.
486   struct CallBaseDtor final : EHScopeStack::Cleanup {
487     const CXXRecordDecl *BaseClass;
488     bool BaseIsVirtual;
489     CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
490       : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
491 
492     void Emit(CodeGenFunction &CGF, Flags flags) override {
493       const CXXRecordDecl *DerivedClass =
494         cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
495 
496       const CXXDestructorDecl *D = BaseClass->getDestructor();
497       // We are already inside a destructor, so presumably the object being
498       // destroyed should have the expected type.
499       QualType ThisTy = D->getThisObjectType();
500       Address Addr =
501         CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
502                                                   DerivedClass, BaseClass,
503                                                   BaseIsVirtual);
504       CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
505                                 /*Delegating=*/false, Addr, ThisTy);
506     }
507   };
508 
509   /// A visitor which checks whether an initializer uses 'this' in a
510   /// way which requires the vtable to be properly set.
511   struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
512     typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
513 
514     bool UsesThis;
515 
516     DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
517 
518     // Black-list all explicit and implicit references to 'this'.
519     //
520     // Do we need to worry about external references to 'this' derived
521     // from arbitrary code?  If so, then anything which runs arbitrary
522     // external code might potentially access the vtable.
523     void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
524   };
525 } // end anonymous namespace
526 
527 static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
528   DynamicThisUseChecker Checker(C);
529   Checker.Visit(Init);
530   return Checker.UsesThis;
531 }
532 
533 static void EmitBaseInitializer(CodeGenFunction &CGF,
534                                 const CXXRecordDecl *ClassDecl,
535                                 CXXCtorInitializer *BaseInit) {
536   assert(BaseInit->isBaseInitializer() &&
537          "Must have base initializer!");
538 
539   Address ThisPtr = CGF.LoadCXXThisAddress();
540 
541   const Type *BaseType = BaseInit->getBaseClass();
542   const auto *BaseClassDecl =
543       cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
544 
545   bool isBaseVirtual = BaseInit->isBaseVirtual();
546 
547   // If the initializer for the base (other than the constructor
548   // itself) accesses 'this' in any way, we need to initialize the
549   // vtables.
550   if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
551     CGF.InitializeVTablePointers(ClassDecl);
552 
553   // We can pretend to be a complete class because it only matters for
554   // virtual bases, and we only do virtual bases for complete ctors.
555   Address V =
556     CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
557                                               BaseClassDecl,
558                                               isBaseVirtual);
559   AggValueSlot AggSlot =
560       AggValueSlot::forAddr(
561           V, Qualifiers(),
562           AggValueSlot::IsDestructed,
563           AggValueSlot::DoesNotNeedGCBarriers,
564           AggValueSlot::IsNotAliased,
565           CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));
566 
567   CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
568 
569   if (CGF.CGM.getLangOpts().Exceptions &&
570       !BaseClassDecl->hasTrivialDestructor())
571     CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
572                                           isBaseVirtual);
573 }
574 
575 static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
576   auto *CD = dyn_cast<CXXConstructorDecl>(D);
577   if (!(CD && CD->isCopyOrMoveConstructor()) &&
578       !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator())
579     return false;
580 
581   // We can emit a memcpy for a trivial copy or move constructor/assignment.
582   if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding())
583     return true;
584 
585   // We *must* emit a memcpy for a defaulted union copy or move op.
586   if (D->getParent()->isUnion() && D->isDefaulted())
587     return true;
588 
589   return false;
590 }
591 
592 static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
593                                                 CXXCtorInitializer *MemberInit,
594                                                 LValue &LHS) {
595   FieldDecl *Field = MemberInit->getAnyMember();
596   if (MemberInit->isIndirectMemberInitializer()) {
597     // If we are initializing an anonymous union field, drill down to the field.
598     IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
599     for (const auto *I : IndirectField->chain())
600       LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
601   } else {
602     LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
603   }
604 }
605 
606 static void EmitMemberInitializer(CodeGenFunction &CGF,
607                                   const CXXRecordDecl *ClassDecl,
608                                   CXXCtorInitializer *MemberInit,
609                                   const CXXConstructorDecl *Constructor,
610                                   FunctionArgList &Args) {
611   ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
612   assert(MemberInit->isAnyMemberInitializer() &&
613          "Must have member initializer!");
614   assert(MemberInit->getInit() && "Must have initializer!");
615 
616   // non-static data member initializers.
617   FieldDecl *Field = MemberInit->getAnyMember();
618   QualType FieldType = Field->getType();
619 
620   llvm::Value *ThisPtr = CGF.LoadCXXThis();
621   QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
622   LValue LHS;
623 
624   // If a base constructor is being emitted, create an LValue that has the
625   // non-virtual alignment.
626   if (CGF.CurGD.getCtorType() == Ctor_Base)
627     LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);
628   else
629     LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
630 
631   EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
632 
633   // Special case: if we are in a copy or move constructor, and we are copying
634   // an array of PODs or classes with trivial copy constructors, ignore the
635   // AST and perform the copy we know is equivalent.
636   // FIXME: This is hacky at best... if we had a bit more explicit information
637   // in the AST, we could generalize it more easily.
638   const ConstantArrayType *Array
639     = CGF.getContext().getAsConstantArrayType(FieldType);
640   if (Array && Constructor->isDefaulted() &&
641       Constructor->isCopyOrMoveConstructor()) {
642     QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
643     CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
644     if (BaseElementTy.isPODType(CGF.getContext()) ||
645         (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) {
646       unsigned SrcArgIndex =
647           CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
648       llvm::Value *SrcPtr
649         = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
650       LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
651       LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
652 
653       // Copy the aggregate.
654       CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),
655                             LHS.isVolatileQualified());
656       // Ensure that we destroy the objects if an exception is thrown later in
657       // the constructor.
658       QualType::DestructionKind dtorKind = FieldType.isDestructedType();
659       if (CGF.needsEHCleanup(dtorKind))
660         CGF.pushEHDestroy(dtorKind, LHS.getAddress(), FieldType);
661       return;
662     }
663   }
664 
665   CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());
666 }
667 
668 void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
669                                               Expr *Init) {
670   QualType FieldType = Field->getType();
671   switch (getEvaluationKind(FieldType)) {
672   case TEK_Scalar:
673     if (LHS.isSimple()) {
674       EmitExprAsInit(Init, Field, LHS, false);
675     } else {
676       RValue RHS = RValue::get(EmitScalarExpr(Init));
677       EmitStoreThroughLValue(RHS, LHS);
678     }
679     break;
680   case TEK_Complex:
681     EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
682     break;
683   case TEK_Aggregate: {
684     AggValueSlot Slot =
685         AggValueSlot::forLValue(
686             LHS,
687             AggValueSlot::IsDestructed,
688             AggValueSlot::DoesNotNeedGCBarriers,
689             AggValueSlot::IsNotAliased,
690             getOverlapForFieldInit(Field),
691             AggValueSlot::IsNotZeroed,
692             // Checks are made by the code that calls constructor.
693             AggValueSlot::IsSanitizerChecked);
694     EmitAggExpr(Init, Slot);
695     break;
696   }
697   }
698 
699   // Ensure that we destroy this object if an exception is thrown
700   // later in the constructor.
701   QualType::DestructionKind dtorKind = FieldType.isDestructedType();
702   if (needsEHCleanup(dtorKind))
703     pushEHDestroy(dtorKind, LHS.getAddress(), FieldType);
704 }
705 
706 /// Checks whether the given constructor is a valid subject for the
707 /// complete-to-base constructor delegation optimization, i.e.
708 /// emitting the complete constructor as a simple call to the base
709 /// constructor.
710 bool CodeGenFunction::IsConstructorDelegationValid(
711     const CXXConstructorDecl *Ctor) {
712 
713   // Currently we disable the optimization for classes with virtual
714   // bases because (1) the addresses of parameter variables need to be
715   // consistent across all initializers but (2) the delegate function
716   // call necessarily creates a second copy of the parameter variable.
717   //
718   // The limiting example (purely theoretical AFAIK):
719   //   struct A { A(int &c) { c++; } };
720   //   struct B : virtual A {
721   //     B(int count) : A(count) { printf("%d\n", count); }
722   //   };
723   // ...although even this example could in principle be emitted as a
724   // delegation since the address of the parameter doesn't escape.
725   if (Ctor->getParent()->getNumVBases()) {
726     // TODO: white-list trivial vbase initializers.  This case wouldn't
727     // be subject to the restrictions below.
728 
729     // TODO: white-list cases where:
730     //  - there are no non-reference parameters to the constructor
731     //  - the initializers don't access any non-reference parameters
732     //  - the initializers don't take the address of non-reference
733     //    parameters
734     //  - etc.
735     // If we ever add any of the above cases, remember that:
736     //  - function-try-blocks will always blacklist this optimization
737     //  - we need to perform the constructor prologue and cleanup in
738     //    EmitConstructorBody.
739 
740     return false;
741   }
742 
743   // We also disable the optimization for variadic functions because
744   // it's impossible to "re-pass" varargs.
745   if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
746     return false;
747 
748   // FIXME: Decide if we can do a delegation of a delegating constructor.
749   if (Ctor->isDelegatingConstructor())
750     return false;
751 
752   return true;
753 }
754 
755 // Emit code in ctor (Prologue==true) or dtor (Prologue==false)
756 // to poison the extra field paddings inserted under
757 // -fsanitize-address-field-padding=1|2.
758 void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
759   ASTContext &Context = getContext();
760   const CXXRecordDecl *ClassDecl =
761       Prologue ? cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()
762                : cast<CXXDestructorDecl>(CurGD.getDecl())->getParent();
763   if (!ClassDecl->mayInsertExtraPadding()) return;
764 
765   struct SizeAndOffset {
766     uint64_t Size;
767     uint64_t Offset;
768   };
769 
770   unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
771   const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
772 
773   // Populate sizes and offsets of fields.
774   SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
775   for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)
776     SSV[i].Offset =
777         Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();
778 
779   size_t NumFields = 0;
780   for (const auto *Field : ClassDecl->fields()) {
781     const FieldDecl *D = Field;
782     std::pair<CharUnits, CharUnits> FieldInfo =
783         Context.getTypeInfoInChars(D->getType());
784     CharUnits FieldSize = FieldInfo.first;
785     assert(NumFields < SSV.size());
786     SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();
787     NumFields++;
788   }
789   assert(NumFields == SSV.size());
790   if (SSV.size() <= 1) return;
791 
792   // We will insert calls to __asan_* run-time functions.
793   // LLVM AddressSanitizer pass may decide to inline them later.
794   llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
795   llvm::FunctionType *FTy =
796       llvm::FunctionType::get(CGM.VoidTy, Args, false);
797   llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
798       FTy, Prologue ? "__asan_poison_intra_object_redzone"
799                     : "__asan_unpoison_intra_object_redzone");
800 
801   llvm::Value *ThisPtr = LoadCXXThis();
802   ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);
803   uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
804   // For each field check if it has sufficient padding,
805   // if so (un)poison it with a call.
806   for (size_t i = 0; i < SSV.size(); i++) {
807     uint64_t AsanAlignment = 8;
808     uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;
809     uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
810     uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
811     if (PoisonSize < AsanAlignment || !SSV[i].Size ||
812         (NextField % AsanAlignment) != 0)
813       continue;
814     Builder.CreateCall(
815         F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
816             Builder.getIntN(PtrSize, PoisonSize)});
817   }
818 }
819 
820 /// EmitConstructorBody - Emits the body of the current constructor.
821 void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
822   EmitAsanPrologueOrEpilogue(true);
823   const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
824   CXXCtorType CtorType = CurGD.getCtorType();
825 
826   assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
827           CtorType == Ctor_Complete) &&
828          "can only generate complete ctor for this ABI");
829 
830   // Before we go any further, try the complete->base constructor
831   // delegation optimization.
832   if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&
833       CGM.getTarget().getCXXABI().hasConstructorVariants()) {
834     EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());
835     return;
836   }
837 
838   const FunctionDecl *Definition = nullptr;
839   Stmt *Body = Ctor->getBody(Definition);
840   assert(Definition == Ctor && "emitting wrong constructor body");
841 
842   // Enter the function-try-block before the constructor prologue if
843   // applicable.
844   bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
845   if (IsTryBody)
846     EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
847 
848   incrementProfileCounter(Body);
849 
850   RunCleanupsScope RunCleanups(*this);
851 
852   // TODO: in restricted cases, we can emit the vbase initializers of
853   // a complete ctor and then delegate to the base ctor.
854 
855   // Emit the constructor prologue, i.e. the base and member
856   // initializers.
857   EmitCtorPrologue(Ctor, CtorType, Args);
858 
859   // Emit the body of the statement.
860   if (IsTryBody)
861     EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
862   else if (Body)
863     EmitStmt(Body);
864 
865   // Emit any cleanup blocks associated with the member or base
866   // initializers, which includes (along the exceptional path) the
867   // destructors for those members and bases that were fully
868   // constructed.
869   RunCleanups.ForceCleanup();
870 
871   if (IsTryBody)
872     ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
873 }
874 
875 namespace {
876   /// RAII object to indicate that codegen is copying the value representation
877   /// instead of the object representation. Useful when copying a struct or
878   /// class which has uninitialized members and we're only performing
879   /// lvalue-to-rvalue conversion on the object but not its members.
880   class CopyingValueRepresentation {
881   public:
882     explicit CopyingValueRepresentation(CodeGenFunction &CGF)
883         : CGF(CGF), OldSanOpts(CGF.SanOpts) {
884       CGF.SanOpts.set(SanitizerKind::Bool, false);
885       CGF.SanOpts.set(SanitizerKind::Enum, false);
886     }
887     ~CopyingValueRepresentation() {
888       CGF.SanOpts = OldSanOpts;
889     }
890   private:
891     CodeGenFunction &CGF;
892     SanitizerSet OldSanOpts;
893   };
894 } // end anonymous namespace
895 
896 namespace {
897   class FieldMemcpyizer {
898   public:
899     FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
900                     const VarDecl *SrcRec)
901       : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
902         RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
903         FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
904         LastFieldOffset(0), LastAddedFieldIndex(0) {}
905 
906     bool isMemcpyableField(FieldDecl *F) const {
907       // Never memcpy fields when we are adding poisoned paddings.
908       if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
909         return false;
910       Qualifiers Qual = F->getType().getQualifiers();
911       if (Qual.hasVolatile() || Qual.hasObjCLifetime())
912         return false;
913       return true;
914     }
915 
916     void addMemcpyableField(FieldDecl *F) {
917       if (!FirstField)
918         addInitialField(F);
919       else
920         addNextField(F);
921     }
922 
923     CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
924       ASTContext &Ctx = CGF.getContext();
925       unsigned LastFieldSize =
926           LastField->isBitField()
927               ? LastField->getBitWidthValue(Ctx)
928               : Ctx.toBits(
929                     Ctx.getTypeInfoDataSizeInChars(LastField->getType()).first);
930       uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
931                                 FirstByteOffset + Ctx.getCharWidth() - 1;
932       CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);
933       return MemcpySize;
934     }
935 
936     void emitMemcpy() {
937       // Give the subclass a chance to bail out if it feels the memcpy isn't
938       // worth it (e.g. Hasn't aggregated enough data).
939       if (!FirstField) {
940         return;
941       }
942 
943       uint64_t FirstByteOffset;
944       if (FirstField->isBitField()) {
945         const CGRecordLayout &RL =
946           CGF.getTypes().getCGRecordLayout(FirstField->getParent());
947         const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
948         // FirstFieldOffset is not appropriate for bitfields,
949         // we need to use the storage offset instead.
950         FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
951       } else {
952         FirstByteOffset = FirstFieldOffset;
953       }
954 
955       CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
956       QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
957       Address ThisPtr = CGF.LoadCXXThisAddress();
958       LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
959       LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
960       llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
961       LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
962       LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
963 
964       emitMemcpyIR(Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(),
965                    Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(),
966                    MemcpySize);
967       reset();
968     }
969 
970     void reset() {
971       FirstField = nullptr;
972     }
973 
974   protected:
975     CodeGenFunction &CGF;
976     const CXXRecordDecl *ClassDecl;
977 
978   private:
979     void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
980       llvm::PointerType *DPT = DestPtr.getType();
981       llvm::Type *DBP =
982         llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace());
983       DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP);
984 
985       llvm::PointerType *SPT = SrcPtr.getType();
986       llvm::Type *SBP =
987         llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace());
988       SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP);
989 
990       CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
991     }
992 
993     void addInitialField(FieldDecl *F) {
994       FirstField = F;
995       LastField = F;
996       FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());
997       LastFieldOffset = FirstFieldOffset;
998       LastAddedFieldIndex = F->getFieldIndex();
999     }
1000 
1001     void addNextField(FieldDecl *F) {
1002       // For the most part, the following invariant will hold:
1003       //   F->getFieldIndex() == LastAddedFieldIndex + 1
1004       // The one exception is that Sema won't add a copy-initializer for an
1005       // unnamed bitfield, which will show up here as a gap in the sequence.
1006       assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1007              "Cannot aggregate fields out of order.");
1008       LastAddedFieldIndex = F->getFieldIndex();
1009 
1010       // The 'first' and 'last' fields are chosen by offset, rather than field
1011       // index. This allows the code to support bitfields, as well as regular
1012       // fields.
1013       uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1014       if (FOffset < FirstFieldOffset) {
1015         FirstField = F;
1016         FirstFieldOffset = FOffset;
1017       } else if (FOffset >= LastFieldOffset) {
1018         LastField = F;
1019         LastFieldOffset = FOffset;
1020       }
1021     }
1022 
1023     const VarDecl *SrcRec;
1024     const ASTRecordLayout &RecLayout;
1025     FieldDecl *FirstField;
1026     FieldDecl *LastField;
1027     uint64_t FirstFieldOffset, LastFieldOffset;
1028     unsigned LastAddedFieldIndex;
1029   };
1030 
1031   class ConstructorMemcpyizer : public FieldMemcpyizer {
1032   private:
1033     /// Get source argument for copy constructor. Returns null if not a copy
1034     /// constructor.
1035     static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1036                                                const CXXConstructorDecl *CD,
1037                                                FunctionArgList &Args) {
1038       if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())
1039         return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1040       return nullptr;
1041     }
1042 
1043     // Returns true if a CXXCtorInitializer represents a member initialization
1044     // that can be rolled into a memcpy.
1045     bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1046       if (!MemcpyableCtor)
1047         return false;
1048       FieldDecl *Field = MemberInit->getMember();
1049       assert(Field && "No field for member init.");
1050       QualType FieldType = Field->getType();
1051       CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
1052 
1053       // Bail out on non-memcpyable, not-trivially-copyable members.
1054       if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) &&
1055           !(FieldType.isTriviallyCopyableType(CGF.getContext()) ||
1056             FieldType->isReferenceType()))
1057         return false;
1058 
1059       // Bail out on volatile fields.
1060       if (!isMemcpyableField(Field))
1061         return false;
1062 
1063       // Otherwise we're good.
1064       return true;
1065     }
1066 
1067   public:
1068     ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1069                           FunctionArgList &Args)
1070       : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1071         ConstructorDecl(CD),
1072         MemcpyableCtor(CD->isDefaulted() &&
1073                        CD->isCopyOrMoveConstructor() &&
1074                        CGF.getLangOpts().getGC() == LangOptions::NonGC),
1075         Args(Args) { }
1076 
1077     void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1078       if (isMemberInitMemcpyable(MemberInit)) {
1079         AggregatedInits.push_back(MemberInit);
1080         addMemcpyableField(MemberInit->getMember());
1081       } else {
1082         emitAggregatedInits();
1083         EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1084                               ConstructorDecl, Args);
1085       }
1086     }
1087 
1088     void emitAggregatedInits() {
1089       if (AggregatedInits.size() <= 1) {
1090         // This memcpy is too small to be worthwhile. Fall back on default
1091         // codegen.
1092         if (!AggregatedInits.empty()) {
1093           CopyingValueRepresentation CVR(CGF);
1094           EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1095                                 AggregatedInits[0], ConstructorDecl, Args);
1096           AggregatedInits.clear();
1097         }
1098         reset();
1099         return;
1100       }
1101 
1102       pushEHDestructors();
1103       emitMemcpy();
1104       AggregatedInits.clear();
1105     }
1106 
1107     void pushEHDestructors() {
1108       Address ThisPtr = CGF.LoadCXXThisAddress();
1109       QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1110       LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
1111 
1112       for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
1113         CXXCtorInitializer *MemberInit = AggregatedInits[i];
1114         QualType FieldType = MemberInit->getAnyMember()->getType();
1115         QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1116         if (!CGF.needsEHCleanup(dtorKind))
1117           continue;
1118         LValue FieldLHS = LHS;
1119         EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
1120         CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(), FieldType);
1121       }
1122     }
1123 
1124     void finish() {
1125       emitAggregatedInits();
1126     }
1127 
1128   private:
1129     const CXXConstructorDecl *ConstructorDecl;
1130     bool MemcpyableCtor;
1131     FunctionArgList &Args;
1132     SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1133   };
1134 
1135   class AssignmentMemcpyizer : public FieldMemcpyizer {
1136   private:
1137     // Returns the memcpyable field copied by the given statement, if one
1138     // exists. Otherwise returns null.
1139     FieldDecl *getMemcpyableField(Stmt *S) {
1140       if (!AssignmentsMemcpyable)
1141         return nullptr;
1142       if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
1143         // Recognise trivial assignments.
1144         if (BO->getOpcode() != BO_Assign)
1145           return nullptr;
1146         MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());
1147         if (!ME)
1148           return nullptr;
1149         FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1150         if (!Field || !isMemcpyableField(Field))
1151           return nullptr;
1152         Stmt *RHS = BO->getRHS();
1153         if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))
1154           RHS = EC->getSubExpr();
1155         if (!RHS)
1156           return nullptr;
1157         if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {
1158           if (ME2->getMemberDecl() == Field)
1159             return Field;
1160         }
1161         return nullptr;
1162       } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
1163         CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1164         if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
1165           return nullptr;
1166         MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
1167         if (!IOA)
1168           return nullptr;
1169         FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());
1170         if (!Field || !isMemcpyableField(Field))
1171           return nullptr;
1172         MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1173         if (!Arg0 || Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl()))
1174           return nullptr;
1175         return Field;
1176       } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
1177         FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());
1178         if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1179           return nullptr;
1180         Expr *DstPtr = CE->getArg(0);
1181         if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))
1182           DstPtr = DC->getSubExpr();
1183         UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);
1184         if (!DUO || DUO->getOpcode() != UO_AddrOf)
1185           return nullptr;
1186         MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());
1187         if (!ME)
1188           return nullptr;
1189         FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1190         if (!Field || !isMemcpyableField(Field))
1191           return nullptr;
1192         Expr *SrcPtr = CE->getArg(1);
1193         if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))
1194           SrcPtr = SC->getSubExpr();
1195         UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);
1196         if (!SUO || SUO->getOpcode() != UO_AddrOf)
1197           return nullptr;
1198         MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());
1199         if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))
1200           return nullptr;
1201         return Field;
1202       }
1203 
1204       return nullptr;
1205     }
1206 
1207     bool AssignmentsMemcpyable;
1208     SmallVector<Stmt*, 16> AggregatedStmts;
1209 
1210   public:
1211     AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1212                          FunctionArgList &Args)
1213       : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1214         AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1215       assert(Args.size() == 2);
1216     }
1217 
1218     void emitAssignment(Stmt *S) {
1219       FieldDecl *F = getMemcpyableField(S);
1220       if (F) {
1221         addMemcpyableField(F);
1222         AggregatedStmts.push_back(S);
1223       } else {
1224         emitAggregatedStmts();
1225         CGF.EmitStmt(S);
1226       }
1227     }
1228 
1229     void emitAggregatedStmts() {
1230       if (AggregatedStmts.size() <= 1) {
1231         if (!AggregatedStmts.empty()) {
1232           CopyingValueRepresentation CVR(CGF);
1233           CGF.EmitStmt(AggregatedStmts[0]);
1234         }
1235         reset();
1236       }
1237 
1238       emitMemcpy();
1239       AggregatedStmts.clear();
1240     }
1241 
1242     void finish() {
1243       emitAggregatedStmts();
1244     }
1245   };
1246 } // end anonymous namespace
1247 
1248 static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1249   const Type *BaseType = BaseInit->getBaseClass();
1250   const auto *BaseClassDecl =
1251       cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
1252   return BaseClassDecl->isDynamicClass();
1253 }
1254 
1255 /// EmitCtorPrologue - This routine generates necessary code to initialize
1256 /// base classes and non-static data members belonging to this constructor.
1257 void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1258                                        CXXCtorType CtorType,
1259                                        FunctionArgList &Args) {
1260   if (CD->isDelegatingConstructor())
1261     return EmitDelegatingCXXConstructorCall(CD, Args);
1262 
1263   const CXXRecordDecl *ClassDecl = CD->getParent();
1264 
1265   CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1266                                           E = CD->init_end();
1267 
1268   // Virtual base initializers first, if any. They aren't needed if:
1269   // - This is a base ctor variant
1270   // - There are no vbases
1271   // - The class is abstract, so a complete object of it cannot be constructed
1272   //
1273   // The check for an abstract class is necessary because sema may not have
1274   // marked virtual base destructors referenced.
1275   bool ConstructVBases = CtorType != Ctor_Base &&
1276                          ClassDecl->getNumVBases() != 0 &&
1277                          !ClassDecl->isAbstract();
1278 
1279   // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1280   // constructor of a class with virtual bases takes an additional parameter to
1281   // conditionally construct the virtual bases. Emit that check here.
1282   llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1283   if (ConstructVBases &&
1284       !CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1285     BaseCtorContinueBB =
1286         CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);
1287     assert(BaseCtorContinueBB);
1288   }
1289 
1290   llvm::Value *const OldThis = CXXThisValue;
1291   for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) {
1292     if (!ConstructVBases)
1293       continue;
1294     if (CGM.getCodeGenOpts().StrictVTablePointers &&
1295         CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1296         isInitializerOfDynamicClass(*B))
1297       CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1298     EmitBaseInitializer(*this, ClassDecl, *B);
1299   }
1300 
1301   if (BaseCtorContinueBB) {
1302     // Complete object handler should continue to the remaining initializers.
1303     Builder.CreateBr(BaseCtorContinueBB);
1304     EmitBlock(BaseCtorContinueBB);
1305   }
1306 
1307   // Then, non-virtual base initializers.
1308   for (; B != E && (*B)->isBaseInitializer(); B++) {
1309     assert(!(*B)->isBaseVirtual());
1310 
1311     if (CGM.getCodeGenOpts().StrictVTablePointers &&
1312         CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1313         isInitializerOfDynamicClass(*B))
1314       CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1315     EmitBaseInitializer(*this, ClassDecl, *B);
1316   }
1317 
1318   CXXThisValue = OldThis;
1319 
1320   InitializeVTablePointers(ClassDecl);
1321 
1322   // And finally, initialize class members.
1323   FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1324   ConstructorMemcpyizer CM(*this, CD, Args);
1325   for (; B != E; B++) {
1326     CXXCtorInitializer *Member = (*B);
1327     assert(!Member->isBaseInitializer());
1328     assert(Member->isAnyMemberInitializer() &&
1329            "Delegating initializer on non-delegating constructor");
1330     CM.addMemberInitializer(Member);
1331   }
1332   CM.finish();
1333 }
1334 
1335 static bool
1336 FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1337 
1338 static bool
1339 HasTrivialDestructorBody(ASTContext &Context,
1340                          const CXXRecordDecl *BaseClassDecl,
1341                          const CXXRecordDecl *MostDerivedClassDecl)
1342 {
1343   // If the destructor is trivial we don't have to check anything else.
1344   if (BaseClassDecl->hasTrivialDestructor())
1345     return true;
1346 
1347   if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1348     return false;
1349 
1350   // Check fields.
1351   for (const auto *Field : BaseClassDecl->fields())
1352     if (!FieldHasTrivialDestructorBody(Context, Field))
1353       return false;
1354 
1355   // Check non-virtual bases.
1356   for (const auto &I : BaseClassDecl->bases()) {
1357     if (I.isVirtual())
1358       continue;
1359 
1360     const CXXRecordDecl *NonVirtualBase =
1361       cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1362     if (!HasTrivialDestructorBody(Context, NonVirtualBase,
1363                                   MostDerivedClassDecl))
1364       return false;
1365   }
1366 
1367   if (BaseClassDecl == MostDerivedClassDecl) {
1368     // Check virtual bases.
1369     for (const auto &I : BaseClassDecl->vbases()) {
1370       const CXXRecordDecl *VirtualBase =
1371         cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1372       if (!HasTrivialDestructorBody(Context, VirtualBase,
1373                                     MostDerivedClassDecl))
1374         return false;
1375     }
1376   }
1377 
1378   return true;
1379 }
1380 
1381 static bool
1382 FieldHasTrivialDestructorBody(ASTContext &Context,
1383                                           const FieldDecl *Field)
1384 {
1385   QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1386 
1387   const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1388   if (!RT)
1389     return true;
1390 
1391   CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
1392 
1393   // The destructor for an implicit anonymous union member is never invoked.
1394   if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
1395     return false;
1396 
1397   return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
1398 }
1399 
1400 /// CanSkipVTablePointerInitialization - Check whether we need to initialize
1401 /// any vtable pointers before calling this destructor.
1402 static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1403                                                const CXXDestructorDecl *Dtor) {
1404   const CXXRecordDecl *ClassDecl = Dtor->getParent();
1405   if (!ClassDecl->isDynamicClass())
1406     return true;
1407 
1408   if (!Dtor->hasTrivialBody())
1409     return false;
1410 
1411   // Check the fields.
1412   for (const auto *Field : ClassDecl->fields())
1413     if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1414       return false;
1415 
1416   return true;
1417 }
1418 
1419 /// EmitDestructorBody - Emits the body of the current destructor.
1420 void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1421   const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
1422   CXXDtorType DtorType = CurGD.getDtorType();
1423 
1424   // For an abstract class, non-base destructors are never used (and can't
1425   // be emitted in general, because vbase dtors may not have been validated
1426   // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1427   // in fact emit references to them from other compilations, so emit them
1428   // as functions containing a trap instruction.
1429   if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {
1430     llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1431     TrapCall->setDoesNotReturn();
1432     TrapCall->setDoesNotThrow();
1433     Builder.CreateUnreachable();
1434     Builder.ClearInsertionPoint();
1435     return;
1436   }
1437 
1438   Stmt *Body = Dtor->getBody();
1439   if (Body)
1440     incrementProfileCounter(Body);
1441 
1442   // The call to operator delete in a deleting destructor happens
1443   // outside of the function-try-block, which means it's always
1444   // possible to delegate the destructor body to the complete
1445   // destructor.  Do so.
1446   if (DtorType == Dtor_Deleting) {
1447     RunCleanupsScope DtorEpilogue(*this);
1448     EnterDtorCleanups(Dtor, Dtor_Deleting);
1449     if (HaveInsertPoint()) {
1450       QualType ThisTy = Dtor->getThisObjectType();
1451       EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
1452                             /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1453     }
1454     return;
1455   }
1456 
1457   // If the body is a function-try-block, enter the try before
1458   // anything else.
1459   bool isTryBody = (Body && isa<CXXTryStmt>(Body));
1460   if (isTryBody)
1461     EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1462   EmitAsanPrologueOrEpilogue(false);
1463 
1464   // Enter the epilogue cleanups.
1465   RunCleanupsScope DtorEpilogue(*this);
1466 
1467   // If this is the complete variant, just invoke the base variant;
1468   // the epilogue will destruct the virtual bases.  But we can't do
1469   // this optimization if the body is a function-try-block, because
1470   // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
1471   // always delegate because we might not have a definition in this TU.
1472   switch (DtorType) {
1473   case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1474   case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1475 
1476   case Dtor_Complete:
1477     assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1478            "can't emit a dtor without a body for non-Microsoft ABIs");
1479 
1480     // Enter the cleanup scopes for virtual bases.
1481     EnterDtorCleanups(Dtor, Dtor_Complete);
1482 
1483     if (!isTryBody) {
1484       QualType ThisTy = Dtor->getThisObjectType();
1485       EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
1486                             /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1487       break;
1488     }
1489 
1490     // Fallthrough: act like we're in the base variant.
1491     LLVM_FALLTHROUGH;
1492 
1493   case Dtor_Base:
1494     assert(Body);
1495 
1496     // Enter the cleanup scopes for fields and non-virtual bases.
1497     EnterDtorCleanups(Dtor, Dtor_Base);
1498 
1499     // Initialize the vtable pointers before entering the body.
1500     if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
1501       // Insert the llvm.launder.invariant.group intrinsic before initializing
1502       // the vptrs to cancel any previous assumptions we might have made.
1503       if (CGM.getCodeGenOpts().StrictVTablePointers &&
1504           CGM.getCodeGenOpts().OptimizationLevel > 0)
1505         CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1506       InitializeVTablePointers(Dtor->getParent());
1507     }
1508 
1509     if (isTryBody)
1510       EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
1511     else if (Body)
1512       EmitStmt(Body);
1513     else {
1514       assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1515       // nothing to do besides what's in the epilogue
1516     }
1517     // -fapple-kext must inline any call to this dtor into
1518     // the caller's body.
1519     if (getLangOpts().AppleKext)
1520       CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1521 
1522     break;
1523   }
1524 
1525   // Jump out through the epilogue cleanups.
1526   DtorEpilogue.ForceCleanup();
1527 
1528   // Exit the try if applicable.
1529   if (isTryBody)
1530     ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1531 }
1532 
1533 void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1534   const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());
1535   const Stmt *RootS = AssignOp->getBody();
1536   assert(isa<CompoundStmt>(RootS) &&
1537          "Body of an implicit assignment operator should be compound stmt.");
1538   const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);
1539 
1540   LexicalScope Scope(*this, RootCS->getSourceRange());
1541 
1542   incrementProfileCounter(RootCS);
1543   AssignmentMemcpyizer AM(*this, AssignOp, Args);
1544   for (auto *I : RootCS->body())
1545     AM.emitAssignment(I);
1546   AM.finish();
1547 }
1548 
1549 namespace {
1550   llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1551                                      const CXXDestructorDecl *DD) {
1552     if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1553       return CGF.EmitScalarExpr(ThisArg);
1554     return CGF.LoadCXXThis();
1555   }
1556 
1557   /// Call the operator delete associated with the current destructor.
1558   struct CallDtorDelete final : EHScopeStack::Cleanup {
1559     CallDtorDelete() {}
1560 
1561     void Emit(CodeGenFunction &CGF, Flags flags) override {
1562       const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1563       const CXXRecordDecl *ClassDecl = Dtor->getParent();
1564       CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1565                          LoadThisForDtorDelete(CGF, Dtor),
1566                          CGF.getContext().getTagDeclType(ClassDecl));
1567     }
1568   };
1569 
1570   void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1571                                      llvm::Value *ShouldDeleteCondition,
1572                                      bool ReturnAfterDelete) {
1573     llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");
1574     llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");
1575     llvm::Value *ShouldCallDelete
1576       = CGF.Builder.CreateIsNull(ShouldDeleteCondition);
1577     CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);
1578 
1579     CGF.EmitBlock(callDeleteBB);
1580     const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1581     const CXXRecordDecl *ClassDecl = Dtor->getParent();
1582     CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1583                        LoadThisForDtorDelete(CGF, Dtor),
1584                        CGF.getContext().getTagDeclType(ClassDecl));
1585     assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1586                ReturnAfterDelete &&
1587            "unexpected value for ReturnAfterDelete");
1588     if (ReturnAfterDelete)
1589       CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
1590     else
1591       CGF.Builder.CreateBr(continueBB);
1592 
1593     CGF.EmitBlock(continueBB);
1594   }
1595 
1596   struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1597     llvm::Value *ShouldDeleteCondition;
1598 
1599   public:
1600     CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1601         : ShouldDeleteCondition(ShouldDeleteCondition) {
1602       assert(ShouldDeleteCondition != nullptr);
1603     }
1604 
1605     void Emit(CodeGenFunction &CGF, Flags flags) override {
1606       EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1607                                     /*ReturnAfterDelete*/false);
1608     }
1609   };
1610 
1611   class DestroyField  final : public EHScopeStack::Cleanup {
1612     const FieldDecl *field;
1613     CodeGenFunction::Destroyer *destroyer;
1614     bool useEHCleanupForArray;
1615 
1616   public:
1617     DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1618                  bool useEHCleanupForArray)
1619         : field(field), destroyer(destroyer),
1620           useEHCleanupForArray(useEHCleanupForArray) {}
1621 
1622     void Emit(CodeGenFunction &CGF, Flags flags) override {
1623       // Find the address of the field.
1624       Address thisValue = CGF.LoadCXXThisAddress();
1625       QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1626       LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
1627       LValue LV = CGF.EmitLValueForField(ThisLV, field);
1628       assert(LV.isSimple());
1629 
1630       CGF.emitDestroy(LV.getAddress(), field->getType(), destroyer,
1631                       flags.isForNormalCleanup() && useEHCleanupForArray);
1632     }
1633   };
1634 
1635  static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1636              CharUnits::QuantityType PoisonSize) {
1637    CodeGenFunction::SanitizerScope SanScope(&CGF);
1638    // Pass in void pointer and size of region as arguments to runtime
1639    // function
1640    llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
1641                           llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
1642 
1643    llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
1644 
1645    llvm::FunctionType *FnType =
1646        llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
1647    llvm::FunctionCallee Fn =
1648        CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
1649    CGF.EmitNounwindRuntimeCall(Fn, Args);
1650  }
1651 
1652   class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
1653     const CXXDestructorDecl *Dtor;
1654 
1655   public:
1656     SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1657 
1658     // Generate function call for handling object poisoning.
1659     // Disables tail call elimination, to prevent the current stack frame
1660     // from disappearing from the stack trace.
1661     void Emit(CodeGenFunction &CGF, Flags flags) override {
1662       const ASTRecordLayout &Layout =
1663           CGF.getContext().getASTRecordLayout(Dtor->getParent());
1664 
1665       // Nothing to poison.
1666       if (Layout.getFieldCount() == 0)
1667         return;
1668 
1669       // Prevent the current stack frame from disappearing from the stack trace.
1670       CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1671 
1672       // Construct pointer to region to begin poisoning, and calculate poison
1673       // size, so that only members declared in this class are poisoned.
1674       ASTContext &Context = CGF.getContext();
1675       unsigned fieldIndex = 0;
1676       int startIndex = -1;
1677       // RecordDecl::field_iterator Field;
1678       for (const FieldDecl *Field : Dtor->getParent()->fields()) {
1679         // Poison field if it is trivial
1680         if (FieldHasTrivialDestructorBody(Context, Field)) {
1681           // Start sanitizing at this field
1682           if (startIndex < 0)
1683             startIndex = fieldIndex;
1684 
1685           // Currently on the last field, and it must be poisoned with the
1686           // current block.
1687           if (fieldIndex == Layout.getFieldCount() - 1) {
1688             PoisonMembers(CGF, startIndex, Layout.getFieldCount());
1689           }
1690         } else if (startIndex >= 0) {
1691           // No longer within a block of memory to poison, so poison the block
1692           PoisonMembers(CGF, startIndex, fieldIndex);
1693           // Re-set the start index
1694           startIndex = -1;
1695         }
1696         fieldIndex += 1;
1697       }
1698     }
1699 
1700   private:
1701     /// \param layoutStartOffset index of the ASTRecordLayout field to
1702     ///     start poisoning (inclusive)
1703     /// \param layoutEndOffset index of the ASTRecordLayout field to
1704     ///     end poisoning (exclusive)
1705     void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
1706                      unsigned layoutEndOffset) {
1707       ASTContext &Context = CGF.getContext();
1708       const ASTRecordLayout &Layout =
1709           Context.getASTRecordLayout(Dtor->getParent());
1710 
1711       llvm::ConstantInt *OffsetSizePtr = llvm::ConstantInt::get(
1712           CGF.SizeTy,
1713           Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset))
1714               .getQuantity());
1715 
1716       llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
1717           CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
1718           OffsetSizePtr);
1719 
1720       CharUnits::QuantityType PoisonSize;
1721       if (layoutEndOffset >= Layout.getFieldCount()) {
1722         PoisonSize = Layout.getNonVirtualSize().getQuantity() -
1723                      Context.toCharUnitsFromBits(
1724                                 Layout.getFieldOffset(layoutStartOffset))
1725                          .getQuantity();
1726       } else {
1727         PoisonSize = Context.toCharUnitsFromBits(
1728                                 Layout.getFieldOffset(layoutEndOffset) -
1729                                 Layout.getFieldOffset(layoutStartOffset))
1730                          .getQuantity();
1731       }
1732 
1733       if (PoisonSize == 0)
1734         return;
1735 
1736       EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize);
1737     }
1738   };
1739 
1740  class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1741     const CXXDestructorDecl *Dtor;
1742 
1743   public:
1744     SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1745 
1746     // Generate function call for handling vtable pointer poisoning.
1747     void Emit(CodeGenFunction &CGF, Flags flags) override {
1748       assert(Dtor->getParent()->isDynamicClass());
1749       (void)Dtor;
1750       ASTContext &Context = CGF.getContext();
1751       // Poison vtable and vtable ptr if they exist for this class.
1752       llvm::Value *VTablePtr = CGF.LoadCXXThis();
1753 
1754       CharUnits::QuantityType PoisonSize =
1755           Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
1756       // Pass in void pointer and size of region as arguments to runtime
1757       // function
1758       EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
1759     }
1760  };
1761 } // end anonymous namespace
1762 
1763 /// Emit all code that comes at the end of class's
1764 /// destructor. This is to call destructors on members and base classes
1765 /// in reverse order of their construction.
1766 ///
1767 /// For a deleting destructor, this also handles the case where a destroying
1768 /// operator delete completely overrides the definition.
1769 void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1770                                         CXXDtorType DtorType) {
1771   assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1772          "Should not emit dtor epilogue for non-exported trivial dtor!");
1773 
1774   // The deleting-destructor phase just needs to call the appropriate
1775   // operator delete that Sema picked up.
1776   if (DtorType == Dtor_Deleting) {
1777     assert(DD->getOperatorDelete() &&
1778            "operator delete missing - EnterDtorCleanups");
1779     if (CXXStructorImplicitParamValue) {
1780       // If there is an implicit param to the deleting dtor, it's a boolean
1781       // telling whether this is a deleting destructor.
1782       if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1783         EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,
1784                                       /*ReturnAfterDelete*/true);
1785       else
1786         EHStack.pushCleanup<CallDtorDeleteConditional>(
1787             NormalAndEHCleanup, CXXStructorImplicitParamValue);
1788     } else {
1789       if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1790         const CXXRecordDecl *ClassDecl = DD->getParent();
1791         EmitDeleteCall(DD->getOperatorDelete(),
1792                        LoadThisForDtorDelete(*this, DD),
1793                        getContext().getTagDeclType(ClassDecl));
1794         EmitBranchThroughCleanup(ReturnBlock);
1795       } else {
1796         EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
1797       }
1798     }
1799     return;
1800   }
1801 
1802   const CXXRecordDecl *ClassDecl = DD->getParent();
1803 
1804   // Unions have no bases and do not call field destructors.
1805   if (ClassDecl->isUnion())
1806     return;
1807 
1808   // The complete-destructor phase just destructs all the virtual bases.
1809   if (DtorType == Dtor_Complete) {
1810     // Poison the vtable pointer such that access after the base
1811     // and member destructors are invoked is invalid.
1812     if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1813         SanOpts.has(SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
1814         ClassDecl->isPolymorphic())
1815       EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1816 
1817     // We push them in the forward order so that they'll be popped in
1818     // the reverse order.
1819     for (const auto &Base : ClassDecl->vbases()) {
1820       auto *BaseClassDecl =
1821           cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1822 
1823       // Ignore trivial destructors.
1824       if (BaseClassDecl->hasTrivialDestructor())
1825         continue;
1826 
1827       EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1828                                         BaseClassDecl,
1829                                         /*BaseIsVirtual*/ true);
1830     }
1831 
1832     return;
1833   }
1834 
1835   assert(DtorType == Dtor_Base);
1836   // Poison the vtable pointer if it has no virtual bases, but inherits
1837   // virtual functions.
1838   if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1839       SanOpts.has(SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
1840       ClassDecl->isPolymorphic())
1841     EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1842 
1843   // Destroy non-virtual bases.
1844   for (const auto &Base : ClassDecl->bases()) {
1845     // Ignore virtual bases.
1846     if (Base.isVirtual())
1847       continue;
1848 
1849     CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1850 
1851     // Ignore trivial destructors.
1852     if (BaseClassDecl->hasTrivialDestructor())
1853       continue;
1854 
1855     EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1856                                       BaseClassDecl,
1857                                       /*BaseIsVirtual*/ false);
1858   }
1859 
1860   // Poison fields such that access after their destructors are
1861   // invoked, and before the base class destructor runs, is invalid.
1862   if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1863       SanOpts.has(SanitizerKind::Memory))
1864     EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
1865 
1866   // Destroy direct fields.
1867   for (const auto *Field : ClassDecl->fields()) {
1868     QualType type = Field->getType();
1869     QualType::DestructionKind dtorKind = type.isDestructedType();
1870     if (!dtorKind) continue;
1871 
1872     // Anonymous union members do not have their destructors called.
1873     const RecordType *RT = type->getAsUnionType();
1874     if (RT && RT->getDecl()->isAnonymousStructOrUnion()) continue;
1875 
1876     CleanupKind cleanupKind = getCleanupKind(dtorKind);
1877     EHStack.pushCleanup<DestroyField>(cleanupKind, Field,
1878                                       getDestroyer(dtorKind),
1879                                       cleanupKind & EHCleanup);
1880   }
1881 }
1882 
1883 /// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1884 /// constructor for each of several members of an array.
1885 ///
1886 /// \param ctor the constructor to call for each element
1887 /// \param arrayType the type of the array to initialize
1888 /// \param arrayBegin an arrayType*
1889 /// \param zeroInitialize true if each element should be
1890 ///   zero-initialized before it is constructed
1891 void CodeGenFunction::EmitCXXAggrConstructorCall(
1892     const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1893     Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1894     bool zeroInitialize) {
1895   QualType elementType;
1896   llvm::Value *numElements =
1897     emitArrayLength(arrayType, elementType, arrayBegin);
1898 
1899   EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,
1900                              NewPointerIsChecked, zeroInitialize);
1901 }
1902 
1903 /// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1904 /// constructor for each of several members of an array.
1905 ///
1906 /// \param ctor the constructor to call for each element
1907 /// \param numElements the number of elements in the array;
1908 ///   may be zero
1909 /// \param arrayBase a T*, where T is the type constructed by ctor
1910 /// \param zeroInitialize true if each element should be
1911 ///   zero-initialized before it is constructed
1912 void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1913                                                  llvm::Value *numElements,
1914                                                  Address arrayBase,
1915                                                  const CXXConstructExpr *E,
1916                                                  bool NewPointerIsChecked,
1917                                                  bool zeroInitialize) {
1918   // It's legal for numElements to be zero.  This can happen both
1919   // dynamically, because x can be zero in 'new A[x]', and statically,
1920   // because of GCC extensions that permit zero-length arrays.  There
1921   // are probably legitimate places where we could assume that this
1922   // doesn't happen, but it's not clear that it's worth it.
1923   llvm::BranchInst *zeroCheckBranch = nullptr;
1924 
1925   // Optimize for a constant count.
1926   llvm::ConstantInt *constantCount
1927     = dyn_cast<llvm::ConstantInt>(numElements);
1928   if (constantCount) {
1929     // Just skip out if the constant count is zero.
1930     if (constantCount->isZero()) return;
1931 
1932   // Otherwise, emit the check.
1933   } else {
1934     llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");
1935     llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");
1936     zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
1937     EmitBlock(loopBB);
1938   }
1939 
1940   // Find the end of the array.
1941   llvm::Value *arrayBegin = arrayBase.getPointer();
1942   llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements,
1943                                                     "arrayctor.end");
1944 
1945   // Enter the loop, setting up a phi for the current location to initialize.
1946   llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1947   llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");
1948   EmitBlock(loopBB);
1949   llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,
1950                                          "arrayctor.cur");
1951   cur->addIncoming(arrayBegin, entryBB);
1952 
1953   // Inside the loop body, emit the constructor call on the array element.
1954 
1955   // The alignment of the base, adjusted by the size of a single element,
1956   // provides a conservative estimate of the alignment of every element.
1957   // (This assumes we never start tracking offsetted alignments.)
1958   //
1959   // Note that these are complete objects and so we don't need to
1960   // use the non-virtual size or alignment.
1961   QualType type = getContext().getTypeDeclType(ctor->getParent());
1962   CharUnits eltAlignment =
1963     arrayBase.getAlignment()
1964              .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
1965   Address curAddr = Address(cur, eltAlignment);
1966 
1967   // Zero initialize the storage, if requested.
1968   if (zeroInitialize)
1969     EmitNullInitialization(curAddr, type);
1970 
1971   // C++ [class.temporary]p4:
1972   // There are two contexts in which temporaries are destroyed at a different
1973   // point than the end of the full-expression. The first context is when a
1974   // default constructor is called to initialize an element of an array.
1975   // If the constructor has one or more default arguments, the destruction of
1976   // every temporary created in a default argument expression is sequenced
1977   // before the construction of the next array element, if any.
1978 
1979   {
1980     RunCleanupsScope Scope(*this);
1981 
1982     // Evaluate the constructor and its arguments in a regular
1983     // partial-destroy cleanup.
1984     if (getLangOpts().Exceptions &&
1985         !ctor->getParent()->hasTrivialDestructor()) {
1986       Destroyer *destroyer = destroyCXXObject;
1987       pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
1988                                      *destroyer);
1989     }
1990     auto currAVS = AggValueSlot::forAddr(
1991         curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,
1992         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
1993         AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,
1994         NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
1995                             : AggValueSlot::IsNotSanitizerChecked);
1996     EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
1997                            /*Delegating=*/false, currAVS, E);
1998   }
1999 
2000   // Go to the next element.
2001   llvm::Value *next =
2002     Builder.CreateInBoundsGEP(cur, llvm::ConstantInt::get(SizeTy, 1),
2003                               "arrayctor.next");
2004   cur->addIncoming(next, Builder.GetInsertBlock());
2005 
2006   // Check whether that's the end of the loop.
2007   llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");
2008   llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");
2009   Builder.CreateCondBr(done, contBB, loopBB);
2010 
2011   // Patch the earlier check to skip over the loop.
2012   if (zeroCheckBranch) zeroCheckBranch->setSuccessor(0, contBB);
2013 
2014   EmitBlock(contBB);
2015 }
2016 
2017 void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2018                                        Address addr,
2019                                        QualType type) {
2020   const RecordType *rtype = type->castAs<RecordType>();
2021   const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
2022   const CXXDestructorDecl *dtor = record->getDestructor();
2023   assert(!dtor->isTrivial());
2024   CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
2025                             /*Delegating=*/false, addr, type);
2026 }
2027 
2028 void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2029                                              CXXCtorType Type,
2030                                              bool ForVirtualBase,
2031                                              bool Delegating,
2032                                              AggValueSlot ThisAVS,
2033                                              const CXXConstructExpr *E) {
2034   CallArgList Args;
2035   Address This = ThisAVS.getAddress();
2036   LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2037   QualType ThisType = D->getThisType();
2038   LangAS ThisAS = ThisType.getTypePtr()->getPointeeType().getAddressSpace();
2039   llvm::Value *ThisPtr = This.getPointer();
2040 
2041   if (SlotAS != ThisAS) {
2042     unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
2043     llvm::Type *NewType =
2044         ThisPtr->getType()->getPointerElementType()->getPointerTo(TargetThisAS);
2045     ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
2046                                                     ThisAS, SlotAS, NewType);
2047   }
2048 
2049   // Push the this ptr.
2050   Args.add(RValue::get(ThisPtr), D->getThisType());
2051 
2052   // If this is a trivial constructor, emit a memcpy now before we lose
2053   // the alignment information on the argument.
2054   // FIXME: It would be better to preserve alignment information into CallArg.
2055   if (isMemcpyEquivalentSpecialMember(D)) {
2056     assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2057 
2058     const Expr *Arg = E->getArg(0);
2059     LValue Src = EmitLValue(Arg);
2060     QualType DestTy = getContext().getTypeDeclType(D->getParent());
2061     LValue Dest = MakeAddrLValue(This, DestTy);
2062     EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
2063     return;
2064   }
2065 
2066   // Add the rest of the user-supplied arguments.
2067   const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2068   EvaluationOrder Order = E->isListInitialization()
2069                               ? EvaluationOrder::ForceLeftToRight
2070                               : EvaluationOrder::Default;
2071   EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),
2072                /*ParamsToSkip*/ 0, Order);
2073 
2074   EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2075                          ThisAVS.mayOverlap(), E->getExprLoc(),
2076                          ThisAVS.isSanitizerChecked());
2077 }
2078 
2079 static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2080                                     const CXXConstructorDecl *Ctor,
2081                                     CXXCtorType Type, CallArgList &Args) {
2082   // We can't forward a variadic call.
2083   if (Ctor->isVariadic())
2084     return false;
2085 
2086   if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2087     // If the parameters are callee-cleanup, it's not safe to forward.
2088     for (auto *P : Ctor->parameters())
2089       if (P->needsDestruction(CGF.getContext()))
2090         return false;
2091 
2092     // Likewise if they're inalloca.
2093     const CGFunctionInfo &Info =
2094         CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);
2095     if (Info.usesInAlloca())
2096       return false;
2097   }
2098 
2099   // Anything else should be OK.
2100   return true;
2101 }
2102 
2103 void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2104                                              CXXCtorType Type,
2105                                              bool ForVirtualBase,
2106                                              bool Delegating,
2107                                              Address This,
2108                                              CallArgList &Args,
2109                                              AggValueSlot::Overlap_t Overlap,
2110                                              SourceLocation Loc,
2111                                              bool NewPointerIsChecked) {
2112   const CXXRecordDecl *ClassDecl = D->getParent();
2113 
2114   if (!NewPointerIsChecked)
2115     EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
2116                   getContext().getRecordType(ClassDecl), CharUnits::Zero());
2117 
2118   if (D->isTrivial() && D->isDefaultConstructor()) {
2119     assert(Args.size() == 1 && "trivial default ctor with args");
2120     return;
2121   }
2122 
2123   // If this is a trivial constructor, just emit what's needed. If this is a
2124   // union copy constructor, we must emit a memcpy, because the AST does not
2125   // model that copy.
2126   if (isMemcpyEquivalentSpecialMember(D)) {
2127     assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2128 
2129     QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2130     Address Src(Args[1].getRValue(*this).getScalarVal(),
2131                 getNaturalTypeAlignment(SrcTy));
2132     LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
2133     QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2134     LValue DestLVal = MakeAddrLValue(This, DestTy);
2135     EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
2136     return;
2137   }
2138 
2139   bool PassPrototypeArgs = true;
2140   // Check whether we can actually emit the constructor before trying to do so.
2141   if (auto Inherited = D->getInheritedConstructor()) {
2142     PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2143     if (PassPrototypeArgs && !canEmitDelegateCallArgs(*this, D, Type, Args)) {
2144       EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,
2145                                               Delegating, Args);
2146       return;
2147     }
2148   }
2149 
2150   // Insert any ABI-specific implicit constructor arguments.
2151   CGCXXABI::AddedStructorArgs ExtraArgs =
2152       CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,
2153                                                  Delegating, Args);
2154 
2155   // Emit the call.
2156   llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));
2157   const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2158       Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
2159   CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2160   EmitCall(Info, Callee, ReturnValueSlot(), Args);
2161 
2162   // Generate vtable assumptions if we're constructing a complete object
2163   // with a vtable.  We don't do this for base subobjects for two reasons:
2164   // first, it's incorrect for classes with virtual bases, and second, we're
2165   // about to overwrite the vptrs anyway.
2166   // We also have to make sure if we can refer to vtable:
2167   // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2168   // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2169   // sure that definition of vtable is not hidden,
2170   // then we are always safe to refer to it.
2171   // FIXME: It looks like InstCombine is very inefficient on dealing with
2172   // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2173   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2174       ClassDecl->isDynamicClass() && Type != Ctor_Base &&
2175       CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) &&
2176       CGM.getCodeGenOpts().StrictVTablePointers)
2177     EmitVTableAssumptionLoads(ClassDecl, This);
2178 }
2179 
2180 void CodeGenFunction::EmitInheritedCXXConstructorCall(
2181     const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2182     bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2183   CallArgList Args;
2184   CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
2185 
2186   // Forward the parameters.
2187   if (InheritedFromVBase &&
2188       CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2189     // Nothing to do; this construction is not responsible for constructing
2190     // the base class containing the inherited constructor.
2191     // FIXME: Can we just pass undef's for the remaining arguments if we don't
2192     // have constructor variants?
2193     Args.push_back(ThisArg);
2194   } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2195     // The inheriting constructor was inlined; just inject its arguments.
2196     assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2197            "wrong number of parameters for inherited constructor call");
2198     Args = CXXInheritedCtorInitExprArgs;
2199     Args[0] = ThisArg;
2200   } else {
2201     // The inheriting constructor was not inlined. Emit delegating arguments.
2202     Args.push_back(ThisArg);
2203     const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);
2204     assert(OuterCtor->getNumParams() == D->getNumParams());
2205     assert(!OuterCtor->isVariadic() && "should have been inlined");
2206 
2207     for (const auto *Param : OuterCtor->parameters()) {
2208       assert(getContext().hasSameUnqualifiedType(
2209           OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2210           Param->getType()));
2211       EmitDelegateCallArg(Args, Param, E->getLocation());
2212 
2213       // Forward __attribute__(pass_object_size).
2214       if (Param->hasAttr<PassObjectSizeAttr>()) {
2215         auto *POSParam = SizeArguments[Param];
2216         assert(POSParam && "missing pass_object_size value for forwarding");
2217         EmitDelegateCallArg(Args, POSParam, E->getLocation());
2218       }
2219     }
2220   }
2221 
2222   EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,
2223                          This, Args, AggValueSlot::MayOverlap,
2224                          E->getLocation(), /*NewPointerIsChecked*/true);
2225 }
2226 
2227 void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2228     const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2229     bool Delegating, CallArgList &Args) {
2230   GlobalDecl GD(Ctor, CtorType);
2231   InlinedInheritingConstructorScope Scope(*this, GD);
2232   ApplyInlineDebugLocation DebugScope(*this, GD);
2233   RunCleanupsScope RunCleanups(*this);
2234 
2235   // Save the arguments to be passed to the inherited constructor.
2236   CXXInheritedCtorInitExprArgs = Args;
2237 
2238   FunctionArgList Params;
2239   QualType RetType = BuildFunctionArgList(CurGD, Params);
2240   FnRetTy = RetType;
2241 
2242   // Insert any ABI-specific implicit constructor arguments.
2243   CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,
2244                                              ForVirtualBase, Delegating, Args);
2245 
2246   // Emit a simplified prolog. We only need to emit the implicit params.
2247   assert(Args.size() >= Params.size() && "too few arguments for call");
2248   for (unsigned I = 0, N = Args.size(); I != N; ++I) {
2249     if (I < Params.size() && isa<ImplicitParamDecl>(Params[I])) {
2250       const RValue &RV = Args[I].getRValue(*this);
2251       assert(!RV.isComplex() && "complex indirect params not supported");
2252       ParamValue Val = RV.isScalar()
2253                            ? ParamValue::forDirect(RV.getScalarVal())
2254                            : ParamValue::forIndirect(RV.getAggregateAddress());
2255       EmitParmDecl(*Params[I], Val, I + 1);
2256     }
2257   }
2258 
2259   // Create a return value slot if the ABI implementation wants one.
2260   // FIXME: This is dumb, we should ask the ABI not to try to set the return
2261   // value instead.
2262   if (!RetType->isVoidType())
2263     ReturnValue = CreateIRTemp(RetType, "retval.inhctor");
2264 
2265   CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
2266   CXXThisValue = CXXABIThisValue;
2267 
2268   // Directly emit the constructor initializers.
2269   EmitCtorPrologue(Ctor, CtorType, Params);
2270 }
2271 
2272 void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2273   llvm::Value *VTableGlobal =
2274       CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
2275   if (!VTableGlobal)
2276     return;
2277 
2278   // We can just use the base offset in the complete class.
2279   CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2280 
2281   if (!NonVirtualOffset.isZero())
2282     This =
2283         ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
2284                                         Vptr.VTableClass, Vptr.NearestVBase);
2285 
2286   llvm::Value *VPtrValue =
2287       GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
2288   llvm::Value *Cmp =
2289       Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
2290   Builder.CreateAssumption(Cmp);
2291 }
2292 
2293 void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2294                                                 Address This) {
2295   if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
2296     for (const VPtr &Vptr : getVTablePointers(ClassDecl))
2297       EmitVTableAssumptionLoad(Vptr, This);
2298 }
2299 
2300 void
2301 CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2302                                                 Address This, Address Src,
2303                                                 const CXXConstructExpr *E) {
2304   const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2305 
2306   CallArgList Args;
2307 
2308   // Push the this ptr.
2309   Args.add(RValue::get(This.getPointer()), D->getThisType());
2310 
2311   // Push the src ptr.
2312   QualType QT = *(FPT->param_type_begin());
2313   llvm::Type *t = CGM.getTypes().ConvertType(QT);
2314   Src = Builder.CreateBitCast(Src, t);
2315   Args.add(RValue::get(Src.getPointer()), QT);
2316 
2317   // Skip over first argument (Src).
2318   EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
2319                /*ParamsToSkip*/ 1);
2320 
2321   EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2322                          /*Delegating*/false, This, Args,
2323                          AggValueSlot::MayOverlap, E->getExprLoc(),
2324                          /*NewPointerIsChecked*/false);
2325 }
2326 
2327 void
2328 CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2329                                                 CXXCtorType CtorType,
2330                                                 const FunctionArgList &Args,
2331                                                 SourceLocation Loc) {
2332   CallArgList DelegateArgs;
2333 
2334   FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2335   assert(I != E && "no parameters to constructor");
2336 
2337   // this
2338   Address This = LoadCXXThisAddress();
2339   DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
2340   ++I;
2341 
2342   // FIXME: The location of the VTT parameter in the parameter list is
2343   // specific to the Itanium ABI and shouldn't be hardcoded here.
2344   if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
2345     assert(I != E && "cannot skip vtt parameter, already done with args");
2346     assert((*I)->getType()->isPointerType() &&
2347            "skipping parameter not of vtt type");
2348     ++I;
2349   }
2350 
2351   // Explicit arguments.
2352   for (; I != E; ++I) {
2353     const VarDecl *param = *I;
2354     // FIXME: per-argument source location
2355     EmitDelegateCallArg(DelegateArgs, param, Loc);
2356   }
2357 
2358   EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,
2359                          /*Delegating=*/true, This, DelegateArgs,
2360                          AggValueSlot::MayOverlap, Loc,
2361                          /*NewPointerIsChecked=*/true);
2362 }
2363 
2364 namespace {
2365   struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2366     const CXXDestructorDecl *Dtor;
2367     Address Addr;
2368     CXXDtorType Type;
2369 
2370     CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2371                            CXXDtorType Type)
2372       : Dtor(D), Addr(Addr), Type(Type) {}
2373 
2374     void Emit(CodeGenFunction &CGF, Flags flags) override {
2375       // We are calling the destructor from within the constructor.
2376       // Therefore, "this" should have the expected type.
2377       QualType ThisTy = Dtor->getThisObjectType();
2378       CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,
2379                                 /*Delegating=*/true, Addr, ThisTy);
2380     }
2381   };
2382 } // end anonymous namespace
2383 
2384 void
2385 CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2386                                                   const FunctionArgList &Args) {
2387   assert(Ctor->isDelegatingConstructor());
2388 
2389   Address ThisPtr = LoadCXXThisAddress();
2390 
2391   AggValueSlot AggSlot =
2392     AggValueSlot::forAddr(ThisPtr, Qualifiers(),
2393                           AggValueSlot::IsDestructed,
2394                           AggValueSlot::DoesNotNeedGCBarriers,
2395                           AggValueSlot::IsNotAliased,
2396                           AggValueSlot::MayOverlap,
2397                           AggValueSlot::IsNotZeroed,
2398                           // Checks are made by the code that calls constructor.
2399                           AggValueSlot::IsSanitizerChecked);
2400 
2401   EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
2402 
2403   const CXXRecordDecl *ClassDecl = Ctor->getParent();
2404   if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {
2405     CXXDtorType Type =
2406       CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;
2407 
2408     EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,
2409                                                 ClassDecl->getDestructor(),
2410                                                 ThisPtr, Type);
2411   }
2412 }
2413 
2414 void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2415                                             CXXDtorType Type,
2416                                             bool ForVirtualBase,
2417                                             bool Delegating, Address This,
2418                                             QualType ThisTy) {
2419   CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
2420                                      Delegating, This, ThisTy);
2421 }
2422 
2423 namespace {
2424   struct CallLocalDtor final : EHScopeStack::Cleanup {
2425     const CXXDestructorDecl *Dtor;
2426     Address Addr;
2427     QualType Ty;
2428 
2429     CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2430         : Dtor(D), Addr(Addr), Ty(Ty) {}
2431 
2432     void Emit(CodeGenFunction &CGF, Flags flags) override {
2433       CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2434                                 /*ForVirtualBase=*/false,
2435                                 /*Delegating=*/false, Addr, Ty);
2436     }
2437   };
2438 } // end anonymous namespace
2439 
2440 void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2441                                             QualType T, Address Addr) {
2442   EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);
2443 }
2444 
2445 void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2446   CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2447   if (!ClassDecl) return;
2448   if (ClassDecl->hasTrivialDestructor()) return;
2449 
2450   const CXXDestructorDecl *D = ClassDecl->getDestructor();
2451   assert(D && D->isUsed() && "destructor not marked as used!");
2452   PushDestructorCleanup(D, T, Addr);
2453 }
2454 
2455 void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2456   // Compute the address point.
2457   llvm::Value *VTableAddressPoint =
2458       CGM.getCXXABI().getVTableAddressPointInStructor(
2459           *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
2460 
2461   if (!VTableAddressPoint)
2462     return;
2463 
2464   // Compute where to store the address point.
2465   llvm::Value *VirtualOffset = nullptr;
2466   CharUnits NonVirtualOffset = CharUnits::Zero();
2467 
2468   if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
2469     // We need to use the virtual base offset offset because the virtual base
2470     // might have a different offset in the most derived class.
2471 
2472     VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2473         *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
2474     NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2475   } else {
2476     // We can just use the base offset in the complete class.
2477     NonVirtualOffset = Vptr.Base.getBaseOffset();
2478   }
2479 
2480   // Apply the offsets.
2481   Address VTableField = LoadCXXThisAddress();
2482 
2483   if (!NonVirtualOffset.isZero() || VirtualOffset)
2484     VTableField = ApplyNonVirtualAndVirtualOffset(
2485         *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
2486         Vptr.NearestVBase);
2487 
2488   // Finally, store the address point. Use the same LLVM types as the field to
2489   // support optimization.
2490   llvm::Type *VTablePtrTy =
2491       llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
2492           ->getPointerTo()
2493           ->getPointerTo();
2494   VTableField = Builder.CreateBitCast(VTableField, VTablePtrTy->getPointerTo());
2495   VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
2496 
2497   llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
2498   TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrTy);
2499   CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2500   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2501       CGM.getCodeGenOpts().StrictVTablePointers)
2502     CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
2503 }
2504 
2505 CodeGenFunction::VPtrsVector
2506 CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2507   CodeGenFunction::VPtrsVector VPtrsResult;
2508   VisitedVirtualBasesSetTy VBases;
2509   getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
2510                     /*NearestVBase=*/nullptr,
2511                     /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2512                     /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2513                     VPtrsResult);
2514   return VPtrsResult;
2515 }
2516 
2517 void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2518                                         const CXXRecordDecl *NearestVBase,
2519                                         CharUnits OffsetFromNearestVBase,
2520                                         bool BaseIsNonVirtualPrimaryBase,
2521                                         const CXXRecordDecl *VTableClass,
2522                                         VisitedVirtualBasesSetTy &VBases,
2523                                         VPtrsVector &Vptrs) {
2524   // If this base is a non-virtual primary base the address point has already
2525   // been set.
2526   if (!BaseIsNonVirtualPrimaryBase) {
2527     // Initialize the vtable pointer for this base.
2528     VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
2529     Vptrs.push_back(Vptr);
2530   }
2531 
2532   const CXXRecordDecl *RD = Base.getBase();
2533 
2534   // Traverse bases.
2535   for (const auto &I : RD->bases()) {
2536     auto *BaseDecl =
2537         cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
2538 
2539     // Ignore classes without a vtable.
2540     if (!BaseDecl->isDynamicClass())
2541       continue;
2542 
2543     CharUnits BaseOffset;
2544     CharUnits BaseOffsetFromNearestVBase;
2545     bool BaseDeclIsNonVirtualPrimaryBase;
2546 
2547     if (I.isVirtual()) {
2548       // Check if we've visited this virtual base before.
2549       if (!VBases.insert(BaseDecl).second)
2550         continue;
2551 
2552       const ASTRecordLayout &Layout =
2553         getContext().getASTRecordLayout(VTableClass);
2554 
2555       BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
2556       BaseOffsetFromNearestVBase = CharUnits::Zero();
2557       BaseDeclIsNonVirtualPrimaryBase = false;
2558     } else {
2559       const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2560 
2561       BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
2562       BaseOffsetFromNearestVBase =
2563         OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
2564       BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2565     }
2566 
2567     getVTablePointers(
2568         BaseSubobject(BaseDecl, BaseOffset),
2569         I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase,
2570         BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2571   }
2572 }
2573 
2574 void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2575   // Ignore classes without a vtable.
2576   if (!RD->isDynamicClass())
2577     return;
2578 
2579   // Initialize the vtable pointers for this class and all of its bases.
2580   if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
2581     for (const VPtr &Vptr : getVTablePointers(RD))
2582       InitializeVTablePointer(Vptr);
2583 
2584   if (RD->getNumVBases())
2585     CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
2586 }
2587 
2588 llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2589                                            llvm::Type *VTableTy,
2590                                            const CXXRecordDecl *RD) {
2591   Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
2592   llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
2593   TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);
2594   CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);
2595 
2596   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2597       CGM.getCodeGenOpts().StrictVTablePointers)
2598     CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
2599 
2600   return VTable;
2601 }
2602 
2603 // If a class has a single non-virtual base and does not introduce or override
2604 // virtual member functions or fields, it will have the same layout as its base.
2605 // This function returns the least derived such class.
2606 //
2607 // Casting an instance of a base class to such a derived class is technically
2608 // undefined behavior, but it is a relatively common hack for introducing member
2609 // functions on class instances with specific properties (e.g. llvm::Operator)
2610 // that works under most compilers and should not have security implications, so
2611 // we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2612 static const CXXRecordDecl *
2613 LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2614   if (!RD->field_empty())
2615     return RD;
2616 
2617   if (RD->getNumVBases() != 0)
2618     return RD;
2619 
2620   if (RD->getNumBases() != 1)
2621     return RD;
2622 
2623   for (const CXXMethodDecl *MD : RD->methods()) {
2624     if (MD->isVirtual()) {
2625       // Virtual member functions are only ok if they are implicit destructors
2626       // because the implicit destructor will have the same semantics as the
2627       // base class's destructor if no fields are added.
2628       if (isa<CXXDestructorDecl>(MD) && MD->isImplicit())
2629         continue;
2630       return RD;
2631     }
2632   }
2633 
2634   return LeastDerivedClassWithSameLayout(
2635       RD->bases_begin()->getType()->getAsCXXRecordDecl());
2636 }
2637 
2638 void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2639                                                    llvm::Value *VTable,
2640                                                    SourceLocation Loc) {
2641   if (SanOpts.has(SanitizerKind::CFIVCall))
2642     EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
2643   else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2644            CGM.HasHiddenLTOVisibility(RD)) {
2645     llvm::Metadata *MD =
2646         CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2647     llvm::Value *TypeId =
2648         llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2649 
2650     llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2651     llvm::Value *TypeTest =
2652         Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
2653                            {CastedVTable, TypeId});
2654     Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2655   }
2656 }
2657 
2658 void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2659                                                 llvm::Value *VTable,
2660                                                 CFITypeCheckKind TCK,
2661                                                 SourceLocation Loc) {
2662   if (!SanOpts.has(SanitizerKind::CFICastStrict))
2663     RD = LeastDerivedClassWithSameLayout(RD);
2664 
2665   EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2666 }
2667 
2668 void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
2669                                                 llvm::Value *Derived,
2670                                                 bool MayBeNull,
2671                                                 CFITypeCheckKind TCK,
2672                                                 SourceLocation Loc) {
2673   if (!getLangOpts().CPlusPlus)
2674     return;
2675 
2676   auto *ClassTy = T->getAs<RecordType>();
2677   if (!ClassTy)
2678     return;
2679 
2680   const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
2681 
2682   if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2683     return;
2684 
2685   if (!SanOpts.has(SanitizerKind::CFICastStrict))
2686     ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
2687 
2688   llvm::BasicBlock *ContBlock = nullptr;
2689 
2690   if (MayBeNull) {
2691     llvm::Value *DerivedNotNull =
2692         Builder.CreateIsNotNull(Derived, "cast.nonnull");
2693 
2694     llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
2695     ContBlock = createBasicBlock("cast.cont");
2696 
2697     Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
2698 
2699     EmitBlock(CheckBlock);
2700   }
2701 
2702   llvm::Value *VTable;
2703   std::tie(VTable, ClassDecl) = CGM.getCXXABI().LoadVTablePtr(
2704       *this, Address(Derived, getPointerAlign()), ClassDecl);
2705 
2706   EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
2707 
2708   if (MayBeNull) {
2709     Builder.CreateBr(ContBlock);
2710     EmitBlock(ContBlock);
2711   }
2712 }
2713 
2714 void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2715                                          llvm::Value *VTable,
2716                                          CFITypeCheckKind TCK,
2717                                          SourceLocation Loc) {
2718   if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2719       !CGM.HasHiddenLTOVisibility(RD))
2720     return;
2721 
2722   SanitizerMask M;
2723   llvm::SanitizerStatKind SSK;
2724   switch (TCK) {
2725   case CFITCK_VCall:
2726     M = SanitizerKind::CFIVCall;
2727     SSK = llvm::SanStat_CFI_VCall;
2728     break;
2729   case CFITCK_NVCall:
2730     M = SanitizerKind::CFINVCall;
2731     SSK = llvm::SanStat_CFI_NVCall;
2732     break;
2733   case CFITCK_DerivedCast:
2734     M = SanitizerKind::CFIDerivedCast;
2735     SSK = llvm::SanStat_CFI_DerivedCast;
2736     break;
2737   case CFITCK_UnrelatedCast:
2738     M = SanitizerKind::CFIUnrelatedCast;
2739     SSK = llvm::SanStat_CFI_UnrelatedCast;
2740     break;
2741   case CFITCK_ICall:
2742   case CFITCK_NVMFCall:
2743   case CFITCK_VMFCall:
2744     llvm_unreachable("unexpected sanitizer kind");
2745   }
2746 
2747   std::string TypeName = RD->getQualifiedNameAsString();
2748   if (getContext().getSanitizerBlacklist().isBlacklistedType(M, TypeName))
2749     return;
2750 
2751   SanitizerScope SanScope(this);
2752   EmitSanitizerStatReport(SSK);
2753 
2754   llvm::Metadata *MD =
2755       CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2756   llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
2757 
2758   llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2759   llvm::Value *TypeTest = Builder.CreateCall(
2760       CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, TypeId});
2761 
2762   llvm::Constant *StaticData[] = {
2763       llvm::ConstantInt::get(Int8Ty, TCK),
2764       EmitCheckSourceLocation(Loc),
2765       EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
2766   };
2767 
2768   auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2769   if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
2770     EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, CastedVTable, StaticData);
2771     return;
2772   }
2773 
2774   if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
2775     EmitTrapCheck(TypeTest);
2776     return;
2777   }
2778 
2779   llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2780       CGM.getLLVMContext(),
2781       llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
2782   llvm::Value *ValidVtable = Builder.CreateCall(
2783       CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables});
2784   EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,
2785             StaticData, {CastedVTable, ValidVtable});
2786 }
2787 
2788 bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2789   if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2790       !CGM.HasHiddenLTOVisibility(RD))
2791     return false;
2792 
2793   if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2794     return true;
2795 
2796   if (!SanOpts.has(SanitizerKind::CFIVCall) ||
2797       !CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall))
2798     return false;
2799 
2800   std::string TypeName = RD->getQualifiedNameAsString();
2801   return !getContext().getSanitizerBlacklist().isBlacklistedType(
2802       SanitizerKind::CFIVCall, TypeName);
2803 }
2804 
2805 llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2806     const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) {
2807   SanitizerScope SanScope(this);
2808 
2809   EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
2810 
2811   llvm::Metadata *MD =
2812       CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2813   llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2814 
2815   llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2816   llvm::Value *CheckedLoad = Builder.CreateCall(
2817       CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2818       {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset),
2819        TypeId});
2820   llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
2821 
2822   std::string TypeName = RD->getQualifiedNameAsString();
2823   if (SanOpts.has(SanitizerKind::CFIVCall) &&
2824       !getContext().getSanitizerBlacklist().isBlacklistedType(
2825           SanitizerKind::CFIVCall, TypeName)) {
2826     EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall),
2827               SanitizerHandler::CFICheckFail, {}, {});
2828   }
2829 
2830   return Builder.CreateBitCast(
2831       Builder.CreateExtractValue(CheckedLoad, 0),
2832       cast<llvm::PointerType>(VTable->getType())->getElementType());
2833 }
2834 
2835 void CodeGenFunction::EmitForwardingCallToLambda(
2836                                       const CXXMethodDecl *callOperator,
2837                                       CallArgList &callArgs) {
2838   // Get the address of the call operator.
2839   const CGFunctionInfo &calleeFnInfo =
2840     CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);
2841   llvm::Constant *calleePtr =
2842     CGM.GetAddrOfFunction(GlobalDecl(callOperator),
2843                           CGM.getTypes().GetFunctionType(calleeFnInfo));
2844 
2845   // Prepare the return slot.
2846   const FunctionProtoType *FPT =
2847     callOperator->getType()->castAs<FunctionProtoType>();
2848   QualType resultType = FPT->getReturnType();
2849   ReturnValueSlot returnSlot;
2850   if (!resultType->isVoidType() &&
2851       calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect &&
2852       !hasScalarEvaluationKind(calleeFnInfo.getReturnType()))
2853     returnSlot = ReturnValueSlot(ReturnValue, resultType.isVolatileQualified());
2854 
2855   // We don't need to separately arrange the call arguments because
2856   // the call can't be variadic anyway --- it's impossible to forward
2857   // variadic arguments.
2858 
2859   // Now emit our call.
2860   auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));
2861   RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs);
2862 
2863   // If necessary, copy the returned value into the slot.
2864   if (!resultType->isVoidType() && returnSlot.isNull()) {
2865     if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {
2866       RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));
2867     }
2868     EmitReturnOfRValue(RV, resultType);
2869   } else
2870     EmitBranchThroughCleanup(ReturnBlock);
2871 }
2872 
2873 void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2874   const BlockDecl *BD = BlockInfo->getBlockDecl();
2875   const VarDecl *variable = BD->capture_begin()->getVariable();
2876   const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2877   const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2878 
2879   if (CallOp->isVariadic()) {
2880     // FIXME: Making this work correctly is nasty because it requires either
2881     // cloning the body of the call operator or making the call operator
2882     // forward.
2883     CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");
2884     return;
2885   }
2886 
2887   // Start building arguments for forwarding call
2888   CallArgList CallArgs;
2889 
2890   QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2891   Address ThisPtr = GetAddrOfBlockDecl(variable);
2892   CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
2893 
2894   // Add the rest of the parameters.
2895   for (auto param : BD->parameters())
2896     EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());
2897 
2898   assert(!Lambda->isGenericLambda() &&
2899             "generic lambda interconversion to block not implemented");
2900   EmitForwardingCallToLambda(CallOp, CallArgs);
2901 }
2902 
2903 void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
2904   const CXXRecordDecl *Lambda = MD->getParent();
2905 
2906   // Start building arguments for forwarding call
2907   CallArgList CallArgs;
2908 
2909   QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2910   llvm::Value *ThisPtr = llvm::UndefValue::get(getTypes().ConvertType(ThisType));
2911   CallArgs.add(RValue::get(ThisPtr), ThisType);
2912 
2913   // Add the rest of the parameters.
2914   for (auto Param : MD->parameters())
2915     EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
2916 
2917   const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2918   // For a generic lambda, find the corresponding call operator specialization
2919   // to which the call to the static-invoker shall be forwarded.
2920   if (Lambda->isGenericLambda()) {
2921     assert(MD->isFunctionTemplateSpecialization());
2922     const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
2923     FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
2924     void *InsertPos = nullptr;
2925     FunctionDecl *CorrespondingCallOpSpecialization =
2926         CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
2927     assert(CorrespondingCallOpSpecialization);
2928     CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
2929   }
2930   EmitForwardingCallToLambda(CallOp, CallArgs);
2931 }
2932 
2933 void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2934   if (MD->isVariadic()) {
2935     // FIXME: Making this work correctly is nasty because it requires either
2936     // cloning the body of the call operator or making the call operator forward.
2937     CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2938     return;
2939   }
2940 
2941   EmitLambdaDelegatingInvokeBody(MD);
2942 }
2943