xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGClass.cpp (revision 7877fdebeeb35fad1cbbafce22598b1bdf97c786)
1 //===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code dealing with C++ code generation of classes
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGBlocks.h"
14 #include "CGCXXABI.h"
15 #include "CGDebugInfo.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenFunction.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Attr.h"
20 #include "clang/AST/CXXInheritance.h"
21 #include "clang/AST/DeclTemplate.h"
22 #include "clang/AST/EvaluatedExprVisitor.h"
23 #include "clang/AST/RecordLayout.h"
24 #include "clang/AST/StmtCXX.h"
25 #include "clang/Basic/CodeGenOptions.h"
26 #include "clang/Basic/TargetBuiltins.h"
27 #include "clang/CodeGen/CGFunctionInfo.h"
28 #include "llvm/IR/Intrinsics.h"
29 #include "llvm/IR/Metadata.h"
30 #include "llvm/Transforms/Utils/SanitizerStats.h"
31 
32 using namespace clang;
33 using namespace CodeGen;
34 
35 /// Return the best known alignment for an unknown pointer to a
36 /// particular class.
37 CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
38   if (!RD->hasDefinition())
39     return CharUnits::One(); // Hopefully won't be used anywhere.
40 
41   auto &layout = getContext().getASTRecordLayout(RD);
42 
43   // If the class is final, then we know that the pointer points to an
44   // object of that type and can use the full alignment.
45   if (RD->isEffectivelyFinal())
46     return layout.getAlignment();
47 
48   // Otherwise, we have to assume it could be a subclass.
49   return layout.getNonVirtualAlignment();
50 }
51 
52 /// Return the smallest possible amount of storage that might be allocated
53 /// starting from the beginning of an object of a particular class.
54 ///
55 /// This may be smaller than sizeof(RD) if RD has virtual base classes.
56 CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
57   if (!RD->hasDefinition())
58     return CharUnits::One();
59 
60   auto &layout = getContext().getASTRecordLayout(RD);
61 
62   // If the class is final, then we know that the pointer points to an
63   // object of that type and can use the full alignment.
64   if (RD->isEffectivelyFinal())
65     return layout.getSize();
66 
67   // Otherwise, we have to assume it could be a subclass.
68   return std::max(layout.getNonVirtualSize(), CharUnits::One());
69 }
70 
71 /// Return the best known alignment for a pointer to a virtual base,
72 /// given the alignment of a pointer to the derived class.
73 CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
74                                            const CXXRecordDecl *derivedClass,
75                                            const CXXRecordDecl *vbaseClass) {
76   // The basic idea here is that an underaligned derived pointer might
77   // indicate an underaligned base pointer.
78 
79   assert(vbaseClass->isCompleteDefinition());
80   auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
81   CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
82 
83   return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
84                                    expectedVBaseAlign);
85 }
86 
87 CharUnits
88 CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
89                                          const CXXRecordDecl *baseDecl,
90                                          CharUnits expectedTargetAlign) {
91   // If the base is an incomplete type (which is, alas, possible with
92   // member pointers), be pessimistic.
93   if (!baseDecl->isCompleteDefinition())
94     return std::min(actualBaseAlign, expectedTargetAlign);
95 
96   auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
97   CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
98 
99   // If the class is properly aligned, assume the target offset is, too.
100   //
101   // This actually isn't necessarily the right thing to do --- if the
102   // class is a complete object, but it's only properly aligned for a
103   // base subobject, then the alignments of things relative to it are
104   // probably off as well.  (Note that this requires the alignment of
105   // the target to be greater than the NV alignment of the derived
106   // class.)
107   //
108   // However, our approach to this kind of under-alignment can only
109   // ever be best effort; after all, we're never going to propagate
110   // alignments through variables or parameters.  Note, in particular,
111   // that constructing a polymorphic type in an address that's less
112   // than pointer-aligned will generally trap in the constructor,
113   // unless we someday add some sort of attribute to change the
114   // assumed alignment of 'this'.  So our goal here is pretty much
115   // just to allow the user to explicitly say that a pointer is
116   // under-aligned and then safely access its fields and vtables.
117   if (actualBaseAlign >= expectedBaseAlign) {
118     return expectedTargetAlign;
119   }
120 
121   // Otherwise, we might be offset by an arbitrary multiple of the
122   // actual alignment.  The correct adjustment is to take the min of
123   // the two alignments.
124   return std::min(actualBaseAlign, expectedTargetAlign);
125 }
126 
127 Address CodeGenFunction::LoadCXXThisAddress() {
128   assert(CurFuncDecl && "loading 'this' without a func declaration?");
129   assert(isa<CXXMethodDecl>(CurFuncDecl));
130 
131   // Lazily compute CXXThisAlignment.
132   if (CXXThisAlignment.isZero()) {
133     // Just use the best known alignment for the parent.
134     // TODO: if we're currently emitting a complete-object ctor/dtor,
135     // we can always use the complete-object alignment.
136     auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent();
137     CXXThisAlignment = CGM.getClassPointerAlignment(RD);
138   }
139 
140   return Address(LoadCXXThis(), CXXThisAlignment);
141 }
142 
143 /// Emit the address of a field using a member data pointer.
144 ///
145 /// \param E Only used for emergency diagnostics
146 Address
147 CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
148                                                  llvm::Value *memberPtr,
149                                       const MemberPointerType *memberPtrType,
150                                                  LValueBaseInfo *BaseInfo,
151                                                  TBAAAccessInfo *TBAAInfo) {
152   // Ask the ABI to compute the actual address.
153   llvm::Value *ptr =
154     CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
155                                                  memberPtr, memberPtrType);
156 
157   QualType memberType = memberPtrType->getPointeeType();
158   CharUnits memberAlign =
159       CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo);
160   memberAlign =
161     CGM.getDynamicOffsetAlignment(base.getAlignment(),
162                             memberPtrType->getClass()->getAsCXXRecordDecl(),
163                                   memberAlign);
164   return Address(ptr, memberAlign);
165 }
166 
167 CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
168     const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
169     CastExpr::path_const_iterator End) {
170   CharUnits Offset = CharUnits::Zero();
171 
172   const ASTContext &Context = getContext();
173   const CXXRecordDecl *RD = DerivedClass;
174 
175   for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
176     const CXXBaseSpecifier *Base = *I;
177     assert(!Base->isVirtual() && "Should not see virtual bases here!");
178 
179     // Get the layout.
180     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
181 
182     const auto *BaseDecl =
183         cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
184 
185     // Add the offset.
186     Offset += Layout.getBaseClassOffset(BaseDecl);
187 
188     RD = BaseDecl;
189   }
190 
191   return Offset;
192 }
193 
194 llvm::Constant *
195 CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
196                                    CastExpr::path_const_iterator PathBegin,
197                                    CastExpr::path_const_iterator PathEnd) {
198   assert(PathBegin != PathEnd && "Base path should not be empty!");
199 
200   CharUnits Offset =
201       computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
202   if (Offset.isZero())
203     return nullptr;
204 
205   llvm::Type *PtrDiffTy =
206   Types.ConvertType(getContext().getPointerDiffType());
207 
208   return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
209 }
210 
211 /// Gets the address of a direct base class within a complete object.
212 /// This should only be used for (1) non-virtual bases or (2) virtual bases
213 /// when the type is known to be complete (e.g. in complete destructors).
214 ///
215 /// The object pointed to by 'This' is assumed to be non-null.
216 Address
217 CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
218                                                    const CXXRecordDecl *Derived,
219                                                    const CXXRecordDecl *Base,
220                                                    bool BaseIsVirtual) {
221   // 'this' must be a pointer (in some address space) to Derived.
222   assert(This.getElementType() == ConvertType(Derived));
223 
224   // Compute the offset of the virtual base.
225   CharUnits Offset;
226   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
227   if (BaseIsVirtual)
228     Offset = Layout.getVBaseClassOffset(Base);
229   else
230     Offset = Layout.getBaseClassOffset(Base);
231 
232   // Shift and cast down to the base type.
233   // TODO: for complete types, this should be possible with a GEP.
234   Address V = This;
235   if (!Offset.isZero()) {
236     V = Builder.CreateElementBitCast(V, Int8Ty);
237     V = Builder.CreateConstInBoundsByteGEP(V, Offset);
238   }
239   V = Builder.CreateElementBitCast(V, ConvertType(Base));
240 
241   return V;
242 }
243 
244 static Address
245 ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
246                                 CharUnits nonVirtualOffset,
247                                 llvm::Value *virtualOffset,
248                                 const CXXRecordDecl *derivedClass,
249                                 const CXXRecordDecl *nearestVBase) {
250   // Assert that we have something to do.
251   assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
252 
253   // Compute the offset from the static and dynamic components.
254   llvm::Value *baseOffset;
255   if (!nonVirtualOffset.isZero()) {
256     llvm::Type *OffsetType =
257         (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
258          CGF.CGM.getItaniumVTableContext().isRelativeLayout())
259             ? CGF.Int32Ty
260             : CGF.PtrDiffTy;
261     baseOffset =
262         llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity());
263     if (virtualOffset) {
264       baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);
265     }
266   } else {
267     baseOffset = virtualOffset;
268   }
269 
270   // Apply the base offset.
271   llvm::Value *ptr = addr.getPointer();
272   unsigned AddrSpace = ptr->getType()->getPointerAddressSpace();
273   ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8Ty->getPointerTo(AddrSpace));
274   ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr");
275 
276   // If we have a virtual component, the alignment of the result will
277   // be relative only to the known alignment of that vbase.
278   CharUnits alignment;
279   if (virtualOffset) {
280     assert(nearestVBase && "virtual offset without vbase?");
281     alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
282                                           derivedClass, nearestVBase);
283   } else {
284     alignment = addr.getAlignment();
285   }
286   alignment = alignment.alignmentAtOffset(nonVirtualOffset);
287 
288   return Address(ptr, alignment);
289 }
290 
291 Address CodeGenFunction::GetAddressOfBaseClass(
292     Address Value, const CXXRecordDecl *Derived,
293     CastExpr::path_const_iterator PathBegin,
294     CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
295     SourceLocation Loc) {
296   assert(PathBegin != PathEnd && "Base path should not be empty!");
297 
298   CastExpr::path_const_iterator Start = PathBegin;
299   const CXXRecordDecl *VBase = nullptr;
300 
301   // Sema has done some convenient canonicalization here: if the
302   // access path involved any virtual steps, the conversion path will
303   // *start* with a step down to the correct virtual base subobject,
304   // and hence will not require any further steps.
305   if ((*Start)->isVirtual()) {
306     VBase = cast<CXXRecordDecl>(
307         (*Start)->getType()->castAs<RecordType>()->getDecl());
308     ++Start;
309   }
310 
311   // Compute the static offset of the ultimate destination within its
312   // allocating subobject (the virtual base, if there is one, or else
313   // the "complete" object that we see).
314   CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
315       VBase ? VBase : Derived, Start, PathEnd);
316 
317   // If there's a virtual step, we can sometimes "devirtualize" it.
318   // For now, that's limited to when the derived type is final.
319   // TODO: "devirtualize" this for accesses to known-complete objects.
320   if (VBase && Derived->hasAttr<FinalAttr>()) {
321     const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
322     CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
323     NonVirtualOffset += vBaseOffset;
324     VBase = nullptr; // we no longer have a virtual step
325   }
326 
327   // Get the base pointer type.
328   llvm::Type *BasePtrTy =
329       ConvertType((PathEnd[-1])->getType())
330           ->getPointerTo(Value.getType()->getPointerAddressSpace());
331 
332   QualType DerivedTy = getContext().getRecordType(Derived);
333   CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
334 
335   // If the static offset is zero and we don't have a virtual step,
336   // just do a bitcast; null checks are unnecessary.
337   if (NonVirtualOffset.isZero() && !VBase) {
338     if (sanitizePerformTypeCheck()) {
339       SanitizerSet SkippedChecks;
340       SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
341       EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
342                     DerivedTy, DerivedAlign, SkippedChecks);
343     }
344     return Builder.CreateBitCast(Value, BasePtrTy);
345   }
346 
347   llvm::BasicBlock *origBB = nullptr;
348   llvm::BasicBlock *endBB = nullptr;
349 
350   // Skip over the offset (and the vtable load) if we're supposed to
351   // null-check the pointer.
352   if (NullCheckValue) {
353     origBB = Builder.GetInsertBlock();
354     llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
355     endBB = createBasicBlock("cast.end");
356 
357     llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
358     Builder.CreateCondBr(isNull, endBB, notNullBB);
359     EmitBlock(notNullBB);
360   }
361 
362   if (sanitizePerformTypeCheck()) {
363     SanitizerSet SkippedChecks;
364     SkippedChecks.set(SanitizerKind::Null, true);
365     EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
366                   Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
367   }
368 
369   // Compute the virtual offset.
370   llvm::Value *VirtualOffset = nullptr;
371   if (VBase) {
372     VirtualOffset =
373       CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
374   }
375 
376   // Apply both offsets.
377   Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
378                                           VirtualOffset, Derived, VBase);
379 
380   // Cast to the destination type.
381   Value = Builder.CreateBitCast(Value, BasePtrTy);
382 
383   // Build a phi if we needed a null check.
384   if (NullCheckValue) {
385     llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
386     Builder.CreateBr(endBB);
387     EmitBlock(endBB);
388 
389     llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
390     PHI->addIncoming(Value.getPointer(), notNullBB);
391     PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
392     Value = Address(PHI, Value.getAlignment());
393   }
394 
395   return Value;
396 }
397 
398 Address
399 CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
400                                           const CXXRecordDecl *Derived,
401                                         CastExpr::path_const_iterator PathBegin,
402                                           CastExpr::path_const_iterator PathEnd,
403                                           bool NullCheckValue) {
404   assert(PathBegin != PathEnd && "Base path should not be empty!");
405 
406   QualType DerivedTy =
407     getContext().getCanonicalType(getContext().getTagDeclType(Derived));
408   unsigned AddrSpace =
409     BaseAddr.getPointer()->getType()->getPointerAddressSpace();
410   llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo(AddrSpace);
411 
412   llvm::Value *NonVirtualOffset =
413     CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
414 
415   if (!NonVirtualOffset) {
416     // No offset, we can just cast back.
417     return Builder.CreateBitCast(BaseAddr, DerivedPtrTy);
418   }
419 
420   llvm::BasicBlock *CastNull = nullptr;
421   llvm::BasicBlock *CastNotNull = nullptr;
422   llvm::BasicBlock *CastEnd = nullptr;
423 
424   if (NullCheckValue) {
425     CastNull = createBasicBlock("cast.null");
426     CastNotNull = createBasicBlock("cast.notnull");
427     CastEnd = createBasicBlock("cast.end");
428 
429     llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
430     Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
431     EmitBlock(CastNotNull);
432   }
433 
434   // Apply the offset.
435   llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
436   Value = Builder.CreateInBoundsGEP(Value, Builder.CreateNeg(NonVirtualOffset),
437                                     "sub.ptr");
438 
439   // Just cast.
440   Value = Builder.CreateBitCast(Value, DerivedPtrTy);
441 
442   // Produce a PHI if we had a null-check.
443   if (NullCheckValue) {
444     Builder.CreateBr(CastEnd);
445     EmitBlock(CastNull);
446     Builder.CreateBr(CastEnd);
447     EmitBlock(CastEnd);
448 
449     llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
450     PHI->addIncoming(Value, CastNotNull);
451     PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
452     Value = PHI;
453   }
454 
455   return Address(Value, CGM.getClassPointerAlignment(Derived));
456 }
457 
458 llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
459                                               bool ForVirtualBase,
460                                               bool Delegating) {
461   if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
462     // This constructor/destructor does not need a VTT parameter.
463     return nullptr;
464   }
465 
466   const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
467   const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
468 
469   llvm::Value *VTT;
470 
471   uint64_t SubVTTIndex;
472 
473   if (Delegating) {
474     // If this is a delegating constructor call, just load the VTT.
475     return LoadCXXVTT();
476   } else if (RD == Base) {
477     // If the record matches the base, this is the complete ctor/dtor
478     // variant calling the base variant in a class with virtual bases.
479     assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
480            "doing no-op VTT offset in base dtor/ctor?");
481     assert(!ForVirtualBase && "Can't have same class as virtual base!");
482     SubVTTIndex = 0;
483   } else {
484     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
485     CharUnits BaseOffset = ForVirtualBase ?
486       Layout.getVBaseClassOffset(Base) :
487       Layout.getBaseClassOffset(Base);
488 
489     SubVTTIndex =
490       CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
491     assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
492   }
493 
494   if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
495     // A VTT parameter was passed to the constructor, use it.
496     VTT = LoadCXXVTT();
497     VTT = Builder.CreateConstInBoundsGEP1_64(VTT, SubVTTIndex);
498   } else {
499     // We're the complete constructor, so get the VTT by name.
500     VTT = CGM.getVTables().GetAddrOfVTT(RD);
501     VTT = Builder.CreateConstInBoundsGEP2_64(VTT, 0, SubVTTIndex);
502   }
503 
504   return VTT;
505 }
506 
507 namespace {
508   /// Call the destructor for a direct base class.
509   struct CallBaseDtor final : EHScopeStack::Cleanup {
510     const CXXRecordDecl *BaseClass;
511     bool BaseIsVirtual;
512     CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
513       : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
514 
515     void Emit(CodeGenFunction &CGF, Flags flags) override {
516       const CXXRecordDecl *DerivedClass =
517         cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
518 
519       const CXXDestructorDecl *D = BaseClass->getDestructor();
520       // We are already inside a destructor, so presumably the object being
521       // destroyed should have the expected type.
522       QualType ThisTy = D->getThisObjectType();
523       Address Addr =
524         CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
525                                                   DerivedClass, BaseClass,
526                                                   BaseIsVirtual);
527       CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
528                                 /*Delegating=*/false, Addr, ThisTy);
529     }
530   };
531 
532   /// A visitor which checks whether an initializer uses 'this' in a
533   /// way which requires the vtable to be properly set.
534   struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
535     typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
536 
537     bool UsesThis;
538 
539     DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
540 
541     // Black-list all explicit and implicit references to 'this'.
542     //
543     // Do we need to worry about external references to 'this' derived
544     // from arbitrary code?  If so, then anything which runs arbitrary
545     // external code might potentially access the vtable.
546     void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
547   };
548 } // end anonymous namespace
549 
550 static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
551   DynamicThisUseChecker Checker(C);
552   Checker.Visit(Init);
553   return Checker.UsesThis;
554 }
555 
556 static void EmitBaseInitializer(CodeGenFunction &CGF,
557                                 const CXXRecordDecl *ClassDecl,
558                                 CXXCtorInitializer *BaseInit) {
559   assert(BaseInit->isBaseInitializer() &&
560          "Must have base initializer!");
561 
562   Address ThisPtr = CGF.LoadCXXThisAddress();
563 
564   const Type *BaseType = BaseInit->getBaseClass();
565   const auto *BaseClassDecl =
566       cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
567 
568   bool isBaseVirtual = BaseInit->isBaseVirtual();
569 
570   // If the initializer for the base (other than the constructor
571   // itself) accesses 'this' in any way, we need to initialize the
572   // vtables.
573   if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
574     CGF.InitializeVTablePointers(ClassDecl);
575 
576   // We can pretend to be a complete class because it only matters for
577   // virtual bases, and we only do virtual bases for complete ctors.
578   Address V =
579     CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
580                                               BaseClassDecl,
581                                               isBaseVirtual);
582   AggValueSlot AggSlot =
583       AggValueSlot::forAddr(
584           V, Qualifiers(),
585           AggValueSlot::IsDestructed,
586           AggValueSlot::DoesNotNeedGCBarriers,
587           AggValueSlot::IsNotAliased,
588           CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));
589 
590   CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
591 
592   if (CGF.CGM.getLangOpts().Exceptions &&
593       !BaseClassDecl->hasTrivialDestructor())
594     CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
595                                           isBaseVirtual);
596 }
597 
598 static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
599   auto *CD = dyn_cast<CXXConstructorDecl>(D);
600   if (!(CD && CD->isCopyOrMoveConstructor()) &&
601       !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator())
602     return false;
603 
604   // We can emit a memcpy for a trivial copy or move constructor/assignment.
605   if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding())
606     return true;
607 
608   // We *must* emit a memcpy for a defaulted union copy or move op.
609   if (D->getParent()->isUnion() && D->isDefaulted())
610     return true;
611 
612   return false;
613 }
614 
615 static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
616                                                 CXXCtorInitializer *MemberInit,
617                                                 LValue &LHS) {
618   FieldDecl *Field = MemberInit->getAnyMember();
619   if (MemberInit->isIndirectMemberInitializer()) {
620     // If we are initializing an anonymous union field, drill down to the field.
621     IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
622     for (const auto *I : IndirectField->chain())
623       LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
624   } else {
625     LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
626   }
627 }
628 
629 static void EmitMemberInitializer(CodeGenFunction &CGF,
630                                   const CXXRecordDecl *ClassDecl,
631                                   CXXCtorInitializer *MemberInit,
632                                   const CXXConstructorDecl *Constructor,
633                                   FunctionArgList &Args) {
634   ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
635   assert(MemberInit->isAnyMemberInitializer() &&
636          "Must have member initializer!");
637   assert(MemberInit->getInit() && "Must have initializer!");
638 
639   // non-static data member initializers.
640   FieldDecl *Field = MemberInit->getAnyMember();
641   QualType FieldType = Field->getType();
642 
643   llvm::Value *ThisPtr = CGF.LoadCXXThis();
644   QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
645   LValue LHS;
646 
647   // If a base constructor is being emitted, create an LValue that has the
648   // non-virtual alignment.
649   if (CGF.CurGD.getCtorType() == Ctor_Base)
650     LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);
651   else
652     LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
653 
654   EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
655 
656   // Special case: if we are in a copy or move constructor, and we are copying
657   // an array of PODs or classes with trivial copy constructors, ignore the
658   // AST and perform the copy we know is equivalent.
659   // FIXME: This is hacky at best... if we had a bit more explicit information
660   // in the AST, we could generalize it more easily.
661   const ConstantArrayType *Array
662     = CGF.getContext().getAsConstantArrayType(FieldType);
663   if (Array && Constructor->isDefaulted() &&
664       Constructor->isCopyOrMoveConstructor()) {
665     QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
666     CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
667     if (BaseElementTy.isPODType(CGF.getContext()) ||
668         (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) {
669       unsigned SrcArgIndex =
670           CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
671       llvm::Value *SrcPtr
672         = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
673       LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
674       LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
675 
676       // Copy the aggregate.
677       CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),
678                             LHS.isVolatileQualified());
679       // Ensure that we destroy the objects if an exception is thrown later in
680       // the constructor.
681       QualType::DestructionKind dtorKind = FieldType.isDestructedType();
682       if (CGF.needsEHCleanup(dtorKind))
683         CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType);
684       return;
685     }
686   }
687 
688   CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());
689 }
690 
691 void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
692                                               Expr *Init) {
693   QualType FieldType = Field->getType();
694   switch (getEvaluationKind(FieldType)) {
695   case TEK_Scalar:
696     if (LHS.isSimple()) {
697       EmitExprAsInit(Init, Field, LHS, false);
698     } else {
699       RValue RHS = RValue::get(EmitScalarExpr(Init));
700       EmitStoreThroughLValue(RHS, LHS);
701     }
702     break;
703   case TEK_Complex:
704     EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
705     break;
706   case TEK_Aggregate: {
707     AggValueSlot Slot = AggValueSlot::forLValue(
708         LHS, *this, AggValueSlot::IsDestructed,
709         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
710         getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed,
711         // Checks are made by the code that calls constructor.
712         AggValueSlot::IsSanitizerChecked);
713     EmitAggExpr(Init, Slot);
714     break;
715   }
716   }
717 
718   // Ensure that we destroy this object if an exception is thrown
719   // later in the constructor.
720   QualType::DestructionKind dtorKind = FieldType.isDestructedType();
721   if (needsEHCleanup(dtorKind))
722     pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType);
723 }
724 
725 /// Checks whether the given constructor is a valid subject for the
726 /// complete-to-base constructor delegation optimization, i.e.
727 /// emitting the complete constructor as a simple call to the base
728 /// constructor.
729 bool CodeGenFunction::IsConstructorDelegationValid(
730     const CXXConstructorDecl *Ctor) {
731 
732   // Currently we disable the optimization for classes with virtual
733   // bases because (1) the addresses of parameter variables need to be
734   // consistent across all initializers but (2) the delegate function
735   // call necessarily creates a second copy of the parameter variable.
736   //
737   // The limiting example (purely theoretical AFAIK):
738   //   struct A { A(int &c) { c++; } };
739   //   struct B : virtual A {
740   //     B(int count) : A(count) { printf("%d\n", count); }
741   //   };
742   // ...although even this example could in principle be emitted as a
743   // delegation since the address of the parameter doesn't escape.
744   if (Ctor->getParent()->getNumVBases()) {
745     // TODO: white-list trivial vbase initializers.  This case wouldn't
746     // be subject to the restrictions below.
747 
748     // TODO: white-list cases where:
749     //  - there are no non-reference parameters to the constructor
750     //  - the initializers don't access any non-reference parameters
751     //  - the initializers don't take the address of non-reference
752     //    parameters
753     //  - etc.
754     // If we ever add any of the above cases, remember that:
755     //  - function-try-blocks will always exclude this optimization
756     //  - we need to perform the constructor prologue and cleanup in
757     //    EmitConstructorBody.
758 
759     return false;
760   }
761 
762   // We also disable the optimization for variadic functions because
763   // it's impossible to "re-pass" varargs.
764   if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
765     return false;
766 
767   // FIXME: Decide if we can do a delegation of a delegating constructor.
768   if (Ctor->isDelegatingConstructor())
769     return false;
770 
771   return true;
772 }
773 
774 // Emit code in ctor (Prologue==true) or dtor (Prologue==false)
775 // to poison the extra field paddings inserted under
776 // -fsanitize-address-field-padding=1|2.
777 void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
778   ASTContext &Context = getContext();
779   const CXXRecordDecl *ClassDecl =
780       Prologue ? cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()
781                : cast<CXXDestructorDecl>(CurGD.getDecl())->getParent();
782   if (!ClassDecl->mayInsertExtraPadding()) return;
783 
784   struct SizeAndOffset {
785     uint64_t Size;
786     uint64_t Offset;
787   };
788 
789   unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
790   const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
791 
792   // Populate sizes and offsets of fields.
793   SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
794   for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)
795     SSV[i].Offset =
796         Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();
797 
798   size_t NumFields = 0;
799   for (const auto *Field : ClassDecl->fields()) {
800     const FieldDecl *D = Field;
801     std::pair<CharUnits, CharUnits> FieldInfo =
802         Context.getTypeInfoInChars(D->getType());
803     CharUnits FieldSize = FieldInfo.first;
804     assert(NumFields < SSV.size());
805     SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();
806     NumFields++;
807   }
808   assert(NumFields == SSV.size());
809   if (SSV.size() <= 1) return;
810 
811   // We will insert calls to __asan_* run-time functions.
812   // LLVM AddressSanitizer pass may decide to inline them later.
813   llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
814   llvm::FunctionType *FTy =
815       llvm::FunctionType::get(CGM.VoidTy, Args, false);
816   llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
817       FTy, Prologue ? "__asan_poison_intra_object_redzone"
818                     : "__asan_unpoison_intra_object_redzone");
819 
820   llvm::Value *ThisPtr = LoadCXXThis();
821   ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);
822   uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
823   // For each field check if it has sufficient padding,
824   // if so (un)poison it with a call.
825   for (size_t i = 0; i < SSV.size(); i++) {
826     uint64_t AsanAlignment = 8;
827     uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;
828     uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
829     uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
830     if (PoisonSize < AsanAlignment || !SSV[i].Size ||
831         (NextField % AsanAlignment) != 0)
832       continue;
833     Builder.CreateCall(
834         F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
835             Builder.getIntN(PtrSize, PoisonSize)});
836   }
837 }
838 
839 /// EmitConstructorBody - Emits the body of the current constructor.
840 void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
841   EmitAsanPrologueOrEpilogue(true);
842   const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
843   CXXCtorType CtorType = CurGD.getCtorType();
844 
845   assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
846           CtorType == Ctor_Complete) &&
847          "can only generate complete ctor for this ABI");
848 
849   // Before we go any further, try the complete->base constructor
850   // delegation optimization.
851   if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&
852       CGM.getTarget().getCXXABI().hasConstructorVariants()) {
853     EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());
854     return;
855   }
856 
857   const FunctionDecl *Definition = nullptr;
858   Stmt *Body = Ctor->getBody(Definition);
859   assert(Definition == Ctor && "emitting wrong constructor body");
860 
861   // Enter the function-try-block before the constructor prologue if
862   // applicable.
863   bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
864   if (IsTryBody)
865     EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
866 
867   incrementProfileCounter(Body);
868 
869   RunCleanupsScope RunCleanups(*this);
870 
871   // TODO: in restricted cases, we can emit the vbase initializers of
872   // a complete ctor and then delegate to the base ctor.
873 
874   // Emit the constructor prologue, i.e. the base and member
875   // initializers.
876   EmitCtorPrologue(Ctor, CtorType, Args);
877 
878   // Emit the body of the statement.
879   if (IsTryBody)
880     EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
881   else if (Body)
882     EmitStmt(Body);
883 
884   // Emit any cleanup blocks associated with the member or base
885   // initializers, which includes (along the exceptional path) the
886   // destructors for those members and bases that were fully
887   // constructed.
888   RunCleanups.ForceCleanup();
889 
890   if (IsTryBody)
891     ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
892 }
893 
894 namespace {
895   /// RAII object to indicate that codegen is copying the value representation
896   /// instead of the object representation. Useful when copying a struct or
897   /// class which has uninitialized members and we're only performing
898   /// lvalue-to-rvalue conversion on the object but not its members.
899   class CopyingValueRepresentation {
900   public:
901     explicit CopyingValueRepresentation(CodeGenFunction &CGF)
902         : CGF(CGF), OldSanOpts(CGF.SanOpts) {
903       CGF.SanOpts.set(SanitizerKind::Bool, false);
904       CGF.SanOpts.set(SanitizerKind::Enum, false);
905     }
906     ~CopyingValueRepresentation() {
907       CGF.SanOpts = OldSanOpts;
908     }
909   private:
910     CodeGenFunction &CGF;
911     SanitizerSet OldSanOpts;
912   };
913 } // end anonymous namespace
914 
915 namespace {
916   class FieldMemcpyizer {
917   public:
918     FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
919                     const VarDecl *SrcRec)
920       : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
921         RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
922         FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
923         LastFieldOffset(0), LastAddedFieldIndex(0) {}
924 
925     bool isMemcpyableField(FieldDecl *F) const {
926       // Never memcpy fields when we are adding poisoned paddings.
927       if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
928         return false;
929       Qualifiers Qual = F->getType().getQualifiers();
930       if (Qual.hasVolatile() || Qual.hasObjCLifetime())
931         return false;
932       return true;
933     }
934 
935     void addMemcpyableField(FieldDecl *F) {
936       if (F->isZeroSize(CGF.getContext()))
937         return;
938       if (!FirstField)
939         addInitialField(F);
940       else
941         addNextField(F);
942     }
943 
944     CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
945       ASTContext &Ctx = CGF.getContext();
946       unsigned LastFieldSize =
947           LastField->isBitField()
948               ? LastField->getBitWidthValue(Ctx)
949               : Ctx.toBits(
950                     Ctx.getTypeInfoDataSizeInChars(LastField->getType()).first);
951       uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
952                                 FirstByteOffset + Ctx.getCharWidth() - 1;
953       CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);
954       return MemcpySize;
955     }
956 
957     void emitMemcpy() {
958       // Give the subclass a chance to bail out if it feels the memcpy isn't
959       // worth it (e.g. Hasn't aggregated enough data).
960       if (!FirstField) {
961         return;
962       }
963 
964       uint64_t FirstByteOffset;
965       if (FirstField->isBitField()) {
966         const CGRecordLayout &RL =
967           CGF.getTypes().getCGRecordLayout(FirstField->getParent());
968         const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
969         // FirstFieldOffset is not appropriate for bitfields,
970         // we need to use the storage offset instead.
971         FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
972       } else {
973         FirstByteOffset = FirstFieldOffset;
974       }
975 
976       CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
977       QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
978       Address ThisPtr = CGF.LoadCXXThisAddress();
979       LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
980       LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
981       llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
982       LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
983       LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
984 
985       emitMemcpyIR(
986           Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(CGF),
987           Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(CGF),
988           MemcpySize);
989       reset();
990     }
991 
992     void reset() {
993       FirstField = nullptr;
994     }
995 
996   protected:
997     CodeGenFunction &CGF;
998     const CXXRecordDecl *ClassDecl;
999 
1000   private:
1001     void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
1002       llvm::PointerType *DPT = DestPtr.getType();
1003       llvm::Type *DBP =
1004         llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace());
1005       DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP);
1006 
1007       llvm::PointerType *SPT = SrcPtr.getType();
1008       llvm::Type *SBP =
1009         llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace());
1010       SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP);
1011 
1012       CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
1013     }
1014 
1015     void addInitialField(FieldDecl *F) {
1016       FirstField = F;
1017       LastField = F;
1018       FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1019       LastFieldOffset = FirstFieldOffset;
1020       LastAddedFieldIndex = F->getFieldIndex();
1021     }
1022 
1023     void addNextField(FieldDecl *F) {
1024       // For the most part, the following invariant will hold:
1025       //   F->getFieldIndex() == LastAddedFieldIndex + 1
1026       // The one exception is that Sema won't add a copy-initializer for an
1027       // unnamed bitfield, which will show up here as a gap in the sequence.
1028       assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1029              "Cannot aggregate fields out of order.");
1030       LastAddedFieldIndex = F->getFieldIndex();
1031 
1032       // The 'first' and 'last' fields are chosen by offset, rather than field
1033       // index. This allows the code to support bitfields, as well as regular
1034       // fields.
1035       uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1036       if (FOffset < FirstFieldOffset) {
1037         FirstField = F;
1038         FirstFieldOffset = FOffset;
1039       } else if (FOffset >= LastFieldOffset) {
1040         LastField = F;
1041         LastFieldOffset = FOffset;
1042       }
1043     }
1044 
1045     const VarDecl *SrcRec;
1046     const ASTRecordLayout &RecLayout;
1047     FieldDecl *FirstField;
1048     FieldDecl *LastField;
1049     uint64_t FirstFieldOffset, LastFieldOffset;
1050     unsigned LastAddedFieldIndex;
1051   };
1052 
1053   class ConstructorMemcpyizer : public FieldMemcpyizer {
1054   private:
1055     /// Get source argument for copy constructor. Returns null if not a copy
1056     /// constructor.
1057     static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1058                                                const CXXConstructorDecl *CD,
1059                                                FunctionArgList &Args) {
1060       if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())
1061         return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1062       return nullptr;
1063     }
1064 
1065     // Returns true if a CXXCtorInitializer represents a member initialization
1066     // that can be rolled into a memcpy.
1067     bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1068       if (!MemcpyableCtor)
1069         return false;
1070       FieldDecl *Field = MemberInit->getMember();
1071       assert(Field && "No field for member init.");
1072       QualType FieldType = Field->getType();
1073       CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
1074 
1075       // Bail out on non-memcpyable, not-trivially-copyable members.
1076       if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) &&
1077           !(FieldType.isTriviallyCopyableType(CGF.getContext()) ||
1078             FieldType->isReferenceType()))
1079         return false;
1080 
1081       // Bail out on volatile fields.
1082       if (!isMemcpyableField(Field))
1083         return false;
1084 
1085       // Otherwise we're good.
1086       return true;
1087     }
1088 
1089   public:
1090     ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1091                           FunctionArgList &Args)
1092       : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1093         ConstructorDecl(CD),
1094         MemcpyableCtor(CD->isDefaulted() &&
1095                        CD->isCopyOrMoveConstructor() &&
1096                        CGF.getLangOpts().getGC() == LangOptions::NonGC),
1097         Args(Args) { }
1098 
1099     void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1100       if (isMemberInitMemcpyable(MemberInit)) {
1101         AggregatedInits.push_back(MemberInit);
1102         addMemcpyableField(MemberInit->getMember());
1103       } else {
1104         emitAggregatedInits();
1105         EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1106                               ConstructorDecl, Args);
1107       }
1108     }
1109 
1110     void emitAggregatedInits() {
1111       if (AggregatedInits.size() <= 1) {
1112         // This memcpy is too small to be worthwhile. Fall back on default
1113         // codegen.
1114         if (!AggregatedInits.empty()) {
1115           CopyingValueRepresentation CVR(CGF);
1116           EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1117                                 AggregatedInits[0], ConstructorDecl, Args);
1118           AggregatedInits.clear();
1119         }
1120         reset();
1121         return;
1122       }
1123 
1124       pushEHDestructors();
1125       emitMemcpy();
1126       AggregatedInits.clear();
1127     }
1128 
1129     void pushEHDestructors() {
1130       Address ThisPtr = CGF.LoadCXXThisAddress();
1131       QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1132       LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
1133 
1134       for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
1135         CXXCtorInitializer *MemberInit = AggregatedInits[i];
1136         QualType FieldType = MemberInit->getAnyMember()->getType();
1137         QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1138         if (!CGF.needsEHCleanup(dtorKind))
1139           continue;
1140         LValue FieldLHS = LHS;
1141         EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
1142         CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType);
1143       }
1144     }
1145 
1146     void finish() {
1147       emitAggregatedInits();
1148     }
1149 
1150   private:
1151     const CXXConstructorDecl *ConstructorDecl;
1152     bool MemcpyableCtor;
1153     FunctionArgList &Args;
1154     SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1155   };
1156 
1157   class AssignmentMemcpyizer : public FieldMemcpyizer {
1158   private:
1159     // Returns the memcpyable field copied by the given statement, if one
1160     // exists. Otherwise returns null.
1161     FieldDecl *getMemcpyableField(Stmt *S) {
1162       if (!AssignmentsMemcpyable)
1163         return nullptr;
1164       if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
1165         // Recognise trivial assignments.
1166         if (BO->getOpcode() != BO_Assign)
1167           return nullptr;
1168         MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());
1169         if (!ME)
1170           return nullptr;
1171         FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1172         if (!Field || !isMemcpyableField(Field))
1173           return nullptr;
1174         Stmt *RHS = BO->getRHS();
1175         if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))
1176           RHS = EC->getSubExpr();
1177         if (!RHS)
1178           return nullptr;
1179         if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {
1180           if (ME2->getMemberDecl() == Field)
1181             return Field;
1182         }
1183         return nullptr;
1184       } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
1185         CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1186         if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
1187           return nullptr;
1188         MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
1189         if (!IOA)
1190           return nullptr;
1191         FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());
1192         if (!Field || !isMemcpyableField(Field))
1193           return nullptr;
1194         MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1195         if (!Arg0 || Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl()))
1196           return nullptr;
1197         return Field;
1198       } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
1199         FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());
1200         if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1201           return nullptr;
1202         Expr *DstPtr = CE->getArg(0);
1203         if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))
1204           DstPtr = DC->getSubExpr();
1205         UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);
1206         if (!DUO || DUO->getOpcode() != UO_AddrOf)
1207           return nullptr;
1208         MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());
1209         if (!ME)
1210           return nullptr;
1211         FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1212         if (!Field || !isMemcpyableField(Field))
1213           return nullptr;
1214         Expr *SrcPtr = CE->getArg(1);
1215         if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))
1216           SrcPtr = SC->getSubExpr();
1217         UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);
1218         if (!SUO || SUO->getOpcode() != UO_AddrOf)
1219           return nullptr;
1220         MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());
1221         if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))
1222           return nullptr;
1223         return Field;
1224       }
1225 
1226       return nullptr;
1227     }
1228 
1229     bool AssignmentsMemcpyable;
1230     SmallVector<Stmt*, 16> AggregatedStmts;
1231 
1232   public:
1233     AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1234                          FunctionArgList &Args)
1235       : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1236         AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1237       assert(Args.size() == 2);
1238     }
1239 
1240     void emitAssignment(Stmt *S) {
1241       FieldDecl *F = getMemcpyableField(S);
1242       if (F) {
1243         addMemcpyableField(F);
1244         AggregatedStmts.push_back(S);
1245       } else {
1246         emitAggregatedStmts();
1247         CGF.EmitStmt(S);
1248       }
1249     }
1250 
1251     void emitAggregatedStmts() {
1252       if (AggregatedStmts.size() <= 1) {
1253         if (!AggregatedStmts.empty()) {
1254           CopyingValueRepresentation CVR(CGF);
1255           CGF.EmitStmt(AggregatedStmts[0]);
1256         }
1257         reset();
1258       }
1259 
1260       emitMemcpy();
1261       AggregatedStmts.clear();
1262     }
1263 
1264     void finish() {
1265       emitAggregatedStmts();
1266     }
1267   };
1268 } // end anonymous namespace
1269 
1270 static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1271   const Type *BaseType = BaseInit->getBaseClass();
1272   const auto *BaseClassDecl =
1273       cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
1274   return BaseClassDecl->isDynamicClass();
1275 }
1276 
1277 /// EmitCtorPrologue - This routine generates necessary code to initialize
1278 /// base classes and non-static data members belonging to this constructor.
1279 void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1280                                        CXXCtorType CtorType,
1281                                        FunctionArgList &Args) {
1282   if (CD->isDelegatingConstructor())
1283     return EmitDelegatingCXXConstructorCall(CD, Args);
1284 
1285   const CXXRecordDecl *ClassDecl = CD->getParent();
1286 
1287   CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1288                                           E = CD->init_end();
1289 
1290   // Virtual base initializers first, if any. They aren't needed if:
1291   // - This is a base ctor variant
1292   // - There are no vbases
1293   // - The class is abstract, so a complete object of it cannot be constructed
1294   //
1295   // The check for an abstract class is necessary because sema may not have
1296   // marked virtual base destructors referenced.
1297   bool ConstructVBases = CtorType != Ctor_Base &&
1298                          ClassDecl->getNumVBases() != 0 &&
1299                          !ClassDecl->isAbstract();
1300 
1301   // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1302   // constructor of a class with virtual bases takes an additional parameter to
1303   // conditionally construct the virtual bases. Emit that check here.
1304   llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1305   if (ConstructVBases &&
1306       !CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1307     BaseCtorContinueBB =
1308         CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);
1309     assert(BaseCtorContinueBB);
1310   }
1311 
1312   llvm::Value *const OldThis = CXXThisValue;
1313   for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) {
1314     if (!ConstructVBases)
1315       continue;
1316     if (CGM.getCodeGenOpts().StrictVTablePointers &&
1317         CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1318         isInitializerOfDynamicClass(*B))
1319       CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1320     EmitBaseInitializer(*this, ClassDecl, *B);
1321   }
1322 
1323   if (BaseCtorContinueBB) {
1324     // Complete object handler should continue to the remaining initializers.
1325     Builder.CreateBr(BaseCtorContinueBB);
1326     EmitBlock(BaseCtorContinueBB);
1327   }
1328 
1329   // Then, non-virtual base initializers.
1330   for (; B != E && (*B)->isBaseInitializer(); B++) {
1331     assert(!(*B)->isBaseVirtual());
1332 
1333     if (CGM.getCodeGenOpts().StrictVTablePointers &&
1334         CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1335         isInitializerOfDynamicClass(*B))
1336       CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1337     EmitBaseInitializer(*this, ClassDecl, *B);
1338   }
1339 
1340   CXXThisValue = OldThis;
1341 
1342   InitializeVTablePointers(ClassDecl);
1343 
1344   // And finally, initialize class members.
1345   FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1346   ConstructorMemcpyizer CM(*this, CD, Args);
1347   for (; B != E; B++) {
1348     CXXCtorInitializer *Member = (*B);
1349     assert(!Member->isBaseInitializer());
1350     assert(Member->isAnyMemberInitializer() &&
1351            "Delegating initializer on non-delegating constructor");
1352     CM.addMemberInitializer(Member);
1353   }
1354   CM.finish();
1355 }
1356 
1357 static bool
1358 FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1359 
1360 static bool
1361 HasTrivialDestructorBody(ASTContext &Context,
1362                          const CXXRecordDecl *BaseClassDecl,
1363                          const CXXRecordDecl *MostDerivedClassDecl)
1364 {
1365   // If the destructor is trivial we don't have to check anything else.
1366   if (BaseClassDecl->hasTrivialDestructor())
1367     return true;
1368 
1369   if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1370     return false;
1371 
1372   // Check fields.
1373   for (const auto *Field : BaseClassDecl->fields())
1374     if (!FieldHasTrivialDestructorBody(Context, Field))
1375       return false;
1376 
1377   // Check non-virtual bases.
1378   for (const auto &I : BaseClassDecl->bases()) {
1379     if (I.isVirtual())
1380       continue;
1381 
1382     const CXXRecordDecl *NonVirtualBase =
1383       cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1384     if (!HasTrivialDestructorBody(Context, NonVirtualBase,
1385                                   MostDerivedClassDecl))
1386       return false;
1387   }
1388 
1389   if (BaseClassDecl == MostDerivedClassDecl) {
1390     // Check virtual bases.
1391     for (const auto &I : BaseClassDecl->vbases()) {
1392       const CXXRecordDecl *VirtualBase =
1393         cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1394       if (!HasTrivialDestructorBody(Context, VirtualBase,
1395                                     MostDerivedClassDecl))
1396         return false;
1397     }
1398   }
1399 
1400   return true;
1401 }
1402 
1403 static bool
1404 FieldHasTrivialDestructorBody(ASTContext &Context,
1405                                           const FieldDecl *Field)
1406 {
1407   QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1408 
1409   const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1410   if (!RT)
1411     return true;
1412 
1413   CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
1414 
1415   // The destructor for an implicit anonymous union member is never invoked.
1416   if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
1417     return false;
1418 
1419   return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
1420 }
1421 
1422 /// CanSkipVTablePointerInitialization - Check whether we need to initialize
1423 /// any vtable pointers before calling this destructor.
1424 static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1425                                                const CXXDestructorDecl *Dtor) {
1426   const CXXRecordDecl *ClassDecl = Dtor->getParent();
1427   if (!ClassDecl->isDynamicClass())
1428     return true;
1429 
1430   if (!Dtor->hasTrivialBody())
1431     return false;
1432 
1433   // Check the fields.
1434   for (const auto *Field : ClassDecl->fields())
1435     if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1436       return false;
1437 
1438   return true;
1439 }
1440 
1441 /// EmitDestructorBody - Emits the body of the current destructor.
1442 void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1443   const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
1444   CXXDtorType DtorType = CurGD.getDtorType();
1445 
1446   // For an abstract class, non-base destructors are never used (and can't
1447   // be emitted in general, because vbase dtors may not have been validated
1448   // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1449   // in fact emit references to them from other compilations, so emit them
1450   // as functions containing a trap instruction.
1451   if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {
1452     llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1453     TrapCall->setDoesNotReturn();
1454     TrapCall->setDoesNotThrow();
1455     Builder.CreateUnreachable();
1456     Builder.ClearInsertionPoint();
1457     return;
1458   }
1459 
1460   Stmt *Body = Dtor->getBody();
1461   if (Body)
1462     incrementProfileCounter(Body);
1463 
1464   // The call to operator delete in a deleting destructor happens
1465   // outside of the function-try-block, which means it's always
1466   // possible to delegate the destructor body to the complete
1467   // destructor.  Do so.
1468   if (DtorType == Dtor_Deleting) {
1469     RunCleanupsScope DtorEpilogue(*this);
1470     EnterDtorCleanups(Dtor, Dtor_Deleting);
1471     if (HaveInsertPoint()) {
1472       QualType ThisTy = Dtor->getThisObjectType();
1473       EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
1474                             /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1475     }
1476     return;
1477   }
1478 
1479   // If the body is a function-try-block, enter the try before
1480   // anything else.
1481   bool isTryBody = (Body && isa<CXXTryStmt>(Body));
1482   if (isTryBody)
1483     EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1484   EmitAsanPrologueOrEpilogue(false);
1485 
1486   // Enter the epilogue cleanups.
1487   RunCleanupsScope DtorEpilogue(*this);
1488 
1489   // If this is the complete variant, just invoke the base variant;
1490   // the epilogue will destruct the virtual bases.  But we can't do
1491   // this optimization if the body is a function-try-block, because
1492   // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
1493   // always delegate because we might not have a definition in this TU.
1494   switch (DtorType) {
1495   case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1496   case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1497 
1498   case Dtor_Complete:
1499     assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1500            "can't emit a dtor without a body for non-Microsoft ABIs");
1501 
1502     // Enter the cleanup scopes for virtual bases.
1503     EnterDtorCleanups(Dtor, Dtor_Complete);
1504 
1505     if (!isTryBody) {
1506       QualType ThisTy = Dtor->getThisObjectType();
1507       EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
1508                             /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1509       break;
1510     }
1511 
1512     // Fallthrough: act like we're in the base variant.
1513     LLVM_FALLTHROUGH;
1514 
1515   case Dtor_Base:
1516     assert(Body);
1517 
1518     // Enter the cleanup scopes for fields and non-virtual bases.
1519     EnterDtorCleanups(Dtor, Dtor_Base);
1520 
1521     // Initialize the vtable pointers before entering the body.
1522     if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
1523       // Insert the llvm.launder.invariant.group intrinsic before initializing
1524       // the vptrs to cancel any previous assumptions we might have made.
1525       if (CGM.getCodeGenOpts().StrictVTablePointers &&
1526           CGM.getCodeGenOpts().OptimizationLevel > 0)
1527         CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1528       InitializeVTablePointers(Dtor->getParent());
1529     }
1530 
1531     if (isTryBody)
1532       EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
1533     else if (Body)
1534       EmitStmt(Body);
1535     else {
1536       assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1537       // nothing to do besides what's in the epilogue
1538     }
1539     // -fapple-kext must inline any call to this dtor into
1540     // the caller's body.
1541     if (getLangOpts().AppleKext)
1542       CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1543 
1544     break;
1545   }
1546 
1547   // Jump out through the epilogue cleanups.
1548   DtorEpilogue.ForceCleanup();
1549 
1550   // Exit the try if applicable.
1551   if (isTryBody)
1552     ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1553 }
1554 
1555 void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1556   const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());
1557   const Stmt *RootS = AssignOp->getBody();
1558   assert(isa<CompoundStmt>(RootS) &&
1559          "Body of an implicit assignment operator should be compound stmt.");
1560   const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);
1561 
1562   LexicalScope Scope(*this, RootCS->getSourceRange());
1563 
1564   incrementProfileCounter(RootCS);
1565   AssignmentMemcpyizer AM(*this, AssignOp, Args);
1566   for (auto *I : RootCS->body())
1567     AM.emitAssignment(I);
1568   AM.finish();
1569 }
1570 
1571 namespace {
1572   llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1573                                      const CXXDestructorDecl *DD) {
1574     if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1575       return CGF.EmitScalarExpr(ThisArg);
1576     return CGF.LoadCXXThis();
1577   }
1578 
1579   /// Call the operator delete associated with the current destructor.
1580   struct CallDtorDelete final : EHScopeStack::Cleanup {
1581     CallDtorDelete() {}
1582 
1583     void Emit(CodeGenFunction &CGF, Flags flags) override {
1584       const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1585       const CXXRecordDecl *ClassDecl = Dtor->getParent();
1586       CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1587                          LoadThisForDtorDelete(CGF, Dtor),
1588                          CGF.getContext().getTagDeclType(ClassDecl));
1589     }
1590   };
1591 
1592   void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1593                                      llvm::Value *ShouldDeleteCondition,
1594                                      bool ReturnAfterDelete) {
1595     llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");
1596     llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");
1597     llvm::Value *ShouldCallDelete
1598       = CGF.Builder.CreateIsNull(ShouldDeleteCondition);
1599     CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);
1600 
1601     CGF.EmitBlock(callDeleteBB);
1602     const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1603     const CXXRecordDecl *ClassDecl = Dtor->getParent();
1604     CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1605                        LoadThisForDtorDelete(CGF, Dtor),
1606                        CGF.getContext().getTagDeclType(ClassDecl));
1607     assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1608                ReturnAfterDelete &&
1609            "unexpected value for ReturnAfterDelete");
1610     if (ReturnAfterDelete)
1611       CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
1612     else
1613       CGF.Builder.CreateBr(continueBB);
1614 
1615     CGF.EmitBlock(continueBB);
1616   }
1617 
1618   struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1619     llvm::Value *ShouldDeleteCondition;
1620 
1621   public:
1622     CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1623         : ShouldDeleteCondition(ShouldDeleteCondition) {
1624       assert(ShouldDeleteCondition != nullptr);
1625     }
1626 
1627     void Emit(CodeGenFunction &CGF, Flags flags) override {
1628       EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1629                                     /*ReturnAfterDelete*/false);
1630     }
1631   };
1632 
1633   class DestroyField  final : public EHScopeStack::Cleanup {
1634     const FieldDecl *field;
1635     CodeGenFunction::Destroyer *destroyer;
1636     bool useEHCleanupForArray;
1637 
1638   public:
1639     DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1640                  bool useEHCleanupForArray)
1641         : field(field), destroyer(destroyer),
1642           useEHCleanupForArray(useEHCleanupForArray) {}
1643 
1644     void Emit(CodeGenFunction &CGF, Flags flags) override {
1645       // Find the address of the field.
1646       Address thisValue = CGF.LoadCXXThisAddress();
1647       QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1648       LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
1649       LValue LV = CGF.EmitLValueForField(ThisLV, field);
1650       assert(LV.isSimple());
1651 
1652       CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer,
1653                       flags.isForNormalCleanup() && useEHCleanupForArray);
1654     }
1655   };
1656 
1657  static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1658              CharUnits::QuantityType PoisonSize) {
1659    CodeGenFunction::SanitizerScope SanScope(&CGF);
1660    // Pass in void pointer and size of region as arguments to runtime
1661    // function
1662    llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
1663                           llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
1664 
1665    llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
1666 
1667    llvm::FunctionType *FnType =
1668        llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
1669    llvm::FunctionCallee Fn =
1670        CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
1671    CGF.EmitNounwindRuntimeCall(Fn, Args);
1672  }
1673 
1674   class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
1675     const CXXDestructorDecl *Dtor;
1676 
1677   public:
1678     SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1679 
1680     // Generate function call for handling object poisoning.
1681     // Disables tail call elimination, to prevent the current stack frame
1682     // from disappearing from the stack trace.
1683     void Emit(CodeGenFunction &CGF, Flags flags) override {
1684       const ASTRecordLayout &Layout =
1685           CGF.getContext().getASTRecordLayout(Dtor->getParent());
1686 
1687       // Nothing to poison.
1688       if (Layout.getFieldCount() == 0)
1689         return;
1690 
1691       // Prevent the current stack frame from disappearing from the stack trace.
1692       CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1693 
1694       // Construct pointer to region to begin poisoning, and calculate poison
1695       // size, so that only members declared in this class are poisoned.
1696       ASTContext &Context = CGF.getContext();
1697       unsigned fieldIndex = 0;
1698       int startIndex = -1;
1699       // RecordDecl::field_iterator Field;
1700       for (const FieldDecl *Field : Dtor->getParent()->fields()) {
1701         // Poison field if it is trivial
1702         if (FieldHasTrivialDestructorBody(Context, Field)) {
1703           // Start sanitizing at this field
1704           if (startIndex < 0)
1705             startIndex = fieldIndex;
1706 
1707           // Currently on the last field, and it must be poisoned with the
1708           // current block.
1709           if (fieldIndex == Layout.getFieldCount() - 1) {
1710             PoisonMembers(CGF, startIndex, Layout.getFieldCount());
1711           }
1712         } else if (startIndex >= 0) {
1713           // No longer within a block of memory to poison, so poison the block
1714           PoisonMembers(CGF, startIndex, fieldIndex);
1715           // Re-set the start index
1716           startIndex = -1;
1717         }
1718         fieldIndex += 1;
1719       }
1720     }
1721 
1722   private:
1723     /// \param layoutStartOffset index of the ASTRecordLayout field to
1724     ///     start poisoning (inclusive)
1725     /// \param layoutEndOffset index of the ASTRecordLayout field to
1726     ///     end poisoning (exclusive)
1727     void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
1728                      unsigned layoutEndOffset) {
1729       ASTContext &Context = CGF.getContext();
1730       const ASTRecordLayout &Layout =
1731           Context.getASTRecordLayout(Dtor->getParent());
1732 
1733       llvm::ConstantInt *OffsetSizePtr = llvm::ConstantInt::get(
1734           CGF.SizeTy,
1735           Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset))
1736               .getQuantity());
1737 
1738       llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
1739           CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
1740           OffsetSizePtr);
1741 
1742       CharUnits::QuantityType PoisonSize;
1743       if (layoutEndOffset >= Layout.getFieldCount()) {
1744         PoisonSize = Layout.getNonVirtualSize().getQuantity() -
1745                      Context.toCharUnitsFromBits(
1746                                 Layout.getFieldOffset(layoutStartOffset))
1747                          .getQuantity();
1748       } else {
1749         PoisonSize = Context.toCharUnitsFromBits(
1750                                 Layout.getFieldOffset(layoutEndOffset) -
1751                                 Layout.getFieldOffset(layoutStartOffset))
1752                          .getQuantity();
1753       }
1754 
1755       if (PoisonSize == 0)
1756         return;
1757 
1758       EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize);
1759     }
1760   };
1761 
1762  class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1763     const CXXDestructorDecl *Dtor;
1764 
1765   public:
1766     SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1767 
1768     // Generate function call for handling vtable pointer poisoning.
1769     void Emit(CodeGenFunction &CGF, Flags flags) override {
1770       assert(Dtor->getParent()->isDynamicClass());
1771       (void)Dtor;
1772       ASTContext &Context = CGF.getContext();
1773       // Poison vtable and vtable ptr if they exist for this class.
1774       llvm::Value *VTablePtr = CGF.LoadCXXThis();
1775 
1776       CharUnits::QuantityType PoisonSize =
1777           Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
1778       // Pass in void pointer and size of region as arguments to runtime
1779       // function
1780       EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
1781     }
1782  };
1783 } // end anonymous namespace
1784 
1785 /// Emit all code that comes at the end of class's
1786 /// destructor. This is to call destructors on members and base classes
1787 /// in reverse order of their construction.
1788 ///
1789 /// For a deleting destructor, this also handles the case where a destroying
1790 /// operator delete completely overrides the definition.
1791 void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1792                                         CXXDtorType DtorType) {
1793   assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1794          "Should not emit dtor epilogue for non-exported trivial dtor!");
1795 
1796   // The deleting-destructor phase just needs to call the appropriate
1797   // operator delete that Sema picked up.
1798   if (DtorType == Dtor_Deleting) {
1799     assert(DD->getOperatorDelete() &&
1800            "operator delete missing - EnterDtorCleanups");
1801     if (CXXStructorImplicitParamValue) {
1802       // If there is an implicit param to the deleting dtor, it's a boolean
1803       // telling whether this is a deleting destructor.
1804       if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1805         EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,
1806                                       /*ReturnAfterDelete*/true);
1807       else
1808         EHStack.pushCleanup<CallDtorDeleteConditional>(
1809             NormalAndEHCleanup, CXXStructorImplicitParamValue);
1810     } else {
1811       if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1812         const CXXRecordDecl *ClassDecl = DD->getParent();
1813         EmitDeleteCall(DD->getOperatorDelete(),
1814                        LoadThisForDtorDelete(*this, DD),
1815                        getContext().getTagDeclType(ClassDecl));
1816         EmitBranchThroughCleanup(ReturnBlock);
1817       } else {
1818         EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
1819       }
1820     }
1821     return;
1822   }
1823 
1824   const CXXRecordDecl *ClassDecl = DD->getParent();
1825 
1826   // Unions have no bases and do not call field destructors.
1827   if (ClassDecl->isUnion())
1828     return;
1829 
1830   // The complete-destructor phase just destructs all the virtual bases.
1831   if (DtorType == Dtor_Complete) {
1832     // Poison the vtable pointer such that access after the base
1833     // and member destructors are invoked is invalid.
1834     if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1835         SanOpts.has(SanitizerKind::Memory) && ClassDecl->getNumVBases() &&
1836         ClassDecl->isPolymorphic())
1837       EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1838 
1839     // We push them in the forward order so that they'll be popped in
1840     // the reverse order.
1841     for (const auto &Base : ClassDecl->vbases()) {
1842       auto *BaseClassDecl =
1843           cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1844 
1845       // Ignore trivial destructors.
1846       if (BaseClassDecl->hasTrivialDestructor())
1847         continue;
1848 
1849       EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1850                                         BaseClassDecl,
1851                                         /*BaseIsVirtual*/ true);
1852     }
1853 
1854     return;
1855   }
1856 
1857   assert(DtorType == Dtor_Base);
1858   // Poison the vtable pointer if it has no virtual bases, but inherits
1859   // virtual functions.
1860   if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1861       SanOpts.has(SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&
1862       ClassDecl->isPolymorphic())
1863     EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1864 
1865   // Destroy non-virtual bases.
1866   for (const auto &Base : ClassDecl->bases()) {
1867     // Ignore virtual bases.
1868     if (Base.isVirtual())
1869       continue;
1870 
1871     CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1872 
1873     // Ignore trivial destructors.
1874     if (BaseClassDecl->hasTrivialDestructor())
1875       continue;
1876 
1877     EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1878                                       BaseClassDecl,
1879                                       /*BaseIsVirtual*/ false);
1880   }
1881 
1882   // Poison fields such that access after their destructors are
1883   // invoked, and before the base class destructor runs, is invalid.
1884   if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1885       SanOpts.has(SanitizerKind::Memory))
1886     EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
1887 
1888   // Destroy direct fields.
1889   for (const auto *Field : ClassDecl->fields()) {
1890     QualType type = Field->getType();
1891     QualType::DestructionKind dtorKind = type.isDestructedType();
1892     if (!dtorKind) continue;
1893 
1894     // Anonymous union members do not have their destructors called.
1895     const RecordType *RT = type->getAsUnionType();
1896     if (RT && RT->getDecl()->isAnonymousStructOrUnion()) continue;
1897 
1898     CleanupKind cleanupKind = getCleanupKind(dtorKind);
1899     EHStack.pushCleanup<DestroyField>(cleanupKind, Field,
1900                                       getDestroyer(dtorKind),
1901                                       cleanupKind & EHCleanup);
1902   }
1903 }
1904 
1905 /// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1906 /// constructor for each of several members of an array.
1907 ///
1908 /// \param ctor the constructor to call for each element
1909 /// \param arrayType the type of the array to initialize
1910 /// \param arrayBegin an arrayType*
1911 /// \param zeroInitialize true if each element should be
1912 ///   zero-initialized before it is constructed
1913 void CodeGenFunction::EmitCXXAggrConstructorCall(
1914     const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1915     Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1916     bool zeroInitialize) {
1917   QualType elementType;
1918   llvm::Value *numElements =
1919     emitArrayLength(arrayType, elementType, arrayBegin);
1920 
1921   EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,
1922                              NewPointerIsChecked, zeroInitialize);
1923 }
1924 
1925 /// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1926 /// constructor for each of several members of an array.
1927 ///
1928 /// \param ctor the constructor to call for each element
1929 /// \param numElements the number of elements in the array;
1930 ///   may be zero
1931 /// \param arrayBase a T*, where T is the type constructed by ctor
1932 /// \param zeroInitialize true if each element should be
1933 ///   zero-initialized before it is constructed
1934 void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1935                                                  llvm::Value *numElements,
1936                                                  Address arrayBase,
1937                                                  const CXXConstructExpr *E,
1938                                                  bool NewPointerIsChecked,
1939                                                  bool zeroInitialize) {
1940   // It's legal for numElements to be zero.  This can happen both
1941   // dynamically, because x can be zero in 'new A[x]', and statically,
1942   // because of GCC extensions that permit zero-length arrays.  There
1943   // are probably legitimate places where we could assume that this
1944   // doesn't happen, but it's not clear that it's worth it.
1945   llvm::BranchInst *zeroCheckBranch = nullptr;
1946 
1947   // Optimize for a constant count.
1948   llvm::ConstantInt *constantCount
1949     = dyn_cast<llvm::ConstantInt>(numElements);
1950   if (constantCount) {
1951     // Just skip out if the constant count is zero.
1952     if (constantCount->isZero()) return;
1953 
1954   // Otherwise, emit the check.
1955   } else {
1956     llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");
1957     llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");
1958     zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
1959     EmitBlock(loopBB);
1960   }
1961 
1962   // Find the end of the array.
1963   llvm::Value *arrayBegin = arrayBase.getPointer();
1964   llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements,
1965                                                     "arrayctor.end");
1966 
1967   // Enter the loop, setting up a phi for the current location to initialize.
1968   llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1969   llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");
1970   EmitBlock(loopBB);
1971   llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,
1972                                          "arrayctor.cur");
1973   cur->addIncoming(arrayBegin, entryBB);
1974 
1975   // Inside the loop body, emit the constructor call on the array element.
1976 
1977   // The alignment of the base, adjusted by the size of a single element,
1978   // provides a conservative estimate of the alignment of every element.
1979   // (This assumes we never start tracking offsetted alignments.)
1980   //
1981   // Note that these are complete objects and so we don't need to
1982   // use the non-virtual size or alignment.
1983   QualType type = getContext().getTypeDeclType(ctor->getParent());
1984   CharUnits eltAlignment =
1985     arrayBase.getAlignment()
1986              .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
1987   Address curAddr = Address(cur, eltAlignment);
1988 
1989   // Zero initialize the storage, if requested.
1990   if (zeroInitialize)
1991     EmitNullInitialization(curAddr, type);
1992 
1993   // C++ [class.temporary]p4:
1994   // There are two contexts in which temporaries are destroyed at a different
1995   // point than the end of the full-expression. The first context is when a
1996   // default constructor is called to initialize an element of an array.
1997   // If the constructor has one or more default arguments, the destruction of
1998   // every temporary created in a default argument expression is sequenced
1999   // before the construction of the next array element, if any.
2000 
2001   {
2002     RunCleanupsScope Scope(*this);
2003 
2004     // Evaluate the constructor and its arguments in a regular
2005     // partial-destroy cleanup.
2006     if (getLangOpts().Exceptions &&
2007         !ctor->getParent()->hasTrivialDestructor()) {
2008       Destroyer *destroyer = destroyCXXObject;
2009       pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
2010                                      *destroyer);
2011     }
2012     auto currAVS = AggValueSlot::forAddr(
2013         curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,
2014         AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2015         AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,
2016         NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2017                             : AggValueSlot::IsNotSanitizerChecked);
2018     EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2019                            /*Delegating=*/false, currAVS, E);
2020   }
2021 
2022   // Go to the next element.
2023   llvm::Value *next =
2024     Builder.CreateInBoundsGEP(cur, llvm::ConstantInt::get(SizeTy, 1),
2025                               "arrayctor.next");
2026   cur->addIncoming(next, Builder.GetInsertBlock());
2027 
2028   // Check whether that's the end of the loop.
2029   llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");
2030   llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");
2031   Builder.CreateCondBr(done, contBB, loopBB);
2032 
2033   // Patch the earlier check to skip over the loop.
2034   if (zeroCheckBranch) zeroCheckBranch->setSuccessor(0, contBB);
2035 
2036   EmitBlock(contBB);
2037 }
2038 
2039 void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2040                                        Address addr,
2041                                        QualType type) {
2042   const RecordType *rtype = type->castAs<RecordType>();
2043   const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
2044   const CXXDestructorDecl *dtor = record->getDestructor();
2045   assert(!dtor->isTrivial());
2046   CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
2047                             /*Delegating=*/false, addr, type);
2048 }
2049 
2050 void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2051                                              CXXCtorType Type,
2052                                              bool ForVirtualBase,
2053                                              bool Delegating,
2054                                              AggValueSlot ThisAVS,
2055                                              const CXXConstructExpr *E) {
2056   CallArgList Args;
2057   Address This = ThisAVS.getAddress();
2058   LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2059   QualType ThisType = D->getThisType();
2060   LangAS ThisAS = ThisType.getTypePtr()->getPointeeType().getAddressSpace();
2061   llvm::Value *ThisPtr = This.getPointer();
2062 
2063   if (SlotAS != ThisAS) {
2064     unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
2065     llvm::Type *NewType =
2066         ThisPtr->getType()->getPointerElementType()->getPointerTo(TargetThisAS);
2067     ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
2068                                                     ThisAS, SlotAS, NewType);
2069   }
2070 
2071   // Push the this ptr.
2072   Args.add(RValue::get(ThisPtr), D->getThisType());
2073 
2074   // If this is a trivial constructor, emit a memcpy now before we lose
2075   // the alignment information on the argument.
2076   // FIXME: It would be better to preserve alignment information into CallArg.
2077   if (isMemcpyEquivalentSpecialMember(D)) {
2078     assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2079 
2080     const Expr *Arg = E->getArg(0);
2081     LValue Src = EmitLValue(Arg);
2082     QualType DestTy = getContext().getTypeDeclType(D->getParent());
2083     LValue Dest = MakeAddrLValue(This, DestTy);
2084     EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
2085     return;
2086   }
2087 
2088   // Add the rest of the user-supplied arguments.
2089   const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2090   EvaluationOrder Order = E->isListInitialization()
2091                               ? EvaluationOrder::ForceLeftToRight
2092                               : EvaluationOrder::Default;
2093   EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),
2094                /*ParamsToSkip*/ 0, Order);
2095 
2096   EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2097                          ThisAVS.mayOverlap(), E->getExprLoc(),
2098                          ThisAVS.isSanitizerChecked());
2099 }
2100 
2101 static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2102                                     const CXXConstructorDecl *Ctor,
2103                                     CXXCtorType Type, CallArgList &Args) {
2104   // We can't forward a variadic call.
2105   if (Ctor->isVariadic())
2106     return false;
2107 
2108   if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2109     // If the parameters are callee-cleanup, it's not safe to forward.
2110     for (auto *P : Ctor->parameters())
2111       if (P->needsDestruction(CGF.getContext()))
2112         return false;
2113 
2114     // Likewise if they're inalloca.
2115     const CGFunctionInfo &Info =
2116         CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);
2117     if (Info.usesInAlloca())
2118       return false;
2119   }
2120 
2121   // Anything else should be OK.
2122   return true;
2123 }
2124 
2125 void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2126                                              CXXCtorType Type,
2127                                              bool ForVirtualBase,
2128                                              bool Delegating,
2129                                              Address This,
2130                                              CallArgList &Args,
2131                                              AggValueSlot::Overlap_t Overlap,
2132                                              SourceLocation Loc,
2133                                              bool NewPointerIsChecked) {
2134   const CXXRecordDecl *ClassDecl = D->getParent();
2135 
2136   if (!NewPointerIsChecked)
2137     EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
2138                   getContext().getRecordType(ClassDecl), CharUnits::Zero());
2139 
2140   if (D->isTrivial() && D->isDefaultConstructor()) {
2141     assert(Args.size() == 1 && "trivial default ctor with args");
2142     return;
2143   }
2144 
2145   // If this is a trivial constructor, just emit what's needed. If this is a
2146   // union copy constructor, we must emit a memcpy, because the AST does not
2147   // model that copy.
2148   if (isMemcpyEquivalentSpecialMember(D)) {
2149     assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2150 
2151     QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2152     Address Src(Args[1].getRValue(*this).getScalarVal(),
2153                 CGM.getNaturalTypeAlignment(SrcTy));
2154     LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
2155     QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2156     LValue DestLVal = MakeAddrLValue(This, DestTy);
2157     EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
2158     return;
2159   }
2160 
2161   bool PassPrototypeArgs = true;
2162   // Check whether we can actually emit the constructor before trying to do so.
2163   if (auto Inherited = D->getInheritedConstructor()) {
2164     PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2165     if (PassPrototypeArgs && !canEmitDelegateCallArgs(*this, D, Type, Args)) {
2166       EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,
2167                                               Delegating, Args);
2168       return;
2169     }
2170   }
2171 
2172   // Insert any ABI-specific implicit constructor arguments.
2173   CGCXXABI::AddedStructorArgCounts ExtraArgs =
2174       CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,
2175                                                  Delegating, Args);
2176 
2177   // Emit the call.
2178   llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));
2179   const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2180       Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
2181   CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2182   EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, Loc);
2183 
2184   // Generate vtable assumptions if we're constructing a complete object
2185   // with a vtable.  We don't do this for base subobjects for two reasons:
2186   // first, it's incorrect for classes with virtual bases, and second, we're
2187   // about to overwrite the vptrs anyway.
2188   // We also have to make sure if we can refer to vtable:
2189   // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2190   // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2191   // sure that definition of vtable is not hidden,
2192   // then we are always safe to refer to it.
2193   // FIXME: It looks like InstCombine is very inefficient on dealing with
2194   // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2195   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2196       ClassDecl->isDynamicClass() && Type != Ctor_Base &&
2197       CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) &&
2198       CGM.getCodeGenOpts().StrictVTablePointers)
2199     EmitVTableAssumptionLoads(ClassDecl, This);
2200 }
2201 
2202 void CodeGenFunction::EmitInheritedCXXConstructorCall(
2203     const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2204     bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2205   CallArgList Args;
2206   CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
2207 
2208   // Forward the parameters.
2209   if (InheritedFromVBase &&
2210       CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2211     // Nothing to do; this construction is not responsible for constructing
2212     // the base class containing the inherited constructor.
2213     // FIXME: Can we just pass undef's for the remaining arguments if we don't
2214     // have constructor variants?
2215     Args.push_back(ThisArg);
2216   } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2217     // The inheriting constructor was inlined; just inject its arguments.
2218     assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2219            "wrong number of parameters for inherited constructor call");
2220     Args = CXXInheritedCtorInitExprArgs;
2221     Args[0] = ThisArg;
2222   } else {
2223     // The inheriting constructor was not inlined. Emit delegating arguments.
2224     Args.push_back(ThisArg);
2225     const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);
2226     assert(OuterCtor->getNumParams() == D->getNumParams());
2227     assert(!OuterCtor->isVariadic() && "should have been inlined");
2228 
2229     for (const auto *Param : OuterCtor->parameters()) {
2230       assert(getContext().hasSameUnqualifiedType(
2231           OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2232           Param->getType()));
2233       EmitDelegateCallArg(Args, Param, E->getLocation());
2234 
2235       // Forward __attribute__(pass_object_size).
2236       if (Param->hasAttr<PassObjectSizeAttr>()) {
2237         auto *POSParam = SizeArguments[Param];
2238         assert(POSParam && "missing pass_object_size value for forwarding");
2239         EmitDelegateCallArg(Args, POSParam, E->getLocation());
2240       }
2241     }
2242   }
2243 
2244   EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,
2245                          This, Args, AggValueSlot::MayOverlap,
2246                          E->getLocation(), /*NewPointerIsChecked*/true);
2247 }
2248 
2249 void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2250     const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2251     bool Delegating, CallArgList &Args) {
2252   GlobalDecl GD(Ctor, CtorType);
2253   InlinedInheritingConstructorScope Scope(*this, GD);
2254   ApplyInlineDebugLocation DebugScope(*this, GD);
2255   RunCleanupsScope RunCleanups(*this);
2256 
2257   // Save the arguments to be passed to the inherited constructor.
2258   CXXInheritedCtorInitExprArgs = Args;
2259 
2260   FunctionArgList Params;
2261   QualType RetType = BuildFunctionArgList(CurGD, Params);
2262   FnRetTy = RetType;
2263 
2264   // Insert any ABI-specific implicit constructor arguments.
2265   CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,
2266                                              ForVirtualBase, Delegating, Args);
2267 
2268   // Emit a simplified prolog. We only need to emit the implicit params.
2269   assert(Args.size() >= Params.size() && "too few arguments for call");
2270   for (unsigned I = 0, N = Args.size(); I != N; ++I) {
2271     if (I < Params.size() && isa<ImplicitParamDecl>(Params[I])) {
2272       const RValue &RV = Args[I].getRValue(*this);
2273       assert(!RV.isComplex() && "complex indirect params not supported");
2274       ParamValue Val = RV.isScalar()
2275                            ? ParamValue::forDirect(RV.getScalarVal())
2276                            : ParamValue::forIndirect(RV.getAggregateAddress());
2277       EmitParmDecl(*Params[I], Val, I + 1);
2278     }
2279   }
2280 
2281   // Create a return value slot if the ABI implementation wants one.
2282   // FIXME: This is dumb, we should ask the ABI not to try to set the return
2283   // value instead.
2284   if (!RetType->isVoidType())
2285     ReturnValue = CreateIRTemp(RetType, "retval.inhctor");
2286 
2287   CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
2288   CXXThisValue = CXXABIThisValue;
2289 
2290   // Directly emit the constructor initializers.
2291   EmitCtorPrologue(Ctor, CtorType, Params);
2292 }
2293 
2294 void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2295   llvm::Value *VTableGlobal =
2296       CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
2297   if (!VTableGlobal)
2298     return;
2299 
2300   // We can just use the base offset in the complete class.
2301   CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2302 
2303   if (!NonVirtualOffset.isZero())
2304     This =
2305         ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
2306                                         Vptr.VTableClass, Vptr.NearestVBase);
2307 
2308   llvm::Value *VPtrValue =
2309       GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
2310   llvm::Value *Cmp =
2311       Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
2312   Builder.CreateAssumption(Cmp);
2313 }
2314 
2315 void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2316                                                 Address This) {
2317   if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
2318     for (const VPtr &Vptr : getVTablePointers(ClassDecl))
2319       EmitVTableAssumptionLoad(Vptr, This);
2320 }
2321 
2322 void
2323 CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2324                                                 Address This, Address Src,
2325                                                 const CXXConstructExpr *E) {
2326   const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2327 
2328   CallArgList Args;
2329 
2330   // Push the this ptr.
2331   Args.add(RValue::get(This.getPointer()), D->getThisType());
2332 
2333   // Push the src ptr.
2334   QualType QT = *(FPT->param_type_begin());
2335   llvm::Type *t = CGM.getTypes().ConvertType(QT);
2336   Src = Builder.CreateBitCast(Src, t);
2337   Args.add(RValue::get(Src.getPointer()), QT);
2338 
2339   // Skip over first argument (Src).
2340   EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
2341                /*ParamsToSkip*/ 1);
2342 
2343   EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2344                          /*Delegating*/false, This, Args,
2345                          AggValueSlot::MayOverlap, E->getExprLoc(),
2346                          /*NewPointerIsChecked*/false);
2347 }
2348 
2349 void
2350 CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2351                                                 CXXCtorType CtorType,
2352                                                 const FunctionArgList &Args,
2353                                                 SourceLocation Loc) {
2354   CallArgList DelegateArgs;
2355 
2356   FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2357   assert(I != E && "no parameters to constructor");
2358 
2359   // this
2360   Address This = LoadCXXThisAddress();
2361   DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
2362   ++I;
2363 
2364   // FIXME: The location of the VTT parameter in the parameter list is
2365   // specific to the Itanium ABI and shouldn't be hardcoded here.
2366   if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
2367     assert(I != E && "cannot skip vtt parameter, already done with args");
2368     assert((*I)->getType()->isPointerType() &&
2369            "skipping parameter not of vtt type");
2370     ++I;
2371   }
2372 
2373   // Explicit arguments.
2374   for (; I != E; ++I) {
2375     const VarDecl *param = *I;
2376     // FIXME: per-argument source location
2377     EmitDelegateCallArg(DelegateArgs, param, Loc);
2378   }
2379 
2380   EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,
2381                          /*Delegating=*/true, This, DelegateArgs,
2382                          AggValueSlot::MayOverlap, Loc,
2383                          /*NewPointerIsChecked=*/true);
2384 }
2385 
2386 namespace {
2387   struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2388     const CXXDestructorDecl *Dtor;
2389     Address Addr;
2390     CXXDtorType Type;
2391 
2392     CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2393                            CXXDtorType Type)
2394       : Dtor(D), Addr(Addr), Type(Type) {}
2395 
2396     void Emit(CodeGenFunction &CGF, Flags flags) override {
2397       // We are calling the destructor from within the constructor.
2398       // Therefore, "this" should have the expected type.
2399       QualType ThisTy = Dtor->getThisObjectType();
2400       CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,
2401                                 /*Delegating=*/true, Addr, ThisTy);
2402     }
2403   };
2404 } // end anonymous namespace
2405 
2406 void
2407 CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2408                                                   const FunctionArgList &Args) {
2409   assert(Ctor->isDelegatingConstructor());
2410 
2411   Address ThisPtr = LoadCXXThisAddress();
2412 
2413   AggValueSlot AggSlot =
2414     AggValueSlot::forAddr(ThisPtr, Qualifiers(),
2415                           AggValueSlot::IsDestructed,
2416                           AggValueSlot::DoesNotNeedGCBarriers,
2417                           AggValueSlot::IsNotAliased,
2418                           AggValueSlot::MayOverlap,
2419                           AggValueSlot::IsNotZeroed,
2420                           // Checks are made by the code that calls constructor.
2421                           AggValueSlot::IsSanitizerChecked);
2422 
2423   EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
2424 
2425   const CXXRecordDecl *ClassDecl = Ctor->getParent();
2426   if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {
2427     CXXDtorType Type =
2428       CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;
2429 
2430     EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,
2431                                                 ClassDecl->getDestructor(),
2432                                                 ThisPtr, Type);
2433   }
2434 }
2435 
2436 void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2437                                             CXXDtorType Type,
2438                                             bool ForVirtualBase,
2439                                             bool Delegating, Address This,
2440                                             QualType ThisTy) {
2441   CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
2442                                      Delegating, This, ThisTy);
2443 }
2444 
2445 namespace {
2446   struct CallLocalDtor final : EHScopeStack::Cleanup {
2447     const CXXDestructorDecl *Dtor;
2448     Address Addr;
2449     QualType Ty;
2450 
2451     CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2452         : Dtor(D), Addr(Addr), Ty(Ty) {}
2453 
2454     void Emit(CodeGenFunction &CGF, Flags flags) override {
2455       CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2456                                 /*ForVirtualBase=*/false,
2457                                 /*Delegating=*/false, Addr, Ty);
2458     }
2459   };
2460 } // end anonymous namespace
2461 
2462 void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2463                                             QualType T, Address Addr) {
2464   EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);
2465 }
2466 
2467 void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2468   CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2469   if (!ClassDecl) return;
2470   if (ClassDecl->hasTrivialDestructor()) return;
2471 
2472   const CXXDestructorDecl *D = ClassDecl->getDestructor();
2473   assert(D && D->isUsed() && "destructor not marked as used!");
2474   PushDestructorCleanup(D, T, Addr);
2475 }
2476 
2477 void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2478   // Compute the address point.
2479   llvm::Value *VTableAddressPoint =
2480       CGM.getCXXABI().getVTableAddressPointInStructor(
2481           *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
2482 
2483   if (!VTableAddressPoint)
2484     return;
2485 
2486   // Compute where to store the address point.
2487   llvm::Value *VirtualOffset = nullptr;
2488   CharUnits NonVirtualOffset = CharUnits::Zero();
2489 
2490   if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
2491     // We need to use the virtual base offset offset because the virtual base
2492     // might have a different offset in the most derived class.
2493 
2494     VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2495         *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
2496     NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2497   } else {
2498     // We can just use the base offset in the complete class.
2499     NonVirtualOffset = Vptr.Base.getBaseOffset();
2500   }
2501 
2502   // Apply the offsets.
2503   Address VTableField = LoadCXXThisAddress();
2504 
2505   if (!NonVirtualOffset.isZero() || VirtualOffset)
2506     VTableField = ApplyNonVirtualAndVirtualOffset(
2507         *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
2508         Vptr.NearestVBase);
2509 
2510   // Finally, store the address point. Use the same LLVM types as the field to
2511   // support optimization.
2512   llvm::Type *VTablePtrTy =
2513       llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
2514           ->getPointerTo()
2515           ->getPointerTo();
2516   VTableField = Builder.CreateBitCast(VTableField, VTablePtrTy->getPointerTo());
2517   VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
2518 
2519   llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
2520   TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrTy);
2521   CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2522   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2523       CGM.getCodeGenOpts().StrictVTablePointers)
2524     CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
2525 }
2526 
2527 CodeGenFunction::VPtrsVector
2528 CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2529   CodeGenFunction::VPtrsVector VPtrsResult;
2530   VisitedVirtualBasesSetTy VBases;
2531   getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
2532                     /*NearestVBase=*/nullptr,
2533                     /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2534                     /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2535                     VPtrsResult);
2536   return VPtrsResult;
2537 }
2538 
2539 void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2540                                         const CXXRecordDecl *NearestVBase,
2541                                         CharUnits OffsetFromNearestVBase,
2542                                         bool BaseIsNonVirtualPrimaryBase,
2543                                         const CXXRecordDecl *VTableClass,
2544                                         VisitedVirtualBasesSetTy &VBases,
2545                                         VPtrsVector &Vptrs) {
2546   // If this base is a non-virtual primary base the address point has already
2547   // been set.
2548   if (!BaseIsNonVirtualPrimaryBase) {
2549     // Initialize the vtable pointer for this base.
2550     VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
2551     Vptrs.push_back(Vptr);
2552   }
2553 
2554   const CXXRecordDecl *RD = Base.getBase();
2555 
2556   // Traverse bases.
2557   for (const auto &I : RD->bases()) {
2558     auto *BaseDecl =
2559         cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
2560 
2561     // Ignore classes without a vtable.
2562     if (!BaseDecl->isDynamicClass())
2563       continue;
2564 
2565     CharUnits BaseOffset;
2566     CharUnits BaseOffsetFromNearestVBase;
2567     bool BaseDeclIsNonVirtualPrimaryBase;
2568 
2569     if (I.isVirtual()) {
2570       // Check if we've visited this virtual base before.
2571       if (!VBases.insert(BaseDecl).second)
2572         continue;
2573 
2574       const ASTRecordLayout &Layout =
2575         getContext().getASTRecordLayout(VTableClass);
2576 
2577       BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
2578       BaseOffsetFromNearestVBase = CharUnits::Zero();
2579       BaseDeclIsNonVirtualPrimaryBase = false;
2580     } else {
2581       const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2582 
2583       BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
2584       BaseOffsetFromNearestVBase =
2585         OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
2586       BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2587     }
2588 
2589     getVTablePointers(
2590         BaseSubobject(BaseDecl, BaseOffset),
2591         I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase,
2592         BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2593   }
2594 }
2595 
2596 void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2597   // Ignore classes without a vtable.
2598   if (!RD->isDynamicClass())
2599     return;
2600 
2601   // Initialize the vtable pointers for this class and all of its bases.
2602   if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
2603     for (const VPtr &Vptr : getVTablePointers(RD))
2604       InitializeVTablePointer(Vptr);
2605 
2606   if (RD->getNumVBases())
2607     CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
2608 }
2609 
2610 llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2611                                            llvm::Type *VTableTy,
2612                                            const CXXRecordDecl *RD) {
2613   Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
2614   llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
2615   TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);
2616   CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);
2617 
2618   if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2619       CGM.getCodeGenOpts().StrictVTablePointers)
2620     CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
2621 
2622   return VTable;
2623 }
2624 
2625 // If a class has a single non-virtual base and does not introduce or override
2626 // virtual member functions or fields, it will have the same layout as its base.
2627 // This function returns the least derived such class.
2628 //
2629 // Casting an instance of a base class to such a derived class is technically
2630 // undefined behavior, but it is a relatively common hack for introducing member
2631 // functions on class instances with specific properties (e.g. llvm::Operator)
2632 // that works under most compilers and should not have security implications, so
2633 // we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2634 static const CXXRecordDecl *
2635 LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2636   if (!RD->field_empty())
2637     return RD;
2638 
2639   if (RD->getNumVBases() != 0)
2640     return RD;
2641 
2642   if (RD->getNumBases() != 1)
2643     return RD;
2644 
2645   for (const CXXMethodDecl *MD : RD->methods()) {
2646     if (MD->isVirtual()) {
2647       // Virtual member functions are only ok if they are implicit destructors
2648       // because the implicit destructor will have the same semantics as the
2649       // base class's destructor if no fields are added.
2650       if (isa<CXXDestructorDecl>(MD) && MD->isImplicit())
2651         continue;
2652       return RD;
2653     }
2654   }
2655 
2656   return LeastDerivedClassWithSameLayout(
2657       RD->bases_begin()->getType()->getAsCXXRecordDecl());
2658 }
2659 
2660 void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2661                                                    llvm::Value *VTable,
2662                                                    SourceLocation Loc) {
2663   if (SanOpts.has(SanitizerKind::CFIVCall))
2664     EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
2665   else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2666            // Don't insert type test assumes if we are forcing public std
2667            // visibility.
2668            !CGM.HasLTOVisibilityPublicStd(RD)) {
2669     llvm::Metadata *MD =
2670         CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2671     llvm::Value *TypeId =
2672         llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2673 
2674     llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2675     llvm::Value *TypeTest =
2676         Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
2677                            {CastedVTable, TypeId});
2678     Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2679   }
2680 }
2681 
2682 void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2683                                                 llvm::Value *VTable,
2684                                                 CFITypeCheckKind TCK,
2685                                                 SourceLocation Loc) {
2686   if (!SanOpts.has(SanitizerKind::CFICastStrict))
2687     RD = LeastDerivedClassWithSameLayout(RD);
2688 
2689   EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2690 }
2691 
2692 void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
2693                                                 llvm::Value *Derived,
2694                                                 bool MayBeNull,
2695                                                 CFITypeCheckKind TCK,
2696                                                 SourceLocation Loc) {
2697   if (!getLangOpts().CPlusPlus)
2698     return;
2699 
2700   auto *ClassTy = T->getAs<RecordType>();
2701   if (!ClassTy)
2702     return;
2703 
2704   const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
2705 
2706   if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2707     return;
2708 
2709   if (!SanOpts.has(SanitizerKind::CFICastStrict))
2710     ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
2711 
2712   llvm::BasicBlock *ContBlock = nullptr;
2713 
2714   if (MayBeNull) {
2715     llvm::Value *DerivedNotNull =
2716         Builder.CreateIsNotNull(Derived, "cast.nonnull");
2717 
2718     llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
2719     ContBlock = createBasicBlock("cast.cont");
2720 
2721     Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
2722 
2723     EmitBlock(CheckBlock);
2724   }
2725 
2726   llvm::Value *VTable;
2727   std::tie(VTable, ClassDecl) = CGM.getCXXABI().LoadVTablePtr(
2728       *this, Address(Derived, getPointerAlign()), ClassDecl);
2729 
2730   EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
2731 
2732   if (MayBeNull) {
2733     Builder.CreateBr(ContBlock);
2734     EmitBlock(ContBlock);
2735   }
2736 }
2737 
2738 void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2739                                          llvm::Value *VTable,
2740                                          CFITypeCheckKind TCK,
2741                                          SourceLocation Loc) {
2742   if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2743       !CGM.HasHiddenLTOVisibility(RD))
2744     return;
2745 
2746   SanitizerMask M;
2747   llvm::SanitizerStatKind SSK;
2748   switch (TCK) {
2749   case CFITCK_VCall:
2750     M = SanitizerKind::CFIVCall;
2751     SSK = llvm::SanStat_CFI_VCall;
2752     break;
2753   case CFITCK_NVCall:
2754     M = SanitizerKind::CFINVCall;
2755     SSK = llvm::SanStat_CFI_NVCall;
2756     break;
2757   case CFITCK_DerivedCast:
2758     M = SanitizerKind::CFIDerivedCast;
2759     SSK = llvm::SanStat_CFI_DerivedCast;
2760     break;
2761   case CFITCK_UnrelatedCast:
2762     M = SanitizerKind::CFIUnrelatedCast;
2763     SSK = llvm::SanStat_CFI_UnrelatedCast;
2764     break;
2765   case CFITCK_ICall:
2766   case CFITCK_NVMFCall:
2767   case CFITCK_VMFCall:
2768     llvm_unreachable("unexpected sanitizer kind");
2769   }
2770 
2771   std::string TypeName = RD->getQualifiedNameAsString();
2772   if (getContext().getSanitizerBlacklist().isBlacklistedType(M, TypeName))
2773     return;
2774 
2775   SanitizerScope SanScope(this);
2776   EmitSanitizerStatReport(SSK);
2777 
2778   llvm::Metadata *MD =
2779       CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2780   llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
2781 
2782   llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2783   llvm::Value *TypeTest = Builder.CreateCall(
2784       CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, TypeId});
2785 
2786   llvm::Constant *StaticData[] = {
2787       llvm::ConstantInt::get(Int8Ty, TCK),
2788       EmitCheckSourceLocation(Loc),
2789       EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
2790   };
2791 
2792   auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2793   if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
2794     EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, CastedVTable, StaticData);
2795     return;
2796   }
2797 
2798   if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
2799     EmitTrapCheck(TypeTest);
2800     return;
2801   }
2802 
2803   llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2804       CGM.getLLVMContext(),
2805       llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
2806   llvm::Value *ValidVtable = Builder.CreateCall(
2807       CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables});
2808   EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,
2809             StaticData, {CastedVTable, ValidVtable});
2810 }
2811 
2812 bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2813   if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2814       !CGM.HasHiddenLTOVisibility(RD))
2815     return false;
2816 
2817   if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2818     return true;
2819 
2820   if (!SanOpts.has(SanitizerKind::CFIVCall) ||
2821       !CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall))
2822     return false;
2823 
2824   std::string TypeName = RD->getQualifiedNameAsString();
2825   return !getContext().getSanitizerBlacklist().isBlacklistedType(
2826       SanitizerKind::CFIVCall, TypeName);
2827 }
2828 
2829 llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2830     const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) {
2831   SanitizerScope SanScope(this);
2832 
2833   EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
2834 
2835   llvm::Metadata *MD =
2836       CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2837   llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2838 
2839   llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2840   llvm::Value *CheckedLoad = Builder.CreateCall(
2841       CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2842       {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset),
2843        TypeId});
2844   llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
2845 
2846   std::string TypeName = RD->getQualifiedNameAsString();
2847   if (SanOpts.has(SanitizerKind::CFIVCall) &&
2848       !getContext().getSanitizerBlacklist().isBlacklistedType(
2849           SanitizerKind::CFIVCall, TypeName)) {
2850     EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall),
2851               SanitizerHandler::CFICheckFail, {}, {});
2852   }
2853 
2854   return Builder.CreateBitCast(
2855       Builder.CreateExtractValue(CheckedLoad, 0),
2856       cast<llvm::PointerType>(VTable->getType())->getElementType());
2857 }
2858 
2859 void CodeGenFunction::EmitForwardingCallToLambda(
2860                                       const CXXMethodDecl *callOperator,
2861                                       CallArgList &callArgs) {
2862   // Get the address of the call operator.
2863   const CGFunctionInfo &calleeFnInfo =
2864     CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);
2865   llvm::Constant *calleePtr =
2866     CGM.GetAddrOfFunction(GlobalDecl(callOperator),
2867                           CGM.getTypes().GetFunctionType(calleeFnInfo));
2868 
2869   // Prepare the return slot.
2870   const FunctionProtoType *FPT =
2871     callOperator->getType()->castAs<FunctionProtoType>();
2872   QualType resultType = FPT->getReturnType();
2873   ReturnValueSlot returnSlot;
2874   if (!resultType->isVoidType() &&
2875       calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect &&
2876       !hasScalarEvaluationKind(calleeFnInfo.getReturnType()))
2877     returnSlot =
2878         ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2879                         /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2880 
2881   // We don't need to separately arrange the call arguments because
2882   // the call can't be variadic anyway --- it's impossible to forward
2883   // variadic arguments.
2884 
2885   // Now emit our call.
2886   auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));
2887   RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs);
2888 
2889   // If necessary, copy the returned value into the slot.
2890   if (!resultType->isVoidType() && returnSlot.isNull()) {
2891     if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {
2892       RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));
2893     }
2894     EmitReturnOfRValue(RV, resultType);
2895   } else
2896     EmitBranchThroughCleanup(ReturnBlock);
2897 }
2898 
2899 void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2900   const BlockDecl *BD = BlockInfo->getBlockDecl();
2901   const VarDecl *variable = BD->capture_begin()->getVariable();
2902   const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2903   const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2904 
2905   if (CallOp->isVariadic()) {
2906     // FIXME: Making this work correctly is nasty because it requires either
2907     // cloning the body of the call operator or making the call operator
2908     // forward.
2909     CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");
2910     return;
2911   }
2912 
2913   // Start building arguments for forwarding call
2914   CallArgList CallArgs;
2915 
2916   QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2917   Address ThisPtr = GetAddrOfBlockDecl(variable);
2918   CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
2919 
2920   // Add the rest of the parameters.
2921   for (auto param : BD->parameters())
2922     EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());
2923 
2924   assert(!Lambda->isGenericLambda() &&
2925             "generic lambda interconversion to block not implemented");
2926   EmitForwardingCallToLambda(CallOp, CallArgs);
2927 }
2928 
2929 void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
2930   const CXXRecordDecl *Lambda = MD->getParent();
2931 
2932   // Start building arguments for forwarding call
2933   CallArgList CallArgs;
2934 
2935   QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2936   llvm::Value *ThisPtr = llvm::UndefValue::get(getTypes().ConvertType(ThisType));
2937   CallArgs.add(RValue::get(ThisPtr), ThisType);
2938 
2939   // Add the rest of the parameters.
2940   for (auto Param : MD->parameters())
2941     EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
2942 
2943   const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2944   // For a generic lambda, find the corresponding call operator specialization
2945   // to which the call to the static-invoker shall be forwarded.
2946   if (Lambda->isGenericLambda()) {
2947     assert(MD->isFunctionTemplateSpecialization());
2948     const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
2949     FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
2950     void *InsertPos = nullptr;
2951     FunctionDecl *CorrespondingCallOpSpecialization =
2952         CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
2953     assert(CorrespondingCallOpSpecialization);
2954     CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
2955   }
2956   EmitForwardingCallToLambda(CallOp, CallArgs);
2957 }
2958 
2959 void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2960   if (MD->isVariadic()) {
2961     // FIXME: Making this work correctly is nasty because it requires either
2962     // cloning the body of the call operator or making the call operator forward.
2963     CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2964     return;
2965   }
2966 
2967   EmitLambdaDelegatingInvokeBody(MD);
2968 }
2969