1 //===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This contains code dealing with C++ code generation of classes 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGBlocks.h" 14 #include "CGCXXABI.h" 15 #include "CGDebugInfo.h" 16 #include "CGRecordLayout.h" 17 #include "CodeGenFunction.h" 18 #include "TargetInfo.h" 19 #include "clang/AST/Attr.h" 20 #include "clang/AST/CXXInheritance.h" 21 #include "clang/AST/CharUnits.h" 22 #include "clang/AST/DeclTemplate.h" 23 #include "clang/AST/EvaluatedExprVisitor.h" 24 #include "clang/AST/RecordLayout.h" 25 #include "clang/AST/StmtCXX.h" 26 #include "clang/Basic/CodeGenOptions.h" 27 #include "clang/Basic/TargetBuiltins.h" 28 #include "clang/CodeGen/CGFunctionInfo.h" 29 #include "llvm/IR/Intrinsics.h" 30 #include "llvm/IR/Metadata.h" 31 #include "llvm/Support/SaveAndRestore.h" 32 #include "llvm/Transforms/Utils/SanitizerStats.h" 33 #include <optional> 34 35 using namespace clang; 36 using namespace CodeGen; 37 38 /// Return the best known alignment for an unknown pointer to a 39 /// particular class. 40 CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) { 41 if (!RD->hasDefinition()) 42 return CharUnits::One(); // Hopefully won't be used anywhere. 43 44 auto &layout = getContext().getASTRecordLayout(RD); 45 46 // If the class is final, then we know that the pointer points to an 47 // object of that type and can use the full alignment. 48 if (RD->isEffectivelyFinal()) 49 return layout.getAlignment(); 50 51 // Otherwise, we have to assume it could be a subclass. 52 return layout.getNonVirtualAlignment(); 53 } 54 55 /// Return the smallest possible amount of storage that might be allocated 56 /// starting from the beginning of an object of a particular class. 57 /// 58 /// This may be smaller than sizeof(RD) if RD has virtual base classes. 59 CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) { 60 if (!RD->hasDefinition()) 61 return CharUnits::One(); 62 63 auto &layout = getContext().getASTRecordLayout(RD); 64 65 // If the class is final, then we know that the pointer points to an 66 // object of that type and can use the full alignment. 67 if (RD->isEffectivelyFinal()) 68 return layout.getSize(); 69 70 // Otherwise, we have to assume it could be a subclass. 71 return std::max(layout.getNonVirtualSize(), CharUnits::One()); 72 } 73 74 /// Return the best known alignment for a pointer to a virtual base, 75 /// given the alignment of a pointer to the derived class. 76 CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign, 77 const CXXRecordDecl *derivedClass, 78 const CXXRecordDecl *vbaseClass) { 79 // The basic idea here is that an underaligned derived pointer might 80 // indicate an underaligned base pointer. 81 82 assert(vbaseClass->isCompleteDefinition()); 83 auto &baseLayout = getContext().getASTRecordLayout(vbaseClass); 84 CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment(); 85 86 return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass, 87 expectedVBaseAlign); 88 } 89 90 CharUnits 91 CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign, 92 const CXXRecordDecl *baseDecl, 93 CharUnits expectedTargetAlign) { 94 // If the base is an incomplete type (which is, alas, possible with 95 // member pointers), be pessimistic. 96 if (!baseDecl->isCompleteDefinition()) 97 return std::min(actualBaseAlign, expectedTargetAlign); 98 99 auto &baseLayout = getContext().getASTRecordLayout(baseDecl); 100 CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment(); 101 102 // If the class is properly aligned, assume the target offset is, too. 103 // 104 // This actually isn't necessarily the right thing to do --- if the 105 // class is a complete object, but it's only properly aligned for a 106 // base subobject, then the alignments of things relative to it are 107 // probably off as well. (Note that this requires the alignment of 108 // the target to be greater than the NV alignment of the derived 109 // class.) 110 // 111 // However, our approach to this kind of under-alignment can only 112 // ever be best effort; after all, we're never going to propagate 113 // alignments through variables or parameters. Note, in particular, 114 // that constructing a polymorphic type in an address that's less 115 // than pointer-aligned will generally trap in the constructor, 116 // unless we someday add some sort of attribute to change the 117 // assumed alignment of 'this'. So our goal here is pretty much 118 // just to allow the user to explicitly say that a pointer is 119 // under-aligned and then safely access its fields and vtables. 120 if (actualBaseAlign >= expectedBaseAlign) { 121 return expectedTargetAlign; 122 } 123 124 // Otherwise, we might be offset by an arbitrary multiple of the 125 // actual alignment. The correct adjustment is to take the min of 126 // the two alignments. 127 return std::min(actualBaseAlign, expectedTargetAlign); 128 } 129 130 Address CodeGenFunction::LoadCXXThisAddress() { 131 assert(CurFuncDecl && "loading 'this' without a func declaration?"); 132 auto *MD = cast<CXXMethodDecl>(CurFuncDecl); 133 134 // Lazily compute CXXThisAlignment. 135 if (CXXThisAlignment.isZero()) { 136 // Just use the best known alignment for the parent. 137 // TODO: if we're currently emitting a complete-object ctor/dtor, 138 // we can always use the complete-object alignment. 139 CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent()); 140 } 141 142 llvm::Type *Ty = ConvertType(MD->getFunctionObjectParameterType()); 143 return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull); 144 } 145 146 /// Emit the address of a field using a member data pointer. 147 /// 148 /// \param E Only used for emergency diagnostics 149 Address 150 CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base, 151 llvm::Value *memberPtr, 152 const MemberPointerType *memberPtrType, 153 LValueBaseInfo *BaseInfo, 154 TBAAAccessInfo *TBAAInfo) { 155 // Ask the ABI to compute the actual address. 156 llvm::Value *ptr = 157 CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base, 158 memberPtr, memberPtrType); 159 160 QualType memberType = memberPtrType->getPointeeType(); 161 CharUnits memberAlign = 162 CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo); 163 memberAlign = 164 CGM.getDynamicOffsetAlignment(base.getAlignment(), 165 memberPtrType->getClass()->getAsCXXRecordDecl(), 166 memberAlign); 167 return Address(ptr, ConvertTypeForMem(memberPtrType->getPointeeType()), 168 memberAlign); 169 } 170 171 CharUnits CodeGenModule::computeNonVirtualBaseClassOffset( 172 const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start, 173 CastExpr::path_const_iterator End) { 174 CharUnits Offset = CharUnits::Zero(); 175 176 const ASTContext &Context = getContext(); 177 const CXXRecordDecl *RD = DerivedClass; 178 179 for (CastExpr::path_const_iterator I = Start; I != End; ++I) { 180 const CXXBaseSpecifier *Base = *I; 181 assert(!Base->isVirtual() && "Should not see virtual bases here!"); 182 183 // Get the layout. 184 const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); 185 186 const auto *BaseDecl = 187 cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl()); 188 189 // Add the offset. 190 Offset += Layout.getBaseClassOffset(BaseDecl); 191 192 RD = BaseDecl; 193 } 194 195 return Offset; 196 } 197 198 llvm::Constant * 199 CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl, 200 CastExpr::path_const_iterator PathBegin, 201 CastExpr::path_const_iterator PathEnd) { 202 assert(PathBegin != PathEnd && "Base path should not be empty!"); 203 204 CharUnits Offset = 205 computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd); 206 if (Offset.isZero()) 207 return nullptr; 208 209 llvm::Type *PtrDiffTy = 210 Types.ConvertType(getContext().getPointerDiffType()); 211 212 return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity()); 213 } 214 215 /// Gets the address of a direct base class within a complete object. 216 /// This should only be used for (1) non-virtual bases or (2) virtual bases 217 /// when the type is known to be complete (e.g. in complete destructors). 218 /// 219 /// The object pointed to by 'This' is assumed to be non-null. 220 Address 221 CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This, 222 const CXXRecordDecl *Derived, 223 const CXXRecordDecl *Base, 224 bool BaseIsVirtual) { 225 // 'this' must be a pointer (in some address space) to Derived. 226 assert(This.getElementType() == ConvertType(Derived)); 227 228 // Compute the offset of the virtual base. 229 CharUnits Offset; 230 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived); 231 if (BaseIsVirtual) 232 Offset = Layout.getVBaseClassOffset(Base); 233 else 234 Offset = Layout.getBaseClassOffset(Base); 235 236 // Shift and cast down to the base type. 237 // TODO: for complete types, this should be possible with a GEP. 238 Address V = This; 239 if (!Offset.isZero()) { 240 V = V.withElementType(Int8Ty); 241 V = Builder.CreateConstInBoundsByteGEP(V, Offset); 242 } 243 return V.withElementType(ConvertType(Base)); 244 } 245 246 static Address 247 ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr, 248 CharUnits nonVirtualOffset, 249 llvm::Value *virtualOffset, 250 const CXXRecordDecl *derivedClass, 251 const CXXRecordDecl *nearestVBase) { 252 // Assert that we have something to do. 253 assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr); 254 255 // Compute the offset from the static and dynamic components. 256 llvm::Value *baseOffset; 257 if (!nonVirtualOffset.isZero()) { 258 llvm::Type *OffsetType = 259 (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() && 260 CGF.CGM.getItaniumVTableContext().isRelativeLayout()) 261 ? CGF.Int32Ty 262 : CGF.PtrDiffTy; 263 baseOffset = 264 llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity()); 265 if (virtualOffset) { 266 baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset); 267 } 268 } else { 269 baseOffset = virtualOffset; 270 } 271 272 // Apply the base offset. 273 llvm::Value *ptr = addr.getPointer(); 274 ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr"); 275 276 // If we have a virtual component, the alignment of the result will 277 // be relative only to the known alignment of that vbase. 278 CharUnits alignment; 279 if (virtualOffset) { 280 assert(nearestVBase && "virtual offset without vbase?"); 281 alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(), 282 derivedClass, nearestVBase); 283 } else { 284 alignment = addr.getAlignment(); 285 } 286 alignment = alignment.alignmentAtOffset(nonVirtualOffset); 287 288 return Address(ptr, CGF.Int8Ty, alignment); 289 } 290 291 Address CodeGenFunction::GetAddressOfBaseClass( 292 Address Value, const CXXRecordDecl *Derived, 293 CastExpr::path_const_iterator PathBegin, 294 CastExpr::path_const_iterator PathEnd, bool NullCheckValue, 295 SourceLocation Loc) { 296 assert(PathBegin != PathEnd && "Base path should not be empty!"); 297 298 CastExpr::path_const_iterator Start = PathBegin; 299 const CXXRecordDecl *VBase = nullptr; 300 301 // Sema has done some convenient canonicalization here: if the 302 // access path involved any virtual steps, the conversion path will 303 // *start* with a step down to the correct virtual base subobject, 304 // and hence will not require any further steps. 305 if ((*Start)->isVirtual()) { 306 VBase = cast<CXXRecordDecl>( 307 (*Start)->getType()->castAs<RecordType>()->getDecl()); 308 ++Start; 309 } 310 311 // Compute the static offset of the ultimate destination within its 312 // allocating subobject (the virtual base, if there is one, or else 313 // the "complete" object that we see). 314 CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset( 315 VBase ? VBase : Derived, Start, PathEnd); 316 317 // If there's a virtual step, we can sometimes "devirtualize" it. 318 // For now, that's limited to when the derived type is final. 319 // TODO: "devirtualize" this for accesses to known-complete objects. 320 if (VBase && Derived->hasAttr<FinalAttr>()) { 321 const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived); 322 CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase); 323 NonVirtualOffset += vBaseOffset; 324 VBase = nullptr; // we no longer have a virtual step 325 } 326 327 // Get the base pointer type. 328 llvm::Type *BaseValueTy = ConvertType((PathEnd[-1])->getType()); 329 llvm::Type *PtrTy = llvm::PointerType::get( 330 CGM.getLLVMContext(), Value.getType()->getPointerAddressSpace()); 331 332 QualType DerivedTy = getContext().getRecordType(Derived); 333 CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived); 334 335 // If the static offset is zero and we don't have a virtual step, 336 // just do a bitcast; null checks are unnecessary. 337 if (NonVirtualOffset.isZero() && !VBase) { 338 if (sanitizePerformTypeCheck()) { 339 SanitizerSet SkippedChecks; 340 SkippedChecks.set(SanitizerKind::Null, !NullCheckValue); 341 EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(), 342 DerivedTy, DerivedAlign, SkippedChecks); 343 } 344 return Value.withElementType(BaseValueTy); 345 } 346 347 llvm::BasicBlock *origBB = nullptr; 348 llvm::BasicBlock *endBB = nullptr; 349 350 // Skip over the offset (and the vtable load) if we're supposed to 351 // null-check the pointer. 352 if (NullCheckValue) { 353 origBB = Builder.GetInsertBlock(); 354 llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull"); 355 endBB = createBasicBlock("cast.end"); 356 357 llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer()); 358 Builder.CreateCondBr(isNull, endBB, notNullBB); 359 EmitBlock(notNullBB); 360 } 361 362 if (sanitizePerformTypeCheck()) { 363 SanitizerSet SkippedChecks; 364 SkippedChecks.set(SanitizerKind::Null, true); 365 EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc, 366 Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks); 367 } 368 369 // Compute the virtual offset. 370 llvm::Value *VirtualOffset = nullptr; 371 if (VBase) { 372 VirtualOffset = 373 CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase); 374 } 375 376 // Apply both offsets. 377 Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset, 378 VirtualOffset, Derived, VBase); 379 380 // Cast to the destination type. 381 Value = Value.withElementType(BaseValueTy); 382 383 // Build a phi if we needed a null check. 384 if (NullCheckValue) { 385 llvm::BasicBlock *notNullBB = Builder.GetInsertBlock(); 386 Builder.CreateBr(endBB); 387 EmitBlock(endBB); 388 389 llvm::PHINode *PHI = Builder.CreatePHI(PtrTy, 2, "cast.result"); 390 PHI->addIncoming(Value.getPointer(), notNullBB); 391 PHI->addIncoming(llvm::Constant::getNullValue(PtrTy), origBB); 392 Value = Value.withPointer(PHI, NotKnownNonNull); 393 } 394 395 return Value; 396 } 397 398 Address 399 CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr, 400 const CXXRecordDecl *Derived, 401 CastExpr::path_const_iterator PathBegin, 402 CastExpr::path_const_iterator PathEnd, 403 bool NullCheckValue) { 404 assert(PathBegin != PathEnd && "Base path should not be empty!"); 405 406 QualType DerivedTy = 407 getContext().getCanonicalType(getContext().getTagDeclType(Derived)); 408 llvm::Type *DerivedValueTy = ConvertType(DerivedTy); 409 410 llvm::Value *NonVirtualOffset = 411 CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd); 412 413 if (!NonVirtualOffset) { 414 // No offset, we can just cast back. 415 return BaseAddr.withElementType(DerivedValueTy); 416 } 417 418 llvm::BasicBlock *CastNull = nullptr; 419 llvm::BasicBlock *CastNotNull = nullptr; 420 llvm::BasicBlock *CastEnd = nullptr; 421 422 if (NullCheckValue) { 423 CastNull = createBasicBlock("cast.null"); 424 CastNotNull = createBasicBlock("cast.notnull"); 425 CastEnd = createBasicBlock("cast.end"); 426 427 llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer()); 428 Builder.CreateCondBr(IsNull, CastNull, CastNotNull); 429 EmitBlock(CastNotNull); 430 } 431 432 // Apply the offset. 433 llvm::Value *Value = BaseAddr.getPointer(); 434 Value = Builder.CreateInBoundsGEP( 435 Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr"); 436 437 // Produce a PHI if we had a null-check. 438 if (NullCheckValue) { 439 Builder.CreateBr(CastEnd); 440 EmitBlock(CastNull); 441 Builder.CreateBr(CastEnd); 442 EmitBlock(CastEnd); 443 444 llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2); 445 PHI->addIncoming(Value, CastNotNull); 446 PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull); 447 Value = PHI; 448 } 449 450 return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived)); 451 } 452 453 llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, 454 bool ForVirtualBase, 455 bool Delegating) { 456 if (!CGM.getCXXABI().NeedsVTTParameter(GD)) { 457 // This constructor/destructor does not need a VTT parameter. 458 return nullptr; 459 } 460 461 const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent(); 462 const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent(); 463 464 uint64_t SubVTTIndex; 465 466 if (Delegating) { 467 // If this is a delegating constructor call, just load the VTT. 468 return LoadCXXVTT(); 469 } else if (RD == Base) { 470 // If the record matches the base, this is the complete ctor/dtor 471 // variant calling the base variant in a class with virtual bases. 472 assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) && 473 "doing no-op VTT offset in base dtor/ctor?"); 474 assert(!ForVirtualBase && "Can't have same class as virtual base!"); 475 SubVTTIndex = 0; 476 } else { 477 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); 478 CharUnits BaseOffset = ForVirtualBase ? 479 Layout.getVBaseClassOffset(Base) : 480 Layout.getBaseClassOffset(Base); 481 482 SubVTTIndex = 483 CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset)); 484 assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!"); 485 } 486 487 if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) { 488 // A VTT parameter was passed to the constructor, use it. 489 llvm::Value *VTT = LoadCXXVTT(); 490 return Builder.CreateConstInBoundsGEP1_64(VoidPtrTy, VTT, SubVTTIndex); 491 } else { 492 // We're the complete constructor, so get the VTT by name. 493 llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD); 494 return Builder.CreateConstInBoundsGEP2_64( 495 VTT->getValueType(), VTT, 0, SubVTTIndex); 496 } 497 } 498 499 namespace { 500 /// Call the destructor for a direct base class. 501 struct CallBaseDtor final : EHScopeStack::Cleanup { 502 const CXXRecordDecl *BaseClass; 503 bool BaseIsVirtual; 504 CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual) 505 : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {} 506 507 void Emit(CodeGenFunction &CGF, Flags flags) override { 508 const CXXRecordDecl *DerivedClass = 509 cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent(); 510 511 const CXXDestructorDecl *D = BaseClass->getDestructor(); 512 // We are already inside a destructor, so presumably the object being 513 // destroyed should have the expected type. 514 QualType ThisTy = D->getFunctionObjectParameterType(); 515 Address Addr = 516 CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(), 517 DerivedClass, BaseClass, 518 BaseIsVirtual); 519 CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual, 520 /*Delegating=*/false, Addr, ThisTy); 521 } 522 }; 523 524 /// A visitor which checks whether an initializer uses 'this' in a 525 /// way which requires the vtable to be properly set. 526 struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> { 527 typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super; 528 529 bool UsesThis; 530 531 DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {} 532 533 // Black-list all explicit and implicit references to 'this'. 534 // 535 // Do we need to worry about external references to 'this' derived 536 // from arbitrary code? If so, then anything which runs arbitrary 537 // external code might potentially access the vtable. 538 void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; } 539 }; 540 } // end anonymous namespace 541 542 static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) { 543 DynamicThisUseChecker Checker(C); 544 Checker.Visit(Init); 545 return Checker.UsesThis; 546 } 547 548 static void EmitBaseInitializer(CodeGenFunction &CGF, 549 const CXXRecordDecl *ClassDecl, 550 CXXCtorInitializer *BaseInit) { 551 assert(BaseInit->isBaseInitializer() && 552 "Must have base initializer!"); 553 554 Address ThisPtr = CGF.LoadCXXThisAddress(); 555 556 const Type *BaseType = BaseInit->getBaseClass(); 557 const auto *BaseClassDecl = 558 cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl()); 559 560 bool isBaseVirtual = BaseInit->isBaseVirtual(); 561 562 // If the initializer for the base (other than the constructor 563 // itself) accesses 'this' in any way, we need to initialize the 564 // vtables. 565 if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit())) 566 CGF.InitializeVTablePointers(ClassDecl); 567 568 // We can pretend to be a complete class because it only matters for 569 // virtual bases, and we only do virtual bases for complete ctors. 570 Address V = 571 CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl, 572 BaseClassDecl, 573 isBaseVirtual); 574 AggValueSlot AggSlot = 575 AggValueSlot::forAddr( 576 V, Qualifiers(), 577 AggValueSlot::IsDestructed, 578 AggValueSlot::DoesNotNeedGCBarriers, 579 AggValueSlot::IsNotAliased, 580 CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual)); 581 582 CGF.EmitAggExpr(BaseInit->getInit(), AggSlot); 583 584 if (CGF.CGM.getLangOpts().Exceptions && 585 !BaseClassDecl->hasTrivialDestructor()) 586 CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl, 587 isBaseVirtual); 588 } 589 590 static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) { 591 auto *CD = dyn_cast<CXXConstructorDecl>(D); 592 if (!(CD && CD->isCopyOrMoveConstructor()) && 593 !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator()) 594 return false; 595 596 // We can emit a memcpy for a trivial copy or move constructor/assignment. 597 if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) 598 return true; 599 600 // We *must* emit a memcpy for a defaulted union copy or move op. 601 if (D->getParent()->isUnion() && D->isDefaulted()) 602 return true; 603 604 return false; 605 } 606 607 static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF, 608 CXXCtorInitializer *MemberInit, 609 LValue &LHS) { 610 FieldDecl *Field = MemberInit->getAnyMember(); 611 if (MemberInit->isIndirectMemberInitializer()) { 612 // If we are initializing an anonymous union field, drill down to the field. 613 IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember(); 614 for (const auto *I : IndirectField->chain()) 615 LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I)); 616 } else { 617 LHS = CGF.EmitLValueForFieldInitialization(LHS, Field); 618 } 619 } 620 621 static void EmitMemberInitializer(CodeGenFunction &CGF, 622 const CXXRecordDecl *ClassDecl, 623 CXXCtorInitializer *MemberInit, 624 const CXXConstructorDecl *Constructor, 625 FunctionArgList &Args) { 626 ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation()); 627 assert(MemberInit->isAnyMemberInitializer() && 628 "Must have member initializer!"); 629 assert(MemberInit->getInit() && "Must have initializer!"); 630 631 // non-static data member initializers. 632 FieldDecl *Field = MemberInit->getAnyMember(); 633 QualType FieldType = Field->getType(); 634 635 llvm::Value *ThisPtr = CGF.LoadCXXThis(); 636 QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl); 637 LValue LHS; 638 639 // If a base constructor is being emitted, create an LValue that has the 640 // non-virtual alignment. 641 if (CGF.CurGD.getCtorType() == Ctor_Base) 642 LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy); 643 else 644 LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy); 645 646 EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS); 647 648 // Special case: if we are in a copy or move constructor, and we are copying 649 // an array of PODs or classes with trivial copy constructors, ignore the 650 // AST and perform the copy we know is equivalent. 651 // FIXME: This is hacky at best... if we had a bit more explicit information 652 // in the AST, we could generalize it more easily. 653 const ConstantArrayType *Array 654 = CGF.getContext().getAsConstantArrayType(FieldType); 655 if (Array && Constructor->isDefaulted() && 656 Constructor->isCopyOrMoveConstructor()) { 657 QualType BaseElementTy = CGF.getContext().getBaseElementType(Array); 658 CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit()); 659 if (BaseElementTy.isPODType(CGF.getContext()) || 660 (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) { 661 unsigned SrcArgIndex = 662 CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args); 663 llvm::Value *SrcPtr 664 = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex])); 665 LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy); 666 LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field); 667 668 // Copy the aggregate. 669 CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field), 670 LHS.isVolatileQualified()); 671 // Ensure that we destroy the objects if an exception is thrown later in 672 // the constructor. 673 QualType::DestructionKind dtorKind = FieldType.isDestructedType(); 674 if (CGF.needsEHCleanup(dtorKind)) 675 CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType); 676 return; 677 } 678 } 679 680 CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit()); 681 } 682 683 void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS, 684 Expr *Init) { 685 QualType FieldType = Field->getType(); 686 switch (getEvaluationKind(FieldType)) { 687 case TEK_Scalar: 688 if (LHS.isSimple()) { 689 EmitExprAsInit(Init, Field, LHS, false); 690 } else { 691 RValue RHS = RValue::get(EmitScalarExpr(Init)); 692 EmitStoreThroughLValue(RHS, LHS); 693 } 694 break; 695 case TEK_Complex: 696 EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true); 697 break; 698 case TEK_Aggregate: { 699 AggValueSlot Slot = AggValueSlot::forLValue( 700 LHS, *this, AggValueSlot::IsDestructed, 701 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, 702 getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed, 703 // Checks are made by the code that calls constructor. 704 AggValueSlot::IsSanitizerChecked); 705 EmitAggExpr(Init, Slot); 706 break; 707 } 708 } 709 710 // Ensure that we destroy this object if an exception is thrown 711 // later in the constructor. 712 QualType::DestructionKind dtorKind = FieldType.isDestructedType(); 713 if (needsEHCleanup(dtorKind)) 714 pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType); 715 } 716 717 /// Checks whether the given constructor is a valid subject for the 718 /// complete-to-base constructor delegation optimization, i.e. 719 /// emitting the complete constructor as a simple call to the base 720 /// constructor. 721 bool CodeGenFunction::IsConstructorDelegationValid( 722 const CXXConstructorDecl *Ctor) { 723 724 // Currently we disable the optimization for classes with virtual 725 // bases because (1) the addresses of parameter variables need to be 726 // consistent across all initializers but (2) the delegate function 727 // call necessarily creates a second copy of the parameter variable. 728 // 729 // The limiting example (purely theoretical AFAIK): 730 // struct A { A(int &c) { c++; } }; 731 // struct B : virtual A { 732 // B(int count) : A(count) { printf("%d\n", count); } 733 // }; 734 // ...although even this example could in principle be emitted as a 735 // delegation since the address of the parameter doesn't escape. 736 if (Ctor->getParent()->getNumVBases()) { 737 // TODO: white-list trivial vbase initializers. This case wouldn't 738 // be subject to the restrictions below. 739 740 // TODO: white-list cases where: 741 // - there are no non-reference parameters to the constructor 742 // - the initializers don't access any non-reference parameters 743 // - the initializers don't take the address of non-reference 744 // parameters 745 // - etc. 746 // If we ever add any of the above cases, remember that: 747 // - function-try-blocks will always exclude this optimization 748 // - we need to perform the constructor prologue and cleanup in 749 // EmitConstructorBody. 750 751 return false; 752 } 753 754 // We also disable the optimization for variadic functions because 755 // it's impossible to "re-pass" varargs. 756 if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic()) 757 return false; 758 759 // FIXME: Decide if we can do a delegation of a delegating constructor. 760 if (Ctor->isDelegatingConstructor()) 761 return false; 762 763 return true; 764 } 765 766 // Emit code in ctor (Prologue==true) or dtor (Prologue==false) 767 // to poison the extra field paddings inserted under 768 // -fsanitize-address-field-padding=1|2. 769 void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) { 770 ASTContext &Context = getContext(); 771 const CXXRecordDecl *ClassDecl = 772 Prologue ? cast<CXXConstructorDecl>(CurGD.getDecl())->getParent() 773 : cast<CXXDestructorDecl>(CurGD.getDecl())->getParent(); 774 if (!ClassDecl->mayInsertExtraPadding()) return; 775 776 struct SizeAndOffset { 777 uint64_t Size; 778 uint64_t Offset; 779 }; 780 781 unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits(); 782 const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl); 783 784 // Populate sizes and offsets of fields. 785 SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount()); 786 for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i) 787 SSV[i].Offset = 788 Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity(); 789 790 size_t NumFields = 0; 791 for (const auto *Field : ClassDecl->fields()) { 792 const FieldDecl *D = Field; 793 auto FieldInfo = Context.getTypeInfoInChars(D->getType()); 794 CharUnits FieldSize = FieldInfo.Width; 795 assert(NumFields < SSV.size()); 796 SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity(); 797 NumFields++; 798 } 799 assert(NumFields == SSV.size()); 800 if (SSV.size() <= 1) return; 801 802 // We will insert calls to __asan_* run-time functions. 803 // LLVM AddressSanitizer pass may decide to inline them later. 804 llvm::Type *Args[2] = {IntPtrTy, IntPtrTy}; 805 llvm::FunctionType *FTy = 806 llvm::FunctionType::get(CGM.VoidTy, Args, false); 807 llvm::FunctionCallee F = CGM.CreateRuntimeFunction( 808 FTy, Prologue ? "__asan_poison_intra_object_redzone" 809 : "__asan_unpoison_intra_object_redzone"); 810 811 llvm::Value *ThisPtr = LoadCXXThis(); 812 ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy); 813 uint64_t TypeSize = Info.getNonVirtualSize().getQuantity(); 814 // For each field check if it has sufficient padding, 815 // if so (un)poison it with a call. 816 for (size_t i = 0; i < SSV.size(); i++) { 817 uint64_t AsanAlignment = 8; 818 uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset; 819 uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size; 820 uint64_t EndOffset = SSV[i].Offset + SSV[i].Size; 821 if (PoisonSize < AsanAlignment || !SSV[i].Size || 822 (NextField % AsanAlignment) != 0) 823 continue; 824 Builder.CreateCall( 825 F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)), 826 Builder.getIntN(PtrSize, PoisonSize)}); 827 } 828 } 829 830 /// EmitConstructorBody - Emits the body of the current constructor. 831 void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) { 832 EmitAsanPrologueOrEpilogue(true); 833 const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl()); 834 CXXCtorType CtorType = CurGD.getCtorType(); 835 836 assert((CGM.getTarget().getCXXABI().hasConstructorVariants() || 837 CtorType == Ctor_Complete) && 838 "can only generate complete ctor for this ABI"); 839 840 // Before we go any further, try the complete->base constructor 841 // delegation optimization. 842 if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) && 843 CGM.getTarget().getCXXABI().hasConstructorVariants()) { 844 EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc()); 845 return; 846 } 847 848 const FunctionDecl *Definition = nullptr; 849 Stmt *Body = Ctor->getBody(Definition); 850 assert(Definition == Ctor && "emitting wrong constructor body"); 851 852 // Enter the function-try-block before the constructor prologue if 853 // applicable. 854 bool IsTryBody = (Body && isa<CXXTryStmt>(Body)); 855 if (IsTryBody) 856 EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true); 857 858 incrementProfileCounter(Body); 859 maybeCreateMCDCCondBitmap(); 860 861 RunCleanupsScope RunCleanups(*this); 862 863 // TODO: in restricted cases, we can emit the vbase initializers of 864 // a complete ctor and then delegate to the base ctor. 865 866 // Emit the constructor prologue, i.e. the base and member 867 // initializers. 868 EmitCtorPrologue(Ctor, CtorType, Args); 869 870 // Emit the body of the statement. 871 if (IsTryBody) 872 EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock()); 873 else if (Body) 874 EmitStmt(Body); 875 876 // Emit any cleanup blocks associated with the member or base 877 // initializers, which includes (along the exceptional path) the 878 // destructors for those members and bases that were fully 879 // constructed. 880 RunCleanups.ForceCleanup(); 881 882 if (IsTryBody) 883 ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true); 884 } 885 886 namespace { 887 /// RAII object to indicate that codegen is copying the value representation 888 /// instead of the object representation. Useful when copying a struct or 889 /// class which has uninitialized members and we're only performing 890 /// lvalue-to-rvalue conversion on the object but not its members. 891 class CopyingValueRepresentation { 892 public: 893 explicit CopyingValueRepresentation(CodeGenFunction &CGF) 894 : CGF(CGF), OldSanOpts(CGF.SanOpts) { 895 CGF.SanOpts.set(SanitizerKind::Bool, false); 896 CGF.SanOpts.set(SanitizerKind::Enum, false); 897 } 898 ~CopyingValueRepresentation() { 899 CGF.SanOpts = OldSanOpts; 900 } 901 private: 902 CodeGenFunction &CGF; 903 SanitizerSet OldSanOpts; 904 }; 905 } // end anonymous namespace 906 907 namespace { 908 class FieldMemcpyizer { 909 public: 910 FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl, 911 const VarDecl *SrcRec) 912 : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec), 913 RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)), 914 FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0), 915 LastFieldOffset(0), LastAddedFieldIndex(0) {} 916 917 bool isMemcpyableField(FieldDecl *F) const { 918 // Never memcpy fields when we are adding poisoned paddings. 919 if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding) 920 return false; 921 Qualifiers Qual = F->getType().getQualifiers(); 922 if (Qual.hasVolatile() || Qual.hasObjCLifetime()) 923 return false; 924 return true; 925 } 926 927 void addMemcpyableField(FieldDecl *F) { 928 if (F->isZeroSize(CGF.getContext())) 929 return; 930 if (!FirstField) 931 addInitialField(F); 932 else 933 addNextField(F); 934 } 935 936 CharUnits getMemcpySize(uint64_t FirstByteOffset) const { 937 ASTContext &Ctx = CGF.getContext(); 938 unsigned LastFieldSize = 939 LastField->isBitField() 940 ? LastField->getBitWidthValue(Ctx) 941 : Ctx.toBits( 942 Ctx.getTypeInfoDataSizeInChars(LastField->getType()).Width); 943 uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize - 944 FirstByteOffset + Ctx.getCharWidth() - 1; 945 CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits); 946 return MemcpySize; 947 } 948 949 void emitMemcpy() { 950 // Give the subclass a chance to bail out if it feels the memcpy isn't 951 // worth it (e.g. Hasn't aggregated enough data). 952 if (!FirstField) { 953 return; 954 } 955 956 uint64_t FirstByteOffset; 957 if (FirstField->isBitField()) { 958 const CGRecordLayout &RL = 959 CGF.getTypes().getCGRecordLayout(FirstField->getParent()); 960 const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField); 961 // FirstFieldOffset is not appropriate for bitfields, 962 // we need to use the storage offset instead. 963 FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset); 964 } else { 965 FirstByteOffset = FirstFieldOffset; 966 } 967 968 CharUnits MemcpySize = getMemcpySize(FirstByteOffset); 969 QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl); 970 Address ThisPtr = CGF.LoadCXXThisAddress(); 971 LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy); 972 LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField); 973 llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec)); 974 LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy); 975 LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField); 976 977 emitMemcpyIR( 978 Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(CGF), 979 Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(CGF), 980 MemcpySize); 981 reset(); 982 } 983 984 void reset() { 985 FirstField = nullptr; 986 } 987 988 protected: 989 CodeGenFunction &CGF; 990 const CXXRecordDecl *ClassDecl; 991 992 private: 993 void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) { 994 DestPtr = DestPtr.withElementType(CGF.Int8Ty); 995 SrcPtr = SrcPtr.withElementType(CGF.Int8Ty); 996 CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity()); 997 } 998 999 void addInitialField(FieldDecl *F) { 1000 FirstField = F; 1001 LastField = F; 1002 FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex()); 1003 LastFieldOffset = FirstFieldOffset; 1004 LastAddedFieldIndex = F->getFieldIndex(); 1005 } 1006 1007 void addNextField(FieldDecl *F) { 1008 // For the most part, the following invariant will hold: 1009 // F->getFieldIndex() == LastAddedFieldIndex + 1 1010 // The one exception is that Sema won't add a copy-initializer for an 1011 // unnamed bitfield, which will show up here as a gap in the sequence. 1012 assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 && 1013 "Cannot aggregate fields out of order."); 1014 LastAddedFieldIndex = F->getFieldIndex(); 1015 1016 // The 'first' and 'last' fields are chosen by offset, rather than field 1017 // index. This allows the code to support bitfields, as well as regular 1018 // fields. 1019 uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex()); 1020 if (FOffset < FirstFieldOffset) { 1021 FirstField = F; 1022 FirstFieldOffset = FOffset; 1023 } else if (FOffset >= LastFieldOffset) { 1024 LastField = F; 1025 LastFieldOffset = FOffset; 1026 } 1027 } 1028 1029 const VarDecl *SrcRec; 1030 const ASTRecordLayout &RecLayout; 1031 FieldDecl *FirstField; 1032 FieldDecl *LastField; 1033 uint64_t FirstFieldOffset, LastFieldOffset; 1034 unsigned LastAddedFieldIndex; 1035 }; 1036 1037 class ConstructorMemcpyizer : public FieldMemcpyizer { 1038 private: 1039 /// Get source argument for copy constructor. Returns null if not a copy 1040 /// constructor. 1041 static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF, 1042 const CXXConstructorDecl *CD, 1043 FunctionArgList &Args) { 1044 if (CD->isCopyOrMoveConstructor() && CD->isDefaulted()) 1045 return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)]; 1046 return nullptr; 1047 } 1048 1049 // Returns true if a CXXCtorInitializer represents a member initialization 1050 // that can be rolled into a memcpy. 1051 bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const { 1052 if (!MemcpyableCtor) 1053 return false; 1054 FieldDecl *Field = MemberInit->getMember(); 1055 assert(Field && "No field for member init."); 1056 QualType FieldType = Field->getType(); 1057 CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit()); 1058 1059 // Bail out on non-memcpyable, not-trivially-copyable members. 1060 if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) && 1061 !(FieldType.isTriviallyCopyableType(CGF.getContext()) || 1062 FieldType->isReferenceType())) 1063 return false; 1064 1065 // Bail out on volatile fields. 1066 if (!isMemcpyableField(Field)) 1067 return false; 1068 1069 // Otherwise we're good. 1070 return true; 1071 } 1072 1073 public: 1074 ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD, 1075 FunctionArgList &Args) 1076 : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)), 1077 ConstructorDecl(CD), 1078 MemcpyableCtor(CD->isDefaulted() && 1079 CD->isCopyOrMoveConstructor() && 1080 CGF.getLangOpts().getGC() == LangOptions::NonGC), 1081 Args(Args) { } 1082 1083 void addMemberInitializer(CXXCtorInitializer *MemberInit) { 1084 if (isMemberInitMemcpyable(MemberInit)) { 1085 AggregatedInits.push_back(MemberInit); 1086 addMemcpyableField(MemberInit->getMember()); 1087 } else { 1088 emitAggregatedInits(); 1089 EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit, 1090 ConstructorDecl, Args); 1091 } 1092 } 1093 1094 void emitAggregatedInits() { 1095 if (AggregatedInits.size() <= 1) { 1096 // This memcpy is too small to be worthwhile. Fall back on default 1097 // codegen. 1098 if (!AggregatedInits.empty()) { 1099 CopyingValueRepresentation CVR(CGF); 1100 EmitMemberInitializer(CGF, ConstructorDecl->getParent(), 1101 AggregatedInits[0], ConstructorDecl, Args); 1102 AggregatedInits.clear(); 1103 } 1104 reset(); 1105 return; 1106 } 1107 1108 pushEHDestructors(); 1109 emitMemcpy(); 1110 AggregatedInits.clear(); 1111 } 1112 1113 void pushEHDestructors() { 1114 Address ThisPtr = CGF.LoadCXXThisAddress(); 1115 QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl); 1116 LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy); 1117 1118 for (unsigned i = 0; i < AggregatedInits.size(); ++i) { 1119 CXXCtorInitializer *MemberInit = AggregatedInits[i]; 1120 QualType FieldType = MemberInit->getAnyMember()->getType(); 1121 QualType::DestructionKind dtorKind = FieldType.isDestructedType(); 1122 if (!CGF.needsEHCleanup(dtorKind)) 1123 continue; 1124 LValue FieldLHS = LHS; 1125 EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS); 1126 CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType); 1127 } 1128 } 1129 1130 void finish() { 1131 emitAggregatedInits(); 1132 } 1133 1134 private: 1135 const CXXConstructorDecl *ConstructorDecl; 1136 bool MemcpyableCtor; 1137 FunctionArgList &Args; 1138 SmallVector<CXXCtorInitializer*, 16> AggregatedInits; 1139 }; 1140 1141 class AssignmentMemcpyizer : public FieldMemcpyizer { 1142 private: 1143 // Returns the memcpyable field copied by the given statement, if one 1144 // exists. Otherwise returns null. 1145 FieldDecl *getMemcpyableField(Stmt *S) { 1146 if (!AssignmentsMemcpyable) 1147 return nullptr; 1148 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) { 1149 // Recognise trivial assignments. 1150 if (BO->getOpcode() != BO_Assign) 1151 return nullptr; 1152 MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS()); 1153 if (!ME) 1154 return nullptr; 1155 FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl()); 1156 if (!Field || !isMemcpyableField(Field)) 1157 return nullptr; 1158 Stmt *RHS = BO->getRHS(); 1159 if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS)) 1160 RHS = EC->getSubExpr(); 1161 if (!RHS) 1162 return nullptr; 1163 if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) { 1164 if (ME2->getMemberDecl() == Field) 1165 return Field; 1166 } 1167 return nullptr; 1168 } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) { 1169 CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl()); 1170 if (!(MD && isMemcpyEquivalentSpecialMember(MD))) 1171 return nullptr; 1172 MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument()); 1173 if (!IOA) 1174 return nullptr; 1175 FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl()); 1176 if (!Field || !isMemcpyableField(Field)) 1177 return nullptr; 1178 MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0)); 1179 if (!Arg0 || Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl())) 1180 return nullptr; 1181 return Field; 1182 } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) { 1183 FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl()); 1184 if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy) 1185 return nullptr; 1186 Expr *DstPtr = CE->getArg(0); 1187 if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr)) 1188 DstPtr = DC->getSubExpr(); 1189 UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr); 1190 if (!DUO || DUO->getOpcode() != UO_AddrOf) 1191 return nullptr; 1192 MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr()); 1193 if (!ME) 1194 return nullptr; 1195 FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl()); 1196 if (!Field || !isMemcpyableField(Field)) 1197 return nullptr; 1198 Expr *SrcPtr = CE->getArg(1); 1199 if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr)) 1200 SrcPtr = SC->getSubExpr(); 1201 UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr); 1202 if (!SUO || SUO->getOpcode() != UO_AddrOf) 1203 return nullptr; 1204 MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr()); 1205 if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl())) 1206 return nullptr; 1207 return Field; 1208 } 1209 1210 return nullptr; 1211 } 1212 1213 bool AssignmentsMemcpyable; 1214 SmallVector<Stmt*, 16> AggregatedStmts; 1215 1216 public: 1217 AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD, 1218 FunctionArgList &Args) 1219 : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]), 1220 AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) { 1221 assert(Args.size() == 2); 1222 } 1223 1224 void emitAssignment(Stmt *S) { 1225 FieldDecl *F = getMemcpyableField(S); 1226 if (F) { 1227 addMemcpyableField(F); 1228 AggregatedStmts.push_back(S); 1229 } else { 1230 emitAggregatedStmts(); 1231 CGF.EmitStmt(S); 1232 } 1233 } 1234 1235 void emitAggregatedStmts() { 1236 if (AggregatedStmts.size() <= 1) { 1237 if (!AggregatedStmts.empty()) { 1238 CopyingValueRepresentation CVR(CGF); 1239 CGF.EmitStmt(AggregatedStmts[0]); 1240 } 1241 reset(); 1242 } 1243 1244 emitMemcpy(); 1245 AggregatedStmts.clear(); 1246 } 1247 1248 void finish() { 1249 emitAggregatedStmts(); 1250 } 1251 }; 1252 } // end anonymous namespace 1253 1254 static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) { 1255 const Type *BaseType = BaseInit->getBaseClass(); 1256 const auto *BaseClassDecl = 1257 cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl()); 1258 return BaseClassDecl->isDynamicClass(); 1259 } 1260 1261 /// EmitCtorPrologue - This routine generates necessary code to initialize 1262 /// base classes and non-static data members belonging to this constructor. 1263 void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD, 1264 CXXCtorType CtorType, 1265 FunctionArgList &Args) { 1266 if (CD->isDelegatingConstructor()) 1267 return EmitDelegatingCXXConstructorCall(CD, Args); 1268 1269 const CXXRecordDecl *ClassDecl = CD->getParent(); 1270 1271 CXXConstructorDecl::init_const_iterator B = CD->init_begin(), 1272 E = CD->init_end(); 1273 1274 // Virtual base initializers first, if any. They aren't needed if: 1275 // - This is a base ctor variant 1276 // - There are no vbases 1277 // - The class is abstract, so a complete object of it cannot be constructed 1278 // 1279 // The check for an abstract class is necessary because sema may not have 1280 // marked virtual base destructors referenced. 1281 bool ConstructVBases = CtorType != Ctor_Base && 1282 ClassDecl->getNumVBases() != 0 && 1283 !ClassDecl->isAbstract(); 1284 1285 // In the Microsoft C++ ABI, there are no constructor variants. Instead, the 1286 // constructor of a class with virtual bases takes an additional parameter to 1287 // conditionally construct the virtual bases. Emit that check here. 1288 llvm::BasicBlock *BaseCtorContinueBB = nullptr; 1289 if (ConstructVBases && 1290 !CGM.getTarget().getCXXABI().hasConstructorVariants()) { 1291 BaseCtorContinueBB = 1292 CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl); 1293 assert(BaseCtorContinueBB); 1294 } 1295 1296 for (; B != E && (*B)->isBaseInitializer() && (*B)->isBaseVirtual(); B++) { 1297 if (!ConstructVBases) 1298 continue; 1299 SaveAndRestore ThisRAII(CXXThisValue); 1300 if (CGM.getCodeGenOpts().StrictVTablePointers && 1301 CGM.getCodeGenOpts().OptimizationLevel > 0 && 1302 isInitializerOfDynamicClass(*B)) 1303 CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis()); 1304 EmitBaseInitializer(*this, ClassDecl, *B); 1305 } 1306 1307 if (BaseCtorContinueBB) { 1308 // Complete object handler should continue to the remaining initializers. 1309 Builder.CreateBr(BaseCtorContinueBB); 1310 EmitBlock(BaseCtorContinueBB); 1311 } 1312 1313 // Then, non-virtual base initializers. 1314 for (; B != E && (*B)->isBaseInitializer(); B++) { 1315 assert(!(*B)->isBaseVirtual()); 1316 SaveAndRestore ThisRAII(CXXThisValue); 1317 if (CGM.getCodeGenOpts().StrictVTablePointers && 1318 CGM.getCodeGenOpts().OptimizationLevel > 0 && 1319 isInitializerOfDynamicClass(*B)) 1320 CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis()); 1321 EmitBaseInitializer(*this, ClassDecl, *B); 1322 } 1323 1324 InitializeVTablePointers(ClassDecl); 1325 1326 // And finally, initialize class members. 1327 FieldConstructionScope FCS(*this, LoadCXXThisAddress()); 1328 ConstructorMemcpyizer CM(*this, CD, Args); 1329 for (; B != E; B++) { 1330 CXXCtorInitializer *Member = (*B); 1331 assert(!Member->isBaseInitializer()); 1332 assert(Member->isAnyMemberInitializer() && 1333 "Delegating initializer on non-delegating constructor"); 1334 CM.addMemberInitializer(Member); 1335 } 1336 CM.finish(); 1337 } 1338 1339 static bool 1340 FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field); 1341 1342 static bool 1343 HasTrivialDestructorBody(ASTContext &Context, 1344 const CXXRecordDecl *BaseClassDecl, 1345 const CXXRecordDecl *MostDerivedClassDecl) 1346 { 1347 // If the destructor is trivial we don't have to check anything else. 1348 if (BaseClassDecl->hasTrivialDestructor()) 1349 return true; 1350 1351 if (!BaseClassDecl->getDestructor()->hasTrivialBody()) 1352 return false; 1353 1354 // Check fields. 1355 for (const auto *Field : BaseClassDecl->fields()) 1356 if (!FieldHasTrivialDestructorBody(Context, Field)) 1357 return false; 1358 1359 // Check non-virtual bases. 1360 for (const auto &I : BaseClassDecl->bases()) { 1361 if (I.isVirtual()) 1362 continue; 1363 1364 const CXXRecordDecl *NonVirtualBase = 1365 cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); 1366 if (!HasTrivialDestructorBody(Context, NonVirtualBase, 1367 MostDerivedClassDecl)) 1368 return false; 1369 } 1370 1371 if (BaseClassDecl == MostDerivedClassDecl) { 1372 // Check virtual bases. 1373 for (const auto &I : BaseClassDecl->vbases()) { 1374 const CXXRecordDecl *VirtualBase = 1375 cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); 1376 if (!HasTrivialDestructorBody(Context, VirtualBase, 1377 MostDerivedClassDecl)) 1378 return false; 1379 } 1380 } 1381 1382 return true; 1383 } 1384 1385 static bool 1386 FieldHasTrivialDestructorBody(ASTContext &Context, 1387 const FieldDecl *Field) 1388 { 1389 QualType FieldBaseElementType = Context.getBaseElementType(Field->getType()); 1390 1391 const RecordType *RT = FieldBaseElementType->getAs<RecordType>(); 1392 if (!RT) 1393 return true; 1394 1395 CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl()); 1396 1397 // The destructor for an implicit anonymous union member is never invoked. 1398 if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion()) 1399 return false; 1400 1401 return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl); 1402 } 1403 1404 /// CanSkipVTablePointerInitialization - Check whether we need to initialize 1405 /// any vtable pointers before calling this destructor. 1406 static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF, 1407 const CXXDestructorDecl *Dtor) { 1408 const CXXRecordDecl *ClassDecl = Dtor->getParent(); 1409 if (!ClassDecl->isDynamicClass()) 1410 return true; 1411 1412 // For a final class, the vtable pointer is known to already point to the 1413 // class's vtable. 1414 if (ClassDecl->isEffectivelyFinal()) 1415 return true; 1416 1417 if (!Dtor->hasTrivialBody()) 1418 return false; 1419 1420 // Check the fields. 1421 for (const auto *Field : ClassDecl->fields()) 1422 if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field)) 1423 return false; 1424 1425 return true; 1426 } 1427 1428 /// EmitDestructorBody - Emits the body of the current destructor. 1429 void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { 1430 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl()); 1431 CXXDtorType DtorType = CurGD.getDtorType(); 1432 1433 // For an abstract class, non-base destructors are never used (and can't 1434 // be emitted in general, because vbase dtors may not have been validated 1435 // by Sema), but the Itanium ABI doesn't make them optional and Clang may 1436 // in fact emit references to them from other compilations, so emit them 1437 // as functions containing a trap instruction. 1438 if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) { 1439 llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap); 1440 TrapCall->setDoesNotReturn(); 1441 TrapCall->setDoesNotThrow(); 1442 Builder.CreateUnreachable(); 1443 Builder.ClearInsertionPoint(); 1444 return; 1445 } 1446 1447 Stmt *Body = Dtor->getBody(); 1448 if (Body) { 1449 incrementProfileCounter(Body); 1450 maybeCreateMCDCCondBitmap(); 1451 } 1452 1453 // The call to operator delete in a deleting destructor happens 1454 // outside of the function-try-block, which means it's always 1455 // possible to delegate the destructor body to the complete 1456 // destructor. Do so. 1457 if (DtorType == Dtor_Deleting) { 1458 RunCleanupsScope DtorEpilogue(*this); 1459 EnterDtorCleanups(Dtor, Dtor_Deleting); 1460 if (HaveInsertPoint()) { 1461 QualType ThisTy = Dtor->getFunctionObjectParameterType(); 1462 EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false, 1463 /*Delegating=*/false, LoadCXXThisAddress(), ThisTy); 1464 } 1465 return; 1466 } 1467 1468 // If the body is a function-try-block, enter the try before 1469 // anything else. 1470 bool isTryBody = (Body && isa<CXXTryStmt>(Body)); 1471 if (isTryBody) 1472 EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true); 1473 EmitAsanPrologueOrEpilogue(false); 1474 1475 // Enter the epilogue cleanups. 1476 RunCleanupsScope DtorEpilogue(*this); 1477 1478 // If this is the complete variant, just invoke the base variant; 1479 // the epilogue will destruct the virtual bases. But we can't do 1480 // this optimization if the body is a function-try-block, because 1481 // we'd introduce *two* handler blocks. In the Microsoft ABI, we 1482 // always delegate because we might not have a definition in this TU. 1483 switch (DtorType) { 1484 case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT"); 1485 case Dtor_Deleting: llvm_unreachable("already handled deleting case"); 1486 1487 case Dtor_Complete: 1488 assert((Body || getTarget().getCXXABI().isMicrosoft()) && 1489 "can't emit a dtor without a body for non-Microsoft ABIs"); 1490 1491 // Enter the cleanup scopes for virtual bases. 1492 EnterDtorCleanups(Dtor, Dtor_Complete); 1493 1494 if (!isTryBody) { 1495 QualType ThisTy = Dtor->getFunctionObjectParameterType(); 1496 EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false, 1497 /*Delegating=*/false, LoadCXXThisAddress(), ThisTy); 1498 break; 1499 } 1500 1501 // Fallthrough: act like we're in the base variant. 1502 [[fallthrough]]; 1503 1504 case Dtor_Base: 1505 assert(Body); 1506 1507 // Enter the cleanup scopes for fields and non-virtual bases. 1508 EnterDtorCleanups(Dtor, Dtor_Base); 1509 1510 // Initialize the vtable pointers before entering the body. 1511 if (!CanSkipVTablePointerInitialization(*this, Dtor)) { 1512 // Insert the llvm.launder.invariant.group intrinsic before initializing 1513 // the vptrs to cancel any previous assumptions we might have made. 1514 if (CGM.getCodeGenOpts().StrictVTablePointers && 1515 CGM.getCodeGenOpts().OptimizationLevel > 0) 1516 CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis()); 1517 InitializeVTablePointers(Dtor->getParent()); 1518 } 1519 1520 if (isTryBody) 1521 EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock()); 1522 else if (Body) 1523 EmitStmt(Body); 1524 else { 1525 assert(Dtor->isImplicit() && "bodyless dtor not implicit"); 1526 // nothing to do besides what's in the epilogue 1527 } 1528 // -fapple-kext must inline any call to this dtor into 1529 // the caller's body. 1530 if (getLangOpts().AppleKext) 1531 CurFn->addFnAttr(llvm::Attribute::AlwaysInline); 1532 1533 break; 1534 } 1535 1536 // Jump out through the epilogue cleanups. 1537 DtorEpilogue.ForceCleanup(); 1538 1539 // Exit the try if applicable. 1540 if (isTryBody) 1541 ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true); 1542 } 1543 1544 void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) { 1545 const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl()); 1546 const Stmt *RootS = AssignOp->getBody(); 1547 assert(isa<CompoundStmt>(RootS) && 1548 "Body of an implicit assignment operator should be compound stmt."); 1549 const CompoundStmt *RootCS = cast<CompoundStmt>(RootS); 1550 1551 LexicalScope Scope(*this, RootCS->getSourceRange()); 1552 1553 incrementProfileCounter(RootCS); 1554 maybeCreateMCDCCondBitmap(); 1555 AssignmentMemcpyizer AM(*this, AssignOp, Args); 1556 for (auto *I : RootCS->body()) 1557 AM.emitAssignment(I); 1558 AM.finish(); 1559 } 1560 1561 namespace { 1562 llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF, 1563 const CXXDestructorDecl *DD) { 1564 if (Expr *ThisArg = DD->getOperatorDeleteThisArg()) 1565 return CGF.EmitScalarExpr(ThisArg); 1566 return CGF.LoadCXXThis(); 1567 } 1568 1569 /// Call the operator delete associated with the current destructor. 1570 struct CallDtorDelete final : EHScopeStack::Cleanup { 1571 CallDtorDelete() {} 1572 1573 void Emit(CodeGenFunction &CGF, Flags flags) override { 1574 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl); 1575 const CXXRecordDecl *ClassDecl = Dtor->getParent(); 1576 CGF.EmitDeleteCall(Dtor->getOperatorDelete(), 1577 LoadThisForDtorDelete(CGF, Dtor), 1578 CGF.getContext().getTagDeclType(ClassDecl)); 1579 } 1580 }; 1581 1582 void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF, 1583 llvm::Value *ShouldDeleteCondition, 1584 bool ReturnAfterDelete) { 1585 llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete"); 1586 llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue"); 1587 llvm::Value *ShouldCallDelete 1588 = CGF.Builder.CreateIsNull(ShouldDeleteCondition); 1589 CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB); 1590 1591 CGF.EmitBlock(callDeleteBB); 1592 const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl); 1593 const CXXRecordDecl *ClassDecl = Dtor->getParent(); 1594 CGF.EmitDeleteCall(Dtor->getOperatorDelete(), 1595 LoadThisForDtorDelete(CGF, Dtor), 1596 CGF.getContext().getTagDeclType(ClassDecl)); 1597 assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() == 1598 ReturnAfterDelete && 1599 "unexpected value for ReturnAfterDelete"); 1600 if (ReturnAfterDelete) 1601 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); 1602 else 1603 CGF.Builder.CreateBr(continueBB); 1604 1605 CGF.EmitBlock(continueBB); 1606 } 1607 1608 struct CallDtorDeleteConditional final : EHScopeStack::Cleanup { 1609 llvm::Value *ShouldDeleteCondition; 1610 1611 public: 1612 CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition) 1613 : ShouldDeleteCondition(ShouldDeleteCondition) { 1614 assert(ShouldDeleteCondition != nullptr); 1615 } 1616 1617 void Emit(CodeGenFunction &CGF, Flags flags) override { 1618 EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition, 1619 /*ReturnAfterDelete*/false); 1620 } 1621 }; 1622 1623 class DestroyField final : public EHScopeStack::Cleanup { 1624 const FieldDecl *field; 1625 CodeGenFunction::Destroyer *destroyer; 1626 bool useEHCleanupForArray; 1627 1628 public: 1629 DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer, 1630 bool useEHCleanupForArray) 1631 : field(field), destroyer(destroyer), 1632 useEHCleanupForArray(useEHCleanupForArray) {} 1633 1634 void Emit(CodeGenFunction &CGF, Flags flags) override { 1635 // Find the address of the field. 1636 Address thisValue = CGF.LoadCXXThisAddress(); 1637 QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent()); 1638 LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy); 1639 LValue LV = CGF.EmitLValueForField(ThisLV, field); 1640 assert(LV.isSimple()); 1641 1642 CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer, 1643 flags.isForNormalCleanup() && useEHCleanupForArray); 1644 } 1645 }; 1646 1647 class DeclAsInlineDebugLocation { 1648 CGDebugInfo *DI; 1649 llvm::MDNode *InlinedAt; 1650 std::optional<ApplyDebugLocation> Location; 1651 1652 public: 1653 DeclAsInlineDebugLocation(CodeGenFunction &CGF, const NamedDecl &Decl) 1654 : DI(CGF.getDebugInfo()) { 1655 if (!DI) 1656 return; 1657 InlinedAt = DI->getInlinedAt(); 1658 DI->setInlinedAt(CGF.Builder.getCurrentDebugLocation()); 1659 Location.emplace(CGF, Decl.getLocation()); 1660 } 1661 1662 ~DeclAsInlineDebugLocation() { 1663 if (!DI) 1664 return; 1665 Location.reset(); 1666 DI->setInlinedAt(InlinedAt); 1667 } 1668 }; 1669 1670 static void EmitSanitizerDtorCallback( 1671 CodeGenFunction &CGF, StringRef Name, llvm::Value *Ptr, 1672 std::optional<CharUnits::QuantityType> PoisonSize = {}) { 1673 CodeGenFunction::SanitizerScope SanScope(&CGF); 1674 // Pass in void pointer and size of region as arguments to runtime 1675 // function 1676 SmallVector<llvm::Value *, 2> Args = {Ptr}; 1677 SmallVector<llvm::Type *, 2> ArgTypes = {CGF.VoidPtrTy}; 1678 1679 if (PoisonSize.has_value()) { 1680 Args.emplace_back(llvm::ConstantInt::get(CGF.SizeTy, *PoisonSize)); 1681 ArgTypes.emplace_back(CGF.SizeTy); 1682 } 1683 1684 llvm::FunctionType *FnType = 1685 llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false); 1686 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(FnType, Name); 1687 1688 CGF.EmitNounwindRuntimeCall(Fn, Args); 1689 } 1690 1691 static void 1692 EmitSanitizerDtorFieldsCallback(CodeGenFunction &CGF, llvm::Value *Ptr, 1693 CharUnits::QuantityType PoisonSize) { 1694 EmitSanitizerDtorCallback(CGF, "__sanitizer_dtor_callback_fields", Ptr, 1695 PoisonSize); 1696 } 1697 1698 /// Poison base class with a trivial destructor. 1699 struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup { 1700 const CXXRecordDecl *BaseClass; 1701 bool BaseIsVirtual; 1702 SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual) 1703 : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {} 1704 1705 void Emit(CodeGenFunction &CGF, Flags flags) override { 1706 const CXXRecordDecl *DerivedClass = 1707 cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent(); 1708 1709 Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass( 1710 CGF.LoadCXXThisAddress(), DerivedClass, BaseClass, BaseIsVirtual); 1711 1712 const ASTRecordLayout &BaseLayout = 1713 CGF.getContext().getASTRecordLayout(BaseClass); 1714 CharUnits BaseSize = BaseLayout.getSize(); 1715 1716 if (!BaseSize.isPositive()) 1717 return; 1718 1719 // Use the base class declaration location as inline DebugLocation. All 1720 // fields of the class are destroyed. 1721 DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass); 1722 EmitSanitizerDtorFieldsCallback(CGF, Addr.getPointer(), 1723 BaseSize.getQuantity()); 1724 1725 // Prevent the current stack frame from disappearing from the stack trace. 1726 CGF.CurFn->addFnAttr("disable-tail-calls", "true"); 1727 } 1728 }; 1729 1730 class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup { 1731 const CXXDestructorDecl *Dtor; 1732 unsigned StartIndex; 1733 unsigned EndIndex; 1734 1735 public: 1736 SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex, 1737 unsigned EndIndex) 1738 : Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {} 1739 1740 // Generate function call for handling object poisoning. 1741 // Disables tail call elimination, to prevent the current stack frame 1742 // from disappearing from the stack trace. 1743 void Emit(CodeGenFunction &CGF, Flags flags) override { 1744 const ASTContext &Context = CGF.getContext(); 1745 const ASTRecordLayout &Layout = 1746 Context.getASTRecordLayout(Dtor->getParent()); 1747 1748 // It's a first trivial field so it should be at the begining of a char, 1749 // still round up start offset just in case. 1750 CharUnits PoisonStart = Context.toCharUnitsFromBits( 1751 Layout.getFieldOffset(StartIndex) + Context.getCharWidth() - 1); 1752 llvm::ConstantInt *OffsetSizePtr = 1753 llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity()); 1754 1755 llvm::Value *OffsetPtr = 1756 CGF.Builder.CreateGEP(CGF.Int8Ty, CGF.LoadCXXThis(), OffsetSizePtr); 1757 1758 CharUnits PoisonEnd; 1759 if (EndIndex >= Layout.getFieldCount()) { 1760 PoisonEnd = Layout.getNonVirtualSize(); 1761 } else { 1762 PoisonEnd = 1763 Context.toCharUnitsFromBits(Layout.getFieldOffset(EndIndex)); 1764 } 1765 CharUnits PoisonSize = PoisonEnd - PoisonStart; 1766 if (!PoisonSize.isPositive()) 1767 return; 1768 1769 // Use the top field declaration location as inline DebugLocation. 1770 DeclAsInlineDebugLocation InlineHere( 1771 CGF, **std::next(Dtor->getParent()->field_begin(), StartIndex)); 1772 EmitSanitizerDtorFieldsCallback(CGF, OffsetPtr, PoisonSize.getQuantity()); 1773 1774 // Prevent the current stack frame from disappearing from the stack trace. 1775 CGF.CurFn->addFnAttr("disable-tail-calls", "true"); 1776 } 1777 }; 1778 1779 class SanitizeDtorVTable final : public EHScopeStack::Cleanup { 1780 const CXXDestructorDecl *Dtor; 1781 1782 public: 1783 SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {} 1784 1785 // Generate function call for handling vtable pointer poisoning. 1786 void Emit(CodeGenFunction &CGF, Flags flags) override { 1787 assert(Dtor->getParent()->isDynamicClass()); 1788 (void)Dtor; 1789 // Poison vtable and vtable ptr if they exist for this class. 1790 llvm::Value *VTablePtr = CGF.LoadCXXThis(); 1791 1792 // Pass in void pointer and size of region as arguments to runtime 1793 // function 1794 EmitSanitizerDtorCallback(CGF, "__sanitizer_dtor_callback_vptr", 1795 VTablePtr); 1796 } 1797 }; 1798 1799 class SanitizeDtorCleanupBuilder { 1800 ASTContext &Context; 1801 EHScopeStack &EHStack; 1802 const CXXDestructorDecl *DD; 1803 std::optional<unsigned> StartIndex; 1804 1805 public: 1806 SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack, 1807 const CXXDestructorDecl *DD) 1808 : Context(Context), EHStack(EHStack), DD(DD), StartIndex(std::nullopt) {} 1809 void PushCleanupForField(const FieldDecl *Field) { 1810 if (Field->isZeroSize(Context)) 1811 return; 1812 unsigned FieldIndex = Field->getFieldIndex(); 1813 if (FieldHasTrivialDestructorBody(Context, Field)) { 1814 if (!StartIndex) 1815 StartIndex = FieldIndex; 1816 } else if (StartIndex) { 1817 EHStack.pushCleanup<SanitizeDtorFieldRange>(NormalAndEHCleanup, DD, 1818 *StartIndex, FieldIndex); 1819 StartIndex = std::nullopt; 1820 } 1821 } 1822 void End() { 1823 if (StartIndex) 1824 EHStack.pushCleanup<SanitizeDtorFieldRange>(NormalAndEHCleanup, DD, 1825 *StartIndex, -1); 1826 } 1827 }; 1828 } // end anonymous namespace 1829 1830 /// Emit all code that comes at the end of class's 1831 /// destructor. This is to call destructors on members and base classes 1832 /// in reverse order of their construction. 1833 /// 1834 /// For a deleting destructor, this also handles the case where a destroying 1835 /// operator delete completely overrides the definition. 1836 void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, 1837 CXXDtorType DtorType) { 1838 assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) && 1839 "Should not emit dtor epilogue for non-exported trivial dtor!"); 1840 1841 // The deleting-destructor phase just needs to call the appropriate 1842 // operator delete that Sema picked up. 1843 if (DtorType == Dtor_Deleting) { 1844 assert(DD->getOperatorDelete() && 1845 "operator delete missing - EnterDtorCleanups"); 1846 if (CXXStructorImplicitParamValue) { 1847 // If there is an implicit param to the deleting dtor, it's a boolean 1848 // telling whether this is a deleting destructor. 1849 if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) 1850 EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue, 1851 /*ReturnAfterDelete*/true); 1852 else 1853 EHStack.pushCleanup<CallDtorDeleteConditional>( 1854 NormalAndEHCleanup, CXXStructorImplicitParamValue); 1855 } else { 1856 if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) { 1857 const CXXRecordDecl *ClassDecl = DD->getParent(); 1858 EmitDeleteCall(DD->getOperatorDelete(), 1859 LoadThisForDtorDelete(*this, DD), 1860 getContext().getTagDeclType(ClassDecl)); 1861 EmitBranchThroughCleanup(ReturnBlock); 1862 } else { 1863 EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup); 1864 } 1865 } 1866 return; 1867 } 1868 1869 const CXXRecordDecl *ClassDecl = DD->getParent(); 1870 1871 // Unions have no bases and do not call field destructors. 1872 if (ClassDecl->isUnion()) 1873 return; 1874 1875 // The complete-destructor phase just destructs all the virtual bases. 1876 if (DtorType == Dtor_Complete) { 1877 // Poison the vtable pointer such that access after the base 1878 // and member destructors are invoked is invalid. 1879 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor && 1880 SanOpts.has(SanitizerKind::Memory) && ClassDecl->getNumVBases() && 1881 ClassDecl->isPolymorphic()) 1882 EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD); 1883 1884 // We push them in the forward order so that they'll be popped in 1885 // the reverse order. 1886 for (const auto &Base : ClassDecl->vbases()) { 1887 auto *BaseClassDecl = 1888 cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl()); 1889 1890 if (BaseClassDecl->hasTrivialDestructor()) { 1891 // Under SanitizeMemoryUseAfterDtor, poison the trivial base class 1892 // memory. For non-trival base classes the same is done in the class 1893 // destructor. 1894 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor && 1895 SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty()) 1896 EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup, 1897 BaseClassDecl, 1898 /*BaseIsVirtual*/ true); 1899 } else { 1900 EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl, 1901 /*BaseIsVirtual*/ true); 1902 } 1903 } 1904 1905 return; 1906 } 1907 1908 assert(DtorType == Dtor_Base); 1909 // Poison the vtable pointer if it has no virtual bases, but inherits 1910 // virtual functions. 1911 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor && 1912 SanOpts.has(SanitizerKind::Memory) && !ClassDecl->getNumVBases() && 1913 ClassDecl->isPolymorphic()) 1914 EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD); 1915 1916 // Destroy non-virtual bases. 1917 for (const auto &Base : ClassDecl->bases()) { 1918 // Ignore virtual bases. 1919 if (Base.isVirtual()) 1920 continue; 1921 1922 CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl(); 1923 1924 if (BaseClassDecl->hasTrivialDestructor()) { 1925 if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor && 1926 SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty()) 1927 EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup, 1928 BaseClassDecl, 1929 /*BaseIsVirtual*/ false); 1930 } else { 1931 EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl, 1932 /*BaseIsVirtual*/ false); 1933 } 1934 } 1935 1936 // Poison fields such that access after their destructors are 1937 // invoked, and before the base class destructor runs, is invalid. 1938 bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor && 1939 SanOpts.has(SanitizerKind::Memory); 1940 SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD); 1941 1942 // Destroy direct fields. 1943 for (const auto *Field : ClassDecl->fields()) { 1944 if (SanitizeFields) 1945 SanitizeBuilder.PushCleanupForField(Field); 1946 1947 QualType type = Field->getType(); 1948 QualType::DestructionKind dtorKind = type.isDestructedType(); 1949 if (!dtorKind) 1950 continue; 1951 1952 // Anonymous union members do not have their destructors called. 1953 const RecordType *RT = type->getAsUnionType(); 1954 if (RT && RT->getDecl()->isAnonymousStructOrUnion()) 1955 continue; 1956 1957 CleanupKind cleanupKind = getCleanupKind(dtorKind); 1958 EHStack.pushCleanup<DestroyField>( 1959 cleanupKind, Field, getDestroyer(dtorKind), cleanupKind & EHCleanup); 1960 } 1961 1962 if (SanitizeFields) 1963 SanitizeBuilder.End(); 1964 } 1965 1966 /// EmitCXXAggrConstructorCall - Emit a loop to call a particular 1967 /// constructor for each of several members of an array. 1968 /// 1969 /// \param ctor the constructor to call for each element 1970 /// \param arrayType the type of the array to initialize 1971 /// \param arrayBegin an arrayType* 1972 /// \param zeroInitialize true if each element should be 1973 /// zero-initialized before it is constructed 1974 void CodeGenFunction::EmitCXXAggrConstructorCall( 1975 const CXXConstructorDecl *ctor, const ArrayType *arrayType, 1976 Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked, 1977 bool zeroInitialize) { 1978 QualType elementType; 1979 llvm::Value *numElements = 1980 emitArrayLength(arrayType, elementType, arrayBegin); 1981 1982 EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E, 1983 NewPointerIsChecked, zeroInitialize); 1984 } 1985 1986 /// EmitCXXAggrConstructorCall - Emit a loop to call a particular 1987 /// constructor for each of several members of an array. 1988 /// 1989 /// \param ctor the constructor to call for each element 1990 /// \param numElements the number of elements in the array; 1991 /// may be zero 1992 /// \param arrayBase a T*, where T is the type constructed by ctor 1993 /// \param zeroInitialize true if each element should be 1994 /// zero-initialized before it is constructed 1995 void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, 1996 llvm::Value *numElements, 1997 Address arrayBase, 1998 const CXXConstructExpr *E, 1999 bool NewPointerIsChecked, 2000 bool zeroInitialize) { 2001 // It's legal for numElements to be zero. This can happen both 2002 // dynamically, because x can be zero in 'new A[x]', and statically, 2003 // because of GCC extensions that permit zero-length arrays. There 2004 // are probably legitimate places where we could assume that this 2005 // doesn't happen, but it's not clear that it's worth it. 2006 llvm::BranchInst *zeroCheckBranch = nullptr; 2007 2008 // Optimize for a constant count. 2009 llvm::ConstantInt *constantCount 2010 = dyn_cast<llvm::ConstantInt>(numElements); 2011 if (constantCount) { 2012 // Just skip out if the constant count is zero. 2013 if (constantCount->isZero()) return; 2014 2015 // Otherwise, emit the check. 2016 } else { 2017 llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop"); 2018 llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty"); 2019 zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB); 2020 EmitBlock(loopBB); 2021 } 2022 2023 // Find the end of the array. 2024 llvm::Type *elementType = arrayBase.getElementType(); 2025 llvm::Value *arrayBegin = arrayBase.getPointer(); 2026 llvm::Value *arrayEnd = Builder.CreateInBoundsGEP( 2027 elementType, arrayBegin, numElements, "arrayctor.end"); 2028 2029 // Enter the loop, setting up a phi for the current location to initialize. 2030 llvm::BasicBlock *entryBB = Builder.GetInsertBlock(); 2031 llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop"); 2032 EmitBlock(loopBB); 2033 llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2, 2034 "arrayctor.cur"); 2035 cur->addIncoming(arrayBegin, entryBB); 2036 2037 // Inside the loop body, emit the constructor call on the array element. 2038 2039 // The alignment of the base, adjusted by the size of a single element, 2040 // provides a conservative estimate of the alignment of every element. 2041 // (This assumes we never start tracking offsetted alignments.) 2042 // 2043 // Note that these are complete objects and so we don't need to 2044 // use the non-virtual size or alignment. 2045 QualType type = getContext().getTypeDeclType(ctor->getParent()); 2046 CharUnits eltAlignment = 2047 arrayBase.getAlignment() 2048 .alignmentOfArrayElement(getContext().getTypeSizeInChars(type)); 2049 Address curAddr = Address(cur, elementType, eltAlignment); 2050 2051 // Zero initialize the storage, if requested. 2052 if (zeroInitialize) 2053 EmitNullInitialization(curAddr, type); 2054 2055 // C++ [class.temporary]p4: 2056 // There are two contexts in which temporaries are destroyed at a different 2057 // point than the end of the full-expression. The first context is when a 2058 // default constructor is called to initialize an element of an array. 2059 // If the constructor has one or more default arguments, the destruction of 2060 // every temporary created in a default argument expression is sequenced 2061 // before the construction of the next array element, if any. 2062 2063 { 2064 RunCleanupsScope Scope(*this); 2065 2066 // Evaluate the constructor and its arguments in a regular 2067 // partial-destroy cleanup. 2068 if (getLangOpts().Exceptions && 2069 !ctor->getParent()->hasTrivialDestructor()) { 2070 Destroyer *destroyer = destroyCXXObject; 2071 pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment, 2072 *destroyer); 2073 } 2074 auto currAVS = AggValueSlot::forAddr( 2075 curAddr, type.getQualifiers(), AggValueSlot::IsDestructed, 2076 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, 2077 AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed, 2078 NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked 2079 : AggValueSlot::IsNotSanitizerChecked); 2080 EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false, 2081 /*Delegating=*/false, currAVS, E); 2082 } 2083 2084 // Go to the next element. 2085 llvm::Value *next = Builder.CreateInBoundsGEP( 2086 elementType, cur, llvm::ConstantInt::get(SizeTy, 1), "arrayctor.next"); 2087 cur->addIncoming(next, Builder.GetInsertBlock()); 2088 2089 // Check whether that's the end of the loop. 2090 llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done"); 2091 llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont"); 2092 Builder.CreateCondBr(done, contBB, loopBB); 2093 2094 // Patch the earlier check to skip over the loop. 2095 if (zeroCheckBranch) zeroCheckBranch->setSuccessor(0, contBB); 2096 2097 EmitBlock(contBB); 2098 } 2099 2100 void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF, 2101 Address addr, 2102 QualType type) { 2103 const RecordType *rtype = type->castAs<RecordType>(); 2104 const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl()); 2105 const CXXDestructorDecl *dtor = record->getDestructor(); 2106 assert(!dtor->isTrivial()); 2107 CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false, 2108 /*Delegating=*/false, addr, type); 2109 } 2110 2111 void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, 2112 CXXCtorType Type, 2113 bool ForVirtualBase, 2114 bool Delegating, 2115 AggValueSlot ThisAVS, 2116 const CXXConstructExpr *E) { 2117 CallArgList Args; 2118 Address This = ThisAVS.getAddress(); 2119 LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace(); 2120 LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace(); 2121 llvm::Value *ThisPtr = This.getPointer(); 2122 2123 if (SlotAS != ThisAS) { 2124 unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS); 2125 llvm::Type *NewType = 2126 llvm::PointerType::get(getLLVMContext(), TargetThisAS); 2127 ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(), 2128 ThisAS, SlotAS, NewType); 2129 } 2130 2131 // Push the this ptr. 2132 Args.add(RValue::get(ThisPtr), D->getThisType()); 2133 2134 // If this is a trivial constructor, emit a memcpy now before we lose 2135 // the alignment information on the argument. 2136 // FIXME: It would be better to preserve alignment information into CallArg. 2137 if (isMemcpyEquivalentSpecialMember(D)) { 2138 assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); 2139 2140 const Expr *Arg = E->getArg(0); 2141 LValue Src = EmitLValue(Arg); 2142 QualType DestTy = getContext().getTypeDeclType(D->getParent()); 2143 LValue Dest = MakeAddrLValue(This, DestTy); 2144 EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap()); 2145 return; 2146 } 2147 2148 // Add the rest of the user-supplied arguments. 2149 const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>(); 2150 EvaluationOrder Order = E->isListInitialization() 2151 ? EvaluationOrder::ForceLeftToRight 2152 : EvaluationOrder::Default; 2153 EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(), 2154 /*ParamsToSkip*/ 0, Order); 2155 2156 EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args, 2157 ThisAVS.mayOverlap(), E->getExprLoc(), 2158 ThisAVS.isSanitizerChecked()); 2159 } 2160 2161 static bool canEmitDelegateCallArgs(CodeGenFunction &CGF, 2162 const CXXConstructorDecl *Ctor, 2163 CXXCtorType Type, CallArgList &Args) { 2164 // We can't forward a variadic call. 2165 if (Ctor->isVariadic()) 2166 return false; 2167 2168 if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) { 2169 // If the parameters are callee-cleanup, it's not safe to forward. 2170 for (auto *P : Ctor->parameters()) 2171 if (P->needsDestruction(CGF.getContext())) 2172 return false; 2173 2174 // Likewise if they're inalloca. 2175 const CGFunctionInfo &Info = 2176 CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0); 2177 if (Info.usesInAlloca()) 2178 return false; 2179 } 2180 2181 // Anything else should be OK. 2182 return true; 2183 } 2184 2185 void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, 2186 CXXCtorType Type, 2187 bool ForVirtualBase, 2188 bool Delegating, 2189 Address This, 2190 CallArgList &Args, 2191 AggValueSlot::Overlap_t Overlap, 2192 SourceLocation Loc, 2193 bool NewPointerIsChecked) { 2194 const CXXRecordDecl *ClassDecl = D->getParent(); 2195 2196 if (!NewPointerIsChecked) 2197 EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(), 2198 getContext().getRecordType(ClassDecl), CharUnits::Zero()); 2199 2200 if (D->isTrivial() && D->isDefaultConstructor()) { 2201 assert(Args.size() == 1 && "trivial default ctor with args"); 2202 return; 2203 } 2204 2205 // If this is a trivial constructor, just emit what's needed. If this is a 2206 // union copy constructor, we must emit a memcpy, because the AST does not 2207 // model that copy. 2208 if (isMemcpyEquivalentSpecialMember(D)) { 2209 assert(Args.size() == 2 && "unexpected argcount for trivial ctor"); 2210 2211 QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType(); 2212 Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy), 2213 CGM.getNaturalTypeAlignment(SrcTy)); 2214 LValue SrcLVal = MakeAddrLValue(Src, SrcTy); 2215 QualType DestTy = getContext().getTypeDeclType(ClassDecl); 2216 LValue DestLVal = MakeAddrLValue(This, DestTy); 2217 EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap); 2218 return; 2219 } 2220 2221 bool PassPrototypeArgs = true; 2222 // Check whether we can actually emit the constructor before trying to do so. 2223 if (auto Inherited = D->getInheritedConstructor()) { 2224 PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type); 2225 if (PassPrototypeArgs && !canEmitDelegateCallArgs(*this, D, Type, Args)) { 2226 EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase, 2227 Delegating, Args); 2228 return; 2229 } 2230 } 2231 2232 // Insert any ABI-specific implicit constructor arguments. 2233 CGCXXABI::AddedStructorArgCounts ExtraArgs = 2234 CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase, 2235 Delegating, Args); 2236 2237 // Emit the call. 2238 llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type)); 2239 const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall( 2240 Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs); 2241 CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type)); 2242 EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, false, Loc); 2243 2244 // Generate vtable assumptions if we're constructing a complete object 2245 // with a vtable. We don't do this for base subobjects for two reasons: 2246 // first, it's incorrect for classes with virtual bases, and second, we're 2247 // about to overwrite the vptrs anyway. 2248 // We also have to make sure if we can refer to vtable: 2249 // - Otherwise we can refer to vtable if it's safe to speculatively emit. 2250 // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are 2251 // sure that definition of vtable is not hidden, 2252 // then we are always safe to refer to it. 2253 // FIXME: It looks like InstCombine is very inefficient on dealing with 2254 // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily. 2255 if (CGM.getCodeGenOpts().OptimizationLevel > 0 && 2256 ClassDecl->isDynamicClass() && Type != Ctor_Base && 2257 CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) && 2258 CGM.getCodeGenOpts().StrictVTablePointers) 2259 EmitVTableAssumptionLoads(ClassDecl, This); 2260 } 2261 2262 void CodeGenFunction::EmitInheritedCXXConstructorCall( 2263 const CXXConstructorDecl *D, bool ForVirtualBase, Address This, 2264 bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) { 2265 CallArgList Args; 2266 CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType()); 2267 2268 // Forward the parameters. 2269 if (InheritedFromVBase && 2270 CGM.getTarget().getCXXABI().hasConstructorVariants()) { 2271 // Nothing to do; this construction is not responsible for constructing 2272 // the base class containing the inherited constructor. 2273 // FIXME: Can we just pass undef's for the remaining arguments if we don't 2274 // have constructor variants? 2275 Args.push_back(ThisArg); 2276 } else if (!CXXInheritedCtorInitExprArgs.empty()) { 2277 // The inheriting constructor was inlined; just inject its arguments. 2278 assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() && 2279 "wrong number of parameters for inherited constructor call"); 2280 Args = CXXInheritedCtorInitExprArgs; 2281 Args[0] = ThisArg; 2282 } else { 2283 // The inheriting constructor was not inlined. Emit delegating arguments. 2284 Args.push_back(ThisArg); 2285 const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl); 2286 assert(OuterCtor->getNumParams() == D->getNumParams()); 2287 assert(!OuterCtor->isVariadic() && "should have been inlined"); 2288 2289 for (const auto *Param : OuterCtor->parameters()) { 2290 assert(getContext().hasSameUnqualifiedType( 2291 OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(), 2292 Param->getType())); 2293 EmitDelegateCallArg(Args, Param, E->getLocation()); 2294 2295 // Forward __attribute__(pass_object_size). 2296 if (Param->hasAttr<PassObjectSizeAttr>()) { 2297 auto *POSParam = SizeArguments[Param]; 2298 assert(POSParam && "missing pass_object_size value for forwarding"); 2299 EmitDelegateCallArg(Args, POSParam, E->getLocation()); 2300 } 2301 } 2302 } 2303 2304 EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false, 2305 This, Args, AggValueSlot::MayOverlap, 2306 E->getLocation(), /*NewPointerIsChecked*/true); 2307 } 2308 2309 void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall( 2310 const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase, 2311 bool Delegating, CallArgList &Args) { 2312 GlobalDecl GD(Ctor, CtorType); 2313 InlinedInheritingConstructorScope Scope(*this, GD); 2314 ApplyInlineDebugLocation DebugScope(*this, GD); 2315 RunCleanupsScope RunCleanups(*this); 2316 2317 // Save the arguments to be passed to the inherited constructor. 2318 CXXInheritedCtorInitExprArgs = Args; 2319 2320 FunctionArgList Params; 2321 QualType RetType = BuildFunctionArgList(CurGD, Params); 2322 FnRetTy = RetType; 2323 2324 // Insert any ABI-specific implicit constructor arguments. 2325 CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType, 2326 ForVirtualBase, Delegating, Args); 2327 2328 // Emit a simplified prolog. We only need to emit the implicit params. 2329 assert(Args.size() >= Params.size() && "too few arguments for call"); 2330 for (unsigned I = 0, N = Args.size(); I != N; ++I) { 2331 if (I < Params.size() && isa<ImplicitParamDecl>(Params[I])) { 2332 const RValue &RV = Args[I].getRValue(*this); 2333 assert(!RV.isComplex() && "complex indirect params not supported"); 2334 ParamValue Val = RV.isScalar() 2335 ? ParamValue::forDirect(RV.getScalarVal()) 2336 : ParamValue::forIndirect(RV.getAggregateAddress()); 2337 EmitParmDecl(*Params[I], Val, I + 1); 2338 } 2339 } 2340 2341 // Create a return value slot if the ABI implementation wants one. 2342 // FIXME: This is dumb, we should ask the ABI not to try to set the return 2343 // value instead. 2344 if (!RetType->isVoidType()) 2345 ReturnValue = CreateIRTemp(RetType, "retval.inhctor"); 2346 2347 CGM.getCXXABI().EmitInstanceFunctionProlog(*this); 2348 CXXThisValue = CXXABIThisValue; 2349 2350 // Directly emit the constructor initializers. 2351 EmitCtorPrologue(Ctor, CtorType, Params); 2352 } 2353 2354 void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) { 2355 llvm::Value *VTableGlobal = 2356 CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass); 2357 if (!VTableGlobal) 2358 return; 2359 2360 // We can just use the base offset in the complete class. 2361 CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset(); 2362 2363 if (!NonVirtualOffset.isZero()) 2364 This = 2365 ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr, 2366 Vptr.VTableClass, Vptr.NearestVBase); 2367 2368 llvm::Value *VPtrValue = 2369 GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass); 2370 llvm::Value *Cmp = 2371 Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables"); 2372 Builder.CreateAssumption(Cmp); 2373 } 2374 2375 void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, 2376 Address This) { 2377 if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl)) 2378 for (const VPtr &Vptr : getVTablePointers(ClassDecl)) 2379 EmitVTableAssumptionLoad(Vptr, This); 2380 } 2381 2382 void 2383 CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, 2384 Address This, Address Src, 2385 const CXXConstructExpr *E) { 2386 const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>(); 2387 2388 CallArgList Args; 2389 2390 // Push the this ptr. 2391 Args.add(RValue::get(This.getPointer()), D->getThisType()); 2392 2393 // Push the src ptr. 2394 QualType QT = *(FPT->param_type_begin()); 2395 llvm::Type *t = CGM.getTypes().ConvertType(QT); 2396 llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t); 2397 Args.add(RValue::get(SrcVal), QT); 2398 2399 // Skip over first argument (Src). 2400 EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(), 2401 /*ParamsToSkip*/ 1); 2402 2403 EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false, 2404 /*Delegating*/false, This, Args, 2405 AggValueSlot::MayOverlap, E->getExprLoc(), 2406 /*NewPointerIsChecked*/false); 2407 } 2408 2409 void 2410 CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor, 2411 CXXCtorType CtorType, 2412 const FunctionArgList &Args, 2413 SourceLocation Loc) { 2414 CallArgList DelegateArgs; 2415 2416 FunctionArgList::const_iterator I = Args.begin(), E = Args.end(); 2417 assert(I != E && "no parameters to constructor"); 2418 2419 // this 2420 Address This = LoadCXXThisAddress(); 2421 DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType()); 2422 ++I; 2423 2424 // FIXME: The location of the VTT parameter in the parameter list is 2425 // specific to the Itanium ABI and shouldn't be hardcoded here. 2426 if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) { 2427 assert(I != E && "cannot skip vtt parameter, already done with args"); 2428 assert((*I)->getType()->isPointerType() && 2429 "skipping parameter not of vtt type"); 2430 ++I; 2431 } 2432 2433 // Explicit arguments. 2434 for (; I != E; ++I) { 2435 const VarDecl *param = *I; 2436 // FIXME: per-argument source location 2437 EmitDelegateCallArg(DelegateArgs, param, Loc); 2438 } 2439 2440 EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false, 2441 /*Delegating=*/true, This, DelegateArgs, 2442 AggValueSlot::MayOverlap, Loc, 2443 /*NewPointerIsChecked=*/true); 2444 } 2445 2446 namespace { 2447 struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup { 2448 const CXXDestructorDecl *Dtor; 2449 Address Addr; 2450 CXXDtorType Type; 2451 2452 CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr, 2453 CXXDtorType Type) 2454 : Dtor(D), Addr(Addr), Type(Type) {} 2455 2456 void Emit(CodeGenFunction &CGF, Flags flags) override { 2457 // We are calling the destructor from within the constructor. 2458 // Therefore, "this" should have the expected type. 2459 QualType ThisTy = Dtor->getFunctionObjectParameterType(); 2460 CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false, 2461 /*Delegating=*/true, Addr, ThisTy); 2462 } 2463 }; 2464 } // end anonymous namespace 2465 2466 void 2467 CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor, 2468 const FunctionArgList &Args) { 2469 assert(Ctor->isDelegatingConstructor()); 2470 2471 Address ThisPtr = LoadCXXThisAddress(); 2472 2473 AggValueSlot AggSlot = 2474 AggValueSlot::forAddr(ThisPtr, Qualifiers(), 2475 AggValueSlot::IsDestructed, 2476 AggValueSlot::DoesNotNeedGCBarriers, 2477 AggValueSlot::IsNotAliased, 2478 AggValueSlot::MayOverlap, 2479 AggValueSlot::IsNotZeroed, 2480 // Checks are made by the code that calls constructor. 2481 AggValueSlot::IsSanitizerChecked); 2482 2483 EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot); 2484 2485 const CXXRecordDecl *ClassDecl = Ctor->getParent(); 2486 if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) { 2487 CXXDtorType Type = 2488 CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base; 2489 2490 EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup, 2491 ClassDecl->getDestructor(), 2492 ThisPtr, Type); 2493 } 2494 } 2495 2496 void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD, 2497 CXXDtorType Type, 2498 bool ForVirtualBase, 2499 bool Delegating, Address This, 2500 QualType ThisTy) { 2501 CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase, 2502 Delegating, This, ThisTy); 2503 } 2504 2505 namespace { 2506 struct CallLocalDtor final : EHScopeStack::Cleanup { 2507 const CXXDestructorDecl *Dtor; 2508 Address Addr; 2509 QualType Ty; 2510 2511 CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty) 2512 : Dtor(D), Addr(Addr), Ty(Ty) {} 2513 2514 void Emit(CodeGenFunction &CGF, Flags flags) override { 2515 CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, 2516 /*ForVirtualBase=*/false, 2517 /*Delegating=*/false, Addr, Ty); 2518 } 2519 }; 2520 } // end anonymous namespace 2521 2522 void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D, 2523 QualType T, Address Addr) { 2524 EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T); 2525 } 2526 2527 void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) { 2528 CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl(); 2529 if (!ClassDecl) return; 2530 if (ClassDecl->hasTrivialDestructor()) return; 2531 2532 const CXXDestructorDecl *D = ClassDecl->getDestructor(); 2533 assert(D && D->isUsed() && "destructor not marked as used!"); 2534 PushDestructorCleanup(D, T, Addr); 2535 } 2536 2537 void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) { 2538 // Compute the address point. 2539 llvm::Value *VTableAddressPoint = 2540 CGM.getCXXABI().getVTableAddressPointInStructor( 2541 *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase); 2542 2543 if (!VTableAddressPoint) 2544 return; 2545 2546 // Compute where to store the address point. 2547 llvm::Value *VirtualOffset = nullptr; 2548 CharUnits NonVirtualOffset = CharUnits::Zero(); 2549 2550 if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) { 2551 // We need to use the virtual base offset offset because the virtual base 2552 // might have a different offset in the most derived class. 2553 2554 VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset( 2555 *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase); 2556 NonVirtualOffset = Vptr.OffsetFromNearestVBase; 2557 } else { 2558 // We can just use the base offset in the complete class. 2559 NonVirtualOffset = Vptr.Base.getBaseOffset(); 2560 } 2561 2562 // Apply the offsets. 2563 Address VTableField = LoadCXXThisAddress(); 2564 if (!NonVirtualOffset.isZero() || VirtualOffset) 2565 VTableField = ApplyNonVirtualAndVirtualOffset( 2566 *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass, 2567 Vptr.NearestVBase); 2568 2569 // Finally, store the address point. Use the same LLVM types as the field to 2570 // support optimization. 2571 unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace(); 2572 llvm::Type *PtrTy = llvm::PointerType::get(CGM.getLLVMContext(), GlobalsAS); 2573 // vtable field is derived from `this` pointer, therefore they should be in 2574 // the same addr space. Note that this might not be LLVM address space 0. 2575 VTableField = VTableField.withElementType(PtrTy); 2576 2577 llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField); 2578 TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(PtrTy); 2579 CGM.DecorateInstructionWithTBAA(Store, TBAAInfo); 2580 if (CGM.getCodeGenOpts().OptimizationLevel > 0 && 2581 CGM.getCodeGenOpts().StrictVTablePointers) 2582 CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass); 2583 } 2584 2585 CodeGenFunction::VPtrsVector 2586 CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) { 2587 CodeGenFunction::VPtrsVector VPtrsResult; 2588 VisitedVirtualBasesSetTy VBases; 2589 getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()), 2590 /*NearestVBase=*/nullptr, 2591 /*OffsetFromNearestVBase=*/CharUnits::Zero(), 2592 /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases, 2593 VPtrsResult); 2594 return VPtrsResult; 2595 } 2596 2597 void CodeGenFunction::getVTablePointers(BaseSubobject Base, 2598 const CXXRecordDecl *NearestVBase, 2599 CharUnits OffsetFromNearestVBase, 2600 bool BaseIsNonVirtualPrimaryBase, 2601 const CXXRecordDecl *VTableClass, 2602 VisitedVirtualBasesSetTy &VBases, 2603 VPtrsVector &Vptrs) { 2604 // If this base is a non-virtual primary base the address point has already 2605 // been set. 2606 if (!BaseIsNonVirtualPrimaryBase) { 2607 // Initialize the vtable pointer for this base. 2608 VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass}; 2609 Vptrs.push_back(Vptr); 2610 } 2611 2612 const CXXRecordDecl *RD = Base.getBase(); 2613 2614 // Traverse bases. 2615 for (const auto &I : RD->bases()) { 2616 auto *BaseDecl = 2617 cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); 2618 2619 // Ignore classes without a vtable. 2620 if (!BaseDecl->isDynamicClass()) 2621 continue; 2622 2623 CharUnits BaseOffset; 2624 CharUnits BaseOffsetFromNearestVBase; 2625 bool BaseDeclIsNonVirtualPrimaryBase; 2626 2627 if (I.isVirtual()) { 2628 // Check if we've visited this virtual base before. 2629 if (!VBases.insert(BaseDecl).second) 2630 continue; 2631 2632 const ASTRecordLayout &Layout = 2633 getContext().getASTRecordLayout(VTableClass); 2634 2635 BaseOffset = Layout.getVBaseClassOffset(BaseDecl); 2636 BaseOffsetFromNearestVBase = CharUnits::Zero(); 2637 BaseDeclIsNonVirtualPrimaryBase = false; 2638 } else { 2639 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); 2640 2641 BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl); 2642 BaseOffsetFromNearestVBase = 2643 OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl); 2644 BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl; 2645 } 2646 2647 getVTablePointers( 2648 BaseSubobject(BaseDecl, BaseOffset), 2649 I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase, 2650 BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs); 2651 } 2652 } 2653 2654 void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) { 2655 // Ignore classes without a vtable. 2656 if (!RD->isDynamicClass()) 2657 return; 2658 2659 // Initialize the vtable pointers for this class and all of its bases. 2660 if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD)) 2661 for (const VPtr &Vptr : getVTablePointers(RD)) 2662 InitializeVTablePointer(Vptr); 2663 2664 if (RD->getNumVBases()) 2665 CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD); 2666 } 2667 2668 llvm::Value *CodeGenFunction::GetVTablePtr(Address This, 2669 llvm::Type *VTableTy, 2670 const CXXRecordDecl *RD) { 2671 Address VTablePtrSrc = This.withElementType(VTableTy); 2672 llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable"); 2673 TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy); 2674 CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo); 2675 2676 if (CGM.getCodeGenOpts().OptimizationLevel > 0 && 2677 CGM.getCodeGenOpts().StrictVTablePointers) 2678 CGM.DecorateInstructionWithInvariantGroup(VTable, RD); 2679 2680 return VTable; 2681 } 2682 2683 // If a class has a single non-virtual base and does not introduce or override 2684 // virtual member functions or fields, it will have the same layout as its base. 2685 // This function returns the least derived such class. 2686 // 2687 // Casting an instance of a base class to such a derived class is technically 2688 // undefined behavior, but it is a relatively common hack for introducing member 2689 // functions on class instances with specific properties (e.g. llvm::Operator) 2690 // that works under most compilers and should not have security implications, so 2691 // we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict. 2692 static const CXXRecordDecl * 2693 LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) { 2694 if (!RD->field_empty()) 2695 return RD; 2696 2697 if (RD->getNumVBases() != 0) 2698 return RD; 2699 2700 if (RD->getNumBases() != 1) 2701 return RD; 2702 2703 for (const CXXMethodDecl *MD : RD->methods()) { 2704 if (MD->isVirtual()) { 2705 // Virtual member functions are only ok if they are implicit destructors 2706 // because the implicit destructor will have the same semantics as the 2707 // base class's destructor if no fields are added. 2708 if (isa<CXXDestructorDecl>(MD) && MD->isImplicit()) 2709 continue; 2710 return RD; 2711 } 2712 } 2713 2714 return LeastDerivedClassWithSameLayout( 2715 RD->bases_begin()->getType()->getAsCXXRecordDecl()); 2716 } 2717 2718 void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD, 2719 llvm::Value *VTable, 2720 SourceLocation Loc) { 2721 if (SanOpts.has(SanitizerKind::CFIVCall)) 2722 EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc); 2723 else if (CGM.getCodeGenOpts().WholeProgramVTables && 2724 // Don't insert type test assumes if we are forcing public 2725 // visibility. 2726 !CGM.AlwaysHasLTOVisibilityPublic(RD)) { 2727 QualType Ty = QualType(RD->getTypeForDecl(), 0); 2728 llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(Ty); 2729 llvm::Value *TypeId = 2730 llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD); 2731 2732 // If we already know that the call has hidden LTO visibility, emit 2733 // @llvm.type.test(). Otherwise emit @llvm.public.type.test(), which WPD 2734 // will convert to @llvm.type.test() if we assert at link time that we have 2735 // whole program visibility. 2736 llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD) 2737 ? llvm::Intrinsic::type_test 2738 : llvm::Intrinsic::public_type_test; 2739 llvm::Value *TypeTest = 2740 Builder.CreateCall(CGM.getIntrinsic(IID), {VTable, TypeId}); 2741 Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest); 2742 } 2743 } 2744 2745 void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, 2746 llvm::Value *VTable, 2747 CFITypeCheckKind TCK, 2748 SourceLocation Loc) { 2749 if (!SanOpts.has(SanitizerKind::CFICastStrict)) 2750 RD = LeastDerivedClassWithSameLayout(RD); 2751 2752 EmitVTablePtrCheck(RD, VTable, TCK, Loc); 2753 } 2754 2755 void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived, 2756 bool MayBeNull, 2757 CFITypeCheckKind TCK, 2758 SourceLocation Loc) { 2759 if (!getLangOpts().CPlusPlus) 2760 return; 2761 2762 auto *ClassTy = T->getAs<RecordType>(); 2763 if (!ClassTy) 2764 return; 2765 2766 const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl()); 2767 2768 if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass()) 2769 return; 2770 2771 if (!SanOpts.has(SanitizerKind::CFICastStrict)) 2772 ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl); 2773 2774 llvm::BasicBlock *ContBlock = nullptr; 2775 2776 if (MayBeNull) { 2777 llvm::Value *DerivedNotNull = 2778 Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull"); 2779 2780 llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check"); 2781 ContBlock = createBasicBlock("cast.cont"); 2782 2783 Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock); 2784 2785 EmitBlock(CheckBlock); 2786 } 2787 2788 llvm::Value *VTable; 2789 std::tie(VTable, ClassDecl) = 2790 CGM.getCXXABI().LoadVTablePtr(*this, Derived, ClassDecl); 2791 2792 EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc); 2793 2794 if (MayBeNull) { 2795 Builder.CreateBr(ContBlock); 2796 EmitBlock(ContBlock); 2797 } 2798 } 2799 2800 void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD, 2801 llvm::Value *VTable, 2802 CFITypeCheckKind TCK, 2803 SourceLocation Loc) { 2804 if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso && 2805 !CGM.HasHiddenLTOVisibility(RD)) 2806 return; 2807 2808 SanitizerMask M; 2809 llvm::SanitizerStatKind SSK; 2810 switch (TCK) { 2811 case CFITCK_VCall: 2812 M = SanitizerKind::CFIVCall; 2813 SSK = llvm::SanStat_CFI_VCall; 2814 break; 2815 case CFITCK_NVCall: 2816 M = SanitizerKind::CFINVCall; 2817 SSK = llvm::SanStat_CFI_NVCall; 2818 break; 2819 case CFITCK_DerivedCast: 2820 M = SanitizerKind::CFIDerivedCast; 2821 SSK = llvm::SanStat_CFI_DerivedCast; 2822 break; 2823 case CFITCK_UnrelatedCast: 2824 M = SanitizerKind::CFIUnrelatedCast; 2825 SSK = llvm::SanStat_CFI_UnrelatedCast; 2826 break; 2827 case CFITCK_ICall: 2828 case CFITCK_NVMFCall: 2829 case CFITCK_VMFCall: 2830 llvm_unreachable("unexpected sanitizer kind"); 2831 } 2832 2833 std::string TypeName = RD->getQualifiedNameAsString(); 2834 if (getContext().getNoSanitizeList().containsType(M, TypeName)) 2835 return; 2836 2837 SanitizerScope SanScope(this); 2838 EmitSanitizerStatReport(SSK); 2839 2840 llvm::Metadata *MD = 2841 CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); 2842 llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD); 2843 2844 llvm::Value *TypeTest = Builder.CreateCall( 2845 CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, TypeId}); 2846 2847 llvm::Constant *StaticData[] = { 2848 llvm::ConstantInt::get(Int8Ty, TCK), 2849 EmitCheckSourceLocation(Loc), 2850 EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)), 2851 }; 2852 2853 auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD); 2854 if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) { 2855 EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, VTable, StaticData); 2856 return; 2857 } 2858 2859 if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) { 2860 EmitTrapCheck(TypeTest, SanitizerHandler::CFICheckFail); 2861 return; 2862 } 2863 2864 llvm::Value *AllVtables = llvm::MetadataAsValue::get( 2865 CGM.getLLVMContext(), 2866 llvm::MDString::get(CGM.getLLVMContext(), "all-vtables")); 2867 llvm::Value *ValidVtable = Builder.CreateCall( 2868 CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables}); 2869 EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail, 2870 StaticData, {VTable, ValidVtable}); 2871 } 2872 2873 bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) { 2874 if (!CGM.getCodeGenOpts().WholeProgramVTables || 2875 !CGM.HasHiddenLTOVisibility(RD)) 2876 return false; 2877 2878 if (CGM.getCodeGenOpts().VirtualFunctionElimination) 2879 return true; 2880 2881 if (!SanOpts.has(SanitizerKind::CFIVCall) || 2882 !CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall)) 2883 return false; 2884 2885 std::string TypeName = RD->getQualifiedNameAsString(); 2886 return !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall, 2887 TypeName); 2888 } 2889 2890 llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad( 2891 const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy, 2892 uint64_t VTableByteOffset) { 2893 SanitizerScope SanScope(this); 2894 2895 EmitSanitizerStatReport(llvm::SanStat_CFI_VCall); 2896 2897 llvm::Metadata *MD = 2898 CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); 2899 llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD); 2900 2901 llvm::Value *CheckedLoad = Builder.CreateCall( 2902 CGM.getIntrinsic(llvm::Intrinsic::type_checked_load), 2903 {VTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset), TypeId}); 2904 llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1); 2905 2906 std::string TypeName = RD->getQualifiedNameAsString(); 2907 if (SanOpts.has(SanitizerKind::CFIVCall) && 2908 !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall, 2909 TypeName)) { 2910 EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall), 2911 SanitizerHandler::CFICheckFail, {}, {}); 2912 } 2913 2914 return Builder.CreateBitCast(Builder.CreateExtractValue(CheckedLoad, 0), 2915 VTableTy); 2916 } 2917 2918 void CodeGenFunction::EmitForwardingCallToLambda( 2919 const CXXMethodDecl *callOperator, CallArgList &callArgs, 2920 const CGFunctionInfo *calleeFnInfo, llvm::Constant *calleePtr) { 2921 // Get the address of the call operator. 2922 if (!calleeFnInfo) 2923 calleeFnInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(callOperator); 2924 2925 if (!calleePtr) 2926 calleePtr = 2927 CGM.GetAddrOfFunction(GlobalDecl(callOperator), 2928 CGM.getTypes().GetFunctionType(*calleeFnInfo)); 2929 2930 // Prepare the return slot. 2931 const FunctionProtoType *FPT = 2932 callOperator->getType()->castAs<FunctionProtoType>(); 2933 QualType resultType = FPT->getReturnType(); 2934 ReturnValueSlot returnSlot; 2935 if (!resultType->isVoidType() && 2936 calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect && 2937 !hasScalarEvaluationKind(calleeFnInfo->getReturnType())) 2938 returnSlot = 2939 ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(), 2940 /*IsUnused=*/false, /*IsExternallyDestructed=*/true); 2941 2942 // We don't need to separately arrange the call arguments because 2943 // the call can't be variadic anyway --- it's impossible to forward 2944 // variadic arguments. 2945 2946 // Now emit our call. 2947 auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator)); 2948 RValue RV = EmitCall(*calleeFnInfo, callee, returnSlot, callArgs); 2949 2950 // If necessary, copy the returned value into the slot. 2951 if (!resultType->isVoidType() && returnSlot.isNull()) { 2952 if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) { 2953 RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal())); 2954 } 2955 EmitReturnOfRValue(RV, resultType); 2956 } else 2957 EmitBranchThroughCleanup(ReturnBlock); 2958 } 2959 2960 void CodeGenFunction::EmitLambdaBlockInvokeBody() { 2961 const BlockDecl *BD = BlockInfo->getBlockDecl(); 2962 const VarDecl *variable = BD->capture_begin()->getVariable(); 2963 const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl(); 2964 const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator(); 2965 2966 if (CallOp->isVariadic()) { 2967 // FIXME: Making this work correctly is nasty because it requires either 2968 // cloning the body of the call operator or making the call operator 2969 // forward. 2970 CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function"); 2971 return; 2972 } 2973 2974 // Start building arguments for forwarding call 2975 CallArgList CallArgs; 2976 2977 QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda)); 2978 Address ThisPtr = GetAddrOfBlockDecl(variable); 2979 CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); 2980 2981 // Add the rest of the parameters. 2982 for (auto *param : BD->parameters()) 2983 EmitDelegateCallArg(CallArgs, param, param->getBeginLoc()); 2984 2985 assert(!Lambda->isGenericLambda() && 2986 "generic lambda interconversion to block not implemented"); 2987 EmitForwardingCallToLambda(CallOp, CallArgs); 2988 } 2989 2990 void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) { 2991 if (MD->isVariadic()) { 2992 // FIXME: Making this work correctly is nasty because it requires either 2993 // cloning the body of the call operator or making the call operator 2994 // forward. 2995 CGM.ErrorUnsupported(MD, "lambda conversion to variadic function"); 2996 return; 2997 } 2998 2999 const CXXRecordDecl *Lambda = MD->getParent(); 3000 3001 // Start building arguments for forwarding call 3002 CallArgList CallArgs; 3003 3004 QualType LambdaType = getContext().getRecordType(Lambda); 3005 QualType ThisType = getContext().getPointerType(LambdaType); 3006 Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture"); 3007 CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType); 3008 3009 EmitLambdaDelegatingInvokeBody(MD, CallArgs); 3010 } 3011 3012 void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD, 3013 CallArgList &CallArgs) { 3014 // Add the rest of the forwarded parameters. 3015 for (auto *Param : MD->parameters()) 3016 EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc()); 3017 3018 const CXXRecordDecl *Lambda = MD->getParent(); 3019 const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator(); 3020 // For a generic lambda, find the corresponding call operator specialization 3021 // to which the call to the static-invoker shall be forwarded. 3022 if (Lambda->isGenericLambda()) { 3023 assert(MD->isFunctionTemplateSpecialization()); 3024 const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs(); 3025 FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate(); 3026 void *InsertPos = nullptr; 3027 FunctionDecl *CorrespondingCallOpSpecialization = 3028 CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos); 3029 assert(CorrespondingCallOpSpecialization); 3030 CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization); 3031 } 3032 3033 // Special lambda forwarding when there are inalloca parameters. 3034 if (hasInAllocaArg(MD)) { 3035 const CGFunctionInfo *ImplFnInfo = nullptr; 3036 llvm::Function *ImplFn = nullptr; 3037 EmitLambdaInAllocaImplFn(CallOp, &ImplFnInfo, &ImplFn); 3038 3039 EmitForwardingCallToLambda(CallOp, CallArgs, ImplFnInfo, ImplFn); 3040 return; 3041 } 3042 3043 EmitForwardingCallToLambda(CallOp, CallArgs); 3044 } 3045 3046 void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) { 3047 if (MD->isVariadic()) { 3048 // FIXME: Making this work correctly is nasty because it requires either 3049 // cloning the body of the call operator or making the call operator forward. 3050 CGM.ErrorUnsupported(MD, "lambda conversion to variadic function"); 3051 return; 3052 } 3053 3054 // Forward %this argument. 3055 CallArgList CallArgs; 3056 QualType LambdaType = getContext().getRecordType(MD->getParent()); 3057 QualType ThisType = getContext().getPointerType(LambdaType); 3058 llvm::Value *ThisArg = CurFn->getArg(0); 3059 CallArgs.add(RValue::get(ThisArg), ThisType); 3060 3061 EmitLambdaDelegatingInvokeBody(MD, CallArgs); 3062 } 3063 3064 void CodeGenFunction::EmitLambdaInAllocaImplFn( 3065 const CXXMethodDecl *CallOp, const CGFunctionInfo **ImplFnInfo, 3066 llvm::Function **ImplFn) { 3067 const CGFunctionInfo &FnInfo = 3068 CGM.getTypes().arrangeCXXMethodDeclaration(CallOp); 3069 llvm::Function *CallOpFn = 3070 cast<llvm::Function>(CGM.GetAddrOfFunction(GlobalDecl(CallOp))); 3071 3072 // Emit function containing the original call op body. __invoke will delegate 3073 // to this function. 3074 SmallVector<CanQualType, 4> ArgTypes; 3075 for (auto I = FnInfo.arg_begin(); I != FnInfo.arg_end(); ++I) 3076 ArgTypes.push_back(I->type); 3077 *ImplFnInfo = &CGM.getTypes().arrangeLLVMFunctionInfo( 3078 FnInfo.getReturnType(), FnInfoOpts::IsDelegateCall, ArgTypes, 3079 FnInfo.getExtInfo(), {}, FnInfo.getRequiredArgs()); 3080 3081 // Create mangled name as if this was a method named __impl. If for some 3082 // reason the name doesn't look as expected then just tack __impl to the 3083 // front. 3084 // TODO: Use the name mangler to produce the right name instead of using 3085 // string replacement. 3086 StringRef CallOpName = CallOpFn->getName(); 3087 std::string ImplName; 3088 if (size_t Pos = CallOpName.find_first_of("<lambda")) 3089 ImplName = ("?__impl@" + CallOpName.drop_front(Pos)).str(); 3090 else 3091 ImplName = ("__impl" + CallOpName).str(); 3092 3093 llvm::Function *Fn = CallOpFn->getParent()->getFunction(ImplName); 3094 if (!Fn) { 3095 Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(**ImplFnInfo), 3096 llvm::GlobalValue::InternalLinkage, ImplName, 3097 CGM.getModule()); 3098 CGM.SetInternalFunctionAttributes(CallOp, Fn, **ImplFnInfo); 3099 3100 const GlobalDecl &GD = GlobalDecl(CallOp); 3101 const auto *D = cast<FunctionDecl>(GD.getDecl()); 3102 CodeGenFunction(CGM).GenerateCode(GD, Fn, **ImplFnInfo); 3103 CGM.SetLLVMFunctionAttributesForDefinition(D, Fn); 3104 } 3105 *ImplFn = Fn; 3106 } 3107