xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGExprAgg.cpp (revision ccb59683b98360afaf5b5bb641a68fea22c68d0b)
1 //===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Aggregate Expr nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCXXABI.h"
14 #include "CGObjCRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "ConstantEmitter.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/ASTContext.h"
20 #include "clang/AST/Attr.h"
21 #include "clang/AST/DeclCXX.h"
22 #include "clang/AST/DeclTemplate.h"
23 #include "clang/AST/StmtVisitor.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/Function.h"
26 #include "llvm/IR/GlobalVariable.h"
27 #include "llvm/IR/IntrinsicInst.h"
28 #include "llvm/IR/Intrinsics.h"
29 using namespace clang;
30 using namespace CodeGen;
31 
32 //===----------------------------------------------------------------------===//
33 //                        Aggregate Expression Emitter
34 //===----------------------------------------------------------------------===//
35 
36 namespace  {
37 class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
38   CodeGenFunction &CGF;
39   CGBuilderTy &Builder;
40   AggValueSlot Dest;
41   bool IsResultUnused;
42 
43   AggValueSlot EnsureSlot(QualType T) {
44     if (!Dest.isIgnored()) return Dest;
45     return CGF.CreateAggTemp(T, "agg.tmp.ensured");
46   }
47   void EnsureDest(QualType T) {
48     if (!Dest.isIgnored()) return;
49     Dest = CGF.CreateAggTemp(T, "agg.tmp.ensured");
50   }
51 
52   // Calls `Fn` with a valid return value slot, potentially creating a temporary
53   // to do so. If a temporary is created, an appropriate copy into `Dest` will
54   // be emitted, as will lifetime markers.
55   //
56   // The given function should take a ReturnValueSlot, and return an RValue that
57   // points to said slot.
58   void withReturnValueSlot(const Expr *E,
59                            llvm::function_ref<RValue(ReturnValueSlot)> Fn);
60 
61 public:
62   AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
63     : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
64     IsResultUnused(IsResultUnused) { }
65 
66   //===--------------------------------------------------------------------===//
67   //                               Utilities
68   //===--------------------------------------------------------------------===//
69 
70   /// EmitAggLoadOfLValue - Given an expression with aggregate type that
71   /// represents a value lvalue, this method emits the address of the lvalue,
72   /// then loads the result into DestPtr.
73   void EmitAggLoadOfLValue(const Expr *E);
74 
75   enum ExprValueKind {
76     EVK_RValue,
77     EVK_NonRValue
78   };
79 
80   /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
81   /// SrcIsRValue is true if source comes from an RValue.
82   void EmitFinalDestCopy(QualType type, const LValue &src,
83                          ExprValueKind SrcValueKind = EVK_NonRValue);
84   void EmitFinalDestCopy(QualType type, RValue src);
85   void EmitCopy(QualType type, const AggValueSlot &dest,
86                 const AggValueSlot &src);
87 
88   void EmitMoveFromReturnSlot(const Expr *E, RValue Src);
89 
90   void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
91                      QualType ArrayQTy, InitListExpr *E);
92 
93   AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
94     if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
95       return AggValueSlot::NeedsGCBarriers;
96     return AggValueSlot::DoesNotNeedGCBarriers;
97   }
98 
99   bool TypeRequiresGCollection(QualType T);
100 
101   //===--------------------------------------------------------------------===//
102   //                            Visitor Methods
103   //===--------------------------------------------------------------------===//
104 
105   void Visit(Expr *E) {
106     ApplyDebugLocation DL(CGF, E);
107     StmtVisitor<AggExprEmitter>::Visit(E);
108   }
109 
110   void VisitStmt(Stmt *S) {
111     CGF.ErrorUnsupported(S, "aggregate expression");
112   }
113   void VisitParenExpr(ParenExpr *PE) { Visit(PE->getSubExpr()); }
114   void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
115     Visit(GE->getResultExpr());
116   }
117   void VisitCoawaitExpr(CoawaitExpr *E) {
118     CGF.EmitCoawaitExpr(*E, Dest, IsResultUnused);
119   }
120   void VisitCoyieldExpr(CoyieldExpr *E) {
121     CGF.EmitCoyieldExpr(*E, Dest, IsResultUnused);
122   }
123   void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); }
124   void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); }
125   void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
126     return Visit(E->getReplacement());
127   }
128 
129   void VisitConstantExpr(ConstantExpr *E) {
130     EnsureDest(E->getType());
131 
132     if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) {
133       CGF.EmitAggregateStore(Result, Dest.getAddress(),
134                              E->getType().isVolatileQualified());
135       return;
136     }
137     return Visit(E->getSubExpr());
138   }
139 
140   // l-values.
141   void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
142   void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(ME); }
143   void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
144   void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
145   void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
146   void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
147     EmitAggLoadOfLValue(E);
148   }
149   void VisitPredefinedExpr(const PredefinedExpr *E) {
150     EmitAggLoadOfLValue(E);
151   }
152 
153   // Operators.
154   void VisitCastExpr(CastExpr *E);
155   void VisitCallExpr(const CallExpr *E);
156   void VisitStmtExpr(const StmtExpr *E);
157   void VisitBinaryOperator(const BinaryOperator *BO);
158   void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
159   void VisitBinAssign(const BinaryOperator *E);
160   void VisitBinComma(const BinaryOperator *E);
161   void VisitBinCmp(const BinaryOperator *E);
162   void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
163     Visit(E->getSemanticForm());
164   }
165 
166   void VisitObjCMessageExpr(ObjCMessageExpr *E);
167   void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
168     EmitAggLoadOfLValue(E);
169   }
170 
171   void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
172   void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
173   void VisitChooseExpr(const ChooseExpr *CE);
174   void VisitInitListExpr(InitListExpr *E);
175   void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
176                               llvm::Value *outerBegin = nullptr);
177   void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
178   void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing.
179   void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
180     CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
181     Visit(DAE->getExpr());
182   }
183   void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
184     CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
185     Visit(DIE->getExpr());
186   }
187   void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
188   void VisitCXXConstructExpr(const CXXConstructExpr *E);
189   void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
190   void VisitLambdaExpr(LambdaExpr *E);
191   void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
192   void VisitExprWithCleanups(ExprWithCleanups *E);
193   void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
194   void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
195   void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
196   void VisitOpaqueValueExpr(OpaqueValueExpr *E);
197 
198   void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
199     if (E->isGLValue()) {
200       LValue LV = CGF.EmitPseudoObjectLValue(E);
201       return EmitFinalDestCopy(E->getType(), LV);
202     }
203 
204     CGF.EmitPseudoObjectRValue(E, EnsureSlot(E->getType()));
205   }
206 
207   void VisitVAArgExpr(VAArgExpr *E);
208 
209   void EmitInitializationToLValue(Expr *E, LValue Address);
210   void EmitNullInitializationToLValue(LValue Address);
211   //  case Expr::ChooseExprClass:
212   void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
213   void VisitAtomicExpr(AtomicExpr *E) {
214     RValue Res = CGF.EmitAtomicExpr(E);
215     EmitFinalDestCopy(E->getType(), Res);
216   }
217 };
218 }  // end anonymous namespace.
219 
220 //===----------------------------------------------------------------------===//
221 //                                Utilities
222 //===----------------------------------------------------------------------===//
223 
224 /// EmitAggLoadOfLValue - Given an expression with aggregate type that
225 /// represents a value lvalue, this method emits the address of the lvalue,
226 /// then loads the result into DestPtr.
227 void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
228   LValue LV = CGF.EmitLValue(E);
229 
230   // If the type of the l-value is atomic, then do an atomic load.
231   if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) {
232     CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest);
233     return;
234   }
235 
236   EmitFinalDestCopy(E->getType(), LV);
237 }
238 
239 /// True if the given aggregate type requires special GC API calls.
240 bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
241   // Only record types have members that might require garbage collection.
242   const RecordType *RecordTy = T->getAs<RecordType>();
243   if (!RecordTy) return false;
244 
245   // Don't mess with non-trivial C++ types.
246   RecordDecl *Record = RecordTy->getDecl();
247   if (isa<CXXRecordDecl>(Record) &&
248       (cast<CXXRecordDecl>(Record)->hasNonTrivialCopyConstructor() ||
249        !cast<CXXRecordDecl>(Record)->hasTrivialDestructor()))
250     return false;
251 
252   // Check whether the type has an object member.
253   return Record->hasObjectMember();
254 }
255 
256 void AggExprEmitter::withReturnValueSlot(
257     const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
258   QualType RetTy = E->getType();
259   bool RequiresDestruction =
260       !Dest.isExternallyDestructed() &&
261       RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
262 
263   // If it makes no observable difference, save a memcpy + temporary.
264   //
265   // We need to always provide our own temporary if destruction is required.
266   // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
267   // its lifetime before we have the chance to emit a proper destructor call.
268   bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
269                  (RequiresDestruction && !Dest.getAddress().isValid());
270 
271   Address RetAddr = Address::invalid();
272   Address RetAllocaAddr = Address::invalid();
273 
274   EHScopeStack::stable_iterator LifetimeEndBlock;
275   llvm::Value *LifetimeSizePtr = nullptr;
276   llvm::IntrinsicInst *LifetimeStartInst = nullptr;
277   if (!UseTemp) {
278     RetAddr = Dest.getAddress();
279   } else {
280     RetAddr = CGF.CreateMemTemp(RetTy, "tmp", &RetAllocaAddr);
281     llvm::TypeSize Size =
282         CGF.CGM.getDataLayout().getTypeAllocSize(CGF.ConvertTypeForMem(RetTy));
283     LifetimeSizePtr = CGF.EmitLifetimeStart(Size, RetAllocaAddr.getPointer());
284     if (LifetimeSizePtr) {
285       LifetimeStartInst =
286           cast<llvm::IntrinsicInst>(std::prev(Builder.GetInsertPoint()));
287       assert(LifetimeStartInst->getIntrinsicID() ==
288                  llvm::Intrinsic::lifetime_start &&
289              "Last insertion wasn't a lifetime.start?");
290 
291       CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
292           NormalEHLifetimeMarker, RetAllocaAddr, LifetimeSizePtr);
293       LifetimeEndBlock = CGF.EHStack.stable_begin();
294     }
295   }
296 
297   RValue Src =
298       EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
299                                Dest.isExternallyDestructed()));
300 
301   if (!UseTemp)
302     return;
303 
304   assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer());
305   EmitFinalDestCopy(E->getType(), Src);
306 
307   if (!RequiresDestruction && LifetimeStartInst) {
308     // If there's no dtor to run, the copy was the last use of our temporary.
309     // Since we're not guaranteed to be in an ExprWithCleanups, clean up
310     // eagerly.
311     CGF.DeactivateCleanupBlock(LifetimeEndBlock, LifetimeStartInst);
312     CGF.EmitLifetimeEnd(LifetimeSizePtr, RetAllocaAddr.getPointer());
313   }
314 }
315 
316 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
317 void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
318   assert(src.isAggregate() && "value must be aggregate value!");
319   LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type);
320   EmitFinalDestCopy(type, srcLV, EVK_RValue);
321 }
322 
323 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
324 void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src,
325                                        ExprValueKind SrcValueKind) {
326   // If Dest is ignored, then we're evaluating an aggregate expression
327   // in a context that doesn't care about the result.  Note that loads
328   // from volatile l-values force the existence of a non-ignored
329   // destination.
330   if (Dest.isIgnored())
331     return;
332 
333   // Copy non-trivial C structs here.
334   LValue DstLV = CGF.MakeAddrLValue(
335       Dest.getAddress(), Dest.isVolatile() ? type.withVolatile() : type);
336 
337   if (SrcValueKind == EVK_RValue) {
338     if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
339       if (Dest.isPotentiallyAliased())
340         CGF.callCStructMoveAssignmentOperator(DstLV, src);
341       else
342         CGF.callCStructMoveConstructor(DstLV, src);
343       return;
344     }
345   } else {
346     if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
347       if (Dest.isPotentiallyAliased())
348         CGF.callCStructCopyAssignmentOperator(DstLV, src);
349       else
350         CGF.callCStructCopyConstructor(DstLV, src);
351       return;
352     }
353   }
354 
355   AggValueSlot srcAgg = AggValueSlot::forLValue(
356       src, CGF, AggValueSlot::IsDestructed, needsGC(type),
357       AggValueSlot::IsAliased, AggValueSlot::MayOverlap);
358   EmitCopy(type, Dest, srcAgg);
359 }
360 
361 /// Perform a copy from the source into the destination.
362 ///
363 /// \param type - the type of the aggregate being copied; qualifiers are
364 ///   ignored
365 void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
366                               const AggValueSlot &src) {
367   if (dest.requiresGCollection()) {
368     CharUnits sz = dest.getPreferredSize(CGF.getContext(), type);
369     llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity());
370     CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF,
371                                                       dest.getAddress(),
372                                                       src.getAddress(),
373                                                       size);
374     return;
375   }
376 
377   // If the result of the assignment is used, copy the LHS there also.
378   // It's volatile if either side is.  Use the minimum alignment of
379   // the two sides.
380   LValue DestLV = CGF.MakeAddrLValue(dest.getAddress(), type);
381   LValue SrcLV = CGF.MakeAddrLValue(src.getAddress(), type);
382   CGF.EmitAggregateCopy(DestLV, SrcLV, type, dest.mayOverlap(),
383                         dest.isVolatile() || src.isVolatile());
384 }
385 
386 /// Emit the initializer for a std::initializer_list initialized with a
387 /// real initializer list.
388 void
389 AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
390   // Emit an array containing the elements.  The array is externally destructed
391   // if the std::initializer_list object is.
392   ASTContext &Ctx = CGF.getContext();
393   LValue Array = CGF.EmitLValue(E->getSubExpr());
394   assert(Array.isSimple() && "initializer_list array not a simple lvalue");
395   Address ArrayPtr = Array.getAddress(CGF);
396 
397   const ConstantArrayType *ArrayType =
398       Ctx.getAsConstantArrayType(E->getSubExpr()->getType());
399   assert(ArrayType && "std::initializer_list constructed from non-array");
400 
401   // FIXME: Perform the checks on the field types in SemaInit.
402   RecordDecl *Record = E->getType()->castAs<RecordType>()->getDecl();
403   RecordDecl::field_iterator Field = Record->field_begin();
404   if (Field == Record->field_end()) {
405     CGF.ErrorUnsupported(E, "weird std::initializer_list");
406     return;
407   }
408 
409   // Start pointer.
410   if (!Field->getType()->isPointerType() ||
411       !Ctx.hasSameType(Field->getType()->getPointeeType(),
412                        ArrayType->getElementType())) {
413     CGF.ErrorUnsupported(E, "weird std::initializer_list");
414     return;
415   }
416 
417   AggValueSlot Dest = EnsureSlot(E->getType());
418   LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
419   LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field);
420   llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0);
421   llvm::Value *IdxStart[] = { Zero, Zero };
422   llvm::Value *ArrayStart = Builder.CreateInBoundsGEP(
423       ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxStart, "arraystart");
424   CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start);
425   ++Field;
426 
427   if (Field == Record->field_end()) {
428     CGF.ErrorUnsupported(E, "weird std::initializer_list");
429     return;
430   }
431 
432   llvm::Value *Size = Builder.getInt(ArrayType->getSize());
433   LValue EndOrLength = CGF.EmitLValueForFieldInitialization(DestLV, *Field);
434   if (Field->getType()->isPointerType() &&
435       Ctx.hasSameType(Field->getType()->getPointeeType(),
436                       ArrayType->getElementType())) {
437     // End pointer.
438     llvm::Value *IdxEnd[] = { Zero, Size };
439     llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
440         ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxEnd, "arrayend");
441     CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength);
442   } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) {
443     // Length.
444     CGF.EmitStoreThroughLValue(RValue::get(Size), EndOrLength);
445   } else {
446     CGF.ErrorUnsupported(E, "weird std::initializer_list");
447     return;
448   }
449 }
450 
451 /// Determine if E is a trivial array filler, that is, one that is
452 /// equivalent to zero-initialization.
453 static bool isTrivialFiller(Expr *E) {
454   if (!E)
455     return true;
456 
457   if (isa<ImplicitValueInitExpr>(E))
458     return true;
459 
460   if (auto *ILE = dyn_cast<InitListExpr>(E)) {
461     if (ILE->getNumInits())
462       return false;
463     return isTrivialFiller(ILE->getArrayFiller());
464   }
465 
466   if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(E))
467     return Cons->getConstructor()->isDefaultConstructor() &&
468            Cons->getConstructor()->isTrivial();
469 
470   // FIXME: Are there other cases where we can avoid emitting an initializer?
471   return false;
472 }
473 
474 /// Emit initialization of an array from an initializer list.
475 void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
476                                    QualType ArrayQTy, InitListExpr *E) {
477   uint64_t NumInitElements = E->getNumInits();
478 
479   uint64_t NumArrayElements = AType->getNumElements();
480   assert(NumInitElements <= NumArrayElements);
481 
482   QualType elementType =
483       CGF.getContext().getAsArrayType(ArrayQTy)->getElementType();
484 
485   // DestPtr is an array*.  Construct an elementType* by drilling
486   // down a level.
487   llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
488   llvm::Value *indices[] = { zero, zero };
489   llvm::Value *begin = Builder.CreateInBoundsGEP(
490       DestPtr.getElementType(), DestPtr.getPointer(), indices,
491       "arrayinit.begin");
492 
493   CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
494   CharUnits elementAlign =
495     DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
496   llvm::Type *llvmElementType = CGF.ConvertTypeForMem(elementType);
497 
498   // Consider initializing the array by copying from a global. For this to be
499   // more efficient than per-element initialization, the size of the elements
500   // with explicit initializers should be large enough.
501   if (NumInitElements * elementSize.getQuantity() > 16 &&
502       elementType.isTriviallyCopyableType(CGF.getContext())) {
503     CodeGen::CodeGenModule &CGM = CGF.CGM;
504     ConstantEmitter Emitter(CGF);
505     LangAS AS = ArrayQTy.getAddressSpace();
506     if (llvm::Constant *C = Emitter.tryEmitForInitializer(E, AS, ArrayQTy)) {
507       auto GV = new llvm::GlobalVariable(
508           CGM.getModule(), C->getType(),
509           CGM.isTypeConstant(ArrayQTy, /* ExcludeCtorDtor= */ true),
510           llvm::GlobalValue::PrivateLinkage, C, "constinit",
511           /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
512           CGM.getContext().getTargetAddressSpace(AS));
513       Emitter.finalize(GV);
514       CharUnits Align = CGM.getContext().getTypeAlignInChars(ArrayQTy);
515       GV->setAlignment(Align.getAsAlign());
516       Address GVAddr(GV, GV->getValueType(), Align);
517       EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GVAddr, ArrayQTy));
518       return;
519     }
520   }
521 
522   // Exception safety requires us to destroy all the
523   // already-constructed members if an initializer throws.
524   // For that, we'll need an EH cleanup.
525   QualType::DestructionKind dtorKind = elementType.isDestructedType();
526   Address endOfInit = Address::invalid();
527   EHScopeStack::stable_iterator cleanup;
528   llvm::Instruction *cleanupDominator = nullptr;
529   if (CGF.needsEHCleanup(dtorKind)) {
530     // In principle we could tell the cleanup where we are more
531     // directly, but the control flow can get so varied here that it
532     // would actually be quite complex.  Therefore we go through an
533     // alloca.
534     endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(),
535                                      "arrayinit.endOfInit");
536     cleanupDominator = Builder.CreateStore(begin, endOfInit);
537     CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType,
538                                          elementAlign,
539                                          CGF.getDestroyer(dtorKind));
540     cleanup = CGF.EHStack.stable_begin();
541 
542   // Otherwise, remember that we didn't need a cleanup.
543   } else {
544     dtorKind = QualType::DK_none;
545   }
546 
547   llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1);
548 
549   // The 'current element to initialize'.  The invariants on this
550   // variable are complicated.  Essentially, after each iteration of
551   // the loop, it points to the last initialized element, except
552   // that it points to the beginning of the array before any
553   // elements have been initialized.
554   llvm::Value *element = begin;
555 
556   // Emit the explicit initializers.
557   for (uint64_t i = 0; i != NumInitElements; ++i) {
558     // Advance to the next element.
559     if (i > 0) {
560       element = Builder.CreateInBoundsGEP(
561           llvmElementType, element, one, "arrayinit.element");
562 
563       // Tell the cleanup that it needs to destroy up to this
564       // element.  TODO: some of these stores can be trivially
565       // observed to be unnecessary.
566       if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
567     }
568 
569     LValue elementLV = CGF.MakeAddrLValue(
570         Address(element, llvmElementType, elementAlign), elementType);
571     EmitInitializationToLValue(E->getInit(i), elementLV);
572   }
573 
574   // Check whether there's a non-trivial array-fill expression.
575   Expr *filler = E->getArrayFiller();
576   bool hasTrivialFiller = isTrivialFiller(filler);
577 
578   // Any remaining elements need to be zero-initialized, possibly
579   // using the filler expression.  We can skip this if the we're
580   // emitting to zeroed memory.
581   if (NumInitElements != NumArrayElements &&
582       !(Dest.isZeroed() && hasTrivialFiller &&
583         CGF.getTypes().isZeroInitializable(elementType))) {
584 
585     // Use an actual loop.  This is basically
586     //   do { *array++ = filler; } while (array != end);
587 
588     // Advance to the start of the rest of the array.
589     if (NumInitElements) {
590       element = Builder.CreateInBoundsGEP(
591           llvmElementType, element, one, "arrayinit.start");
592       if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
593     }
594 
595     // Compute the end of the array.
596     llvm::Value *end = Builder.CreateInBoundsGEP(
597         llvmElementType, begin,
598         llvm::ConstantInt::get(CGF.SizeTy, NumArrayElements), "arrayinit.end");
599 
600     llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
601     llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body");
602 
603     // Jump into the body.
604     CGF.EmitBlock(bodyBB);
605     llvm::PHINode *currentElement =
606       Builder.CreatePHI(element->getType(), 2, "arrayinit.cur");
607     currentElement->addIncoming(element, entryBB);
608 
609     // Emit the actual filler expression.
610     {
611       // C++1z [class.temporary]p5:
612       //   when a default constructor is called to initialize an element of
613       //   an array with no corresponding initializer [...] the destruction of
614       //   every temporary created in a default argument is sequenced before
615       //   the construction of the next array element, if any
616       CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
617       LValue elementLV = CGF.MakeAddrLValue(
618           Address(currentElement, llvmElementType, elementAlign), elementType);
619       if (filler)
620         EmitInitializationToLValue(filler, elementLV);
621       else
622         EmitNullInitializationToLValue(elementLV);
623     }
624 
625     // Move on to the next element.
626     llvm::Value *nextElement = Builder.CreateInBoundsGEP(
627         llvmElementType, currentElement, one, "arrayinit.next");
628 
629     // Tell the EH cleanup that we finished with the last element.
630     if (endOfInit.isValid()) Builder.CreateStore(nextElement, endOfInit);
631 
632     // Leave the loop if we're done.
633     llvm::Value *done = Builder.CreateICmpEQ(nextElement, end,
634                                              "arrayinit.done");
635     llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end");
636     Builder.CreateCondBr(done, endBB, bodyBB);
637     currentElement->addIncoming(nextElement, Builder.GetInsertBlock());
638 
639     CGF.EmitBlock(endBB);
640   }
641 
642   // Leave the partial-array cleanup if we entered one.
643   if (dtorKind) CGF.DeactivateCleanupBlock(cleanup, cleanupDominator);
644 }
645 
646 //===----------------------------------------------------------------------===//
647 //                            Visitor Methods
648 //===----------------------------------------------------------------------===//
649 
650 void AggExprEmitter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E){
651   Visit(E->getSubExpr());
652 }
653 
654 void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
655   // If this is a unique OVE, just visit its source expression.
656   if (e->isUnique())
657     Visit(e->getSourceExpr());
658   else
659     EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e));
660 }
661 
662 void
663 AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
664   if (Dest.isPotentiallyAliased() &&
665       E->getType().isPODType(CGF.getContext())) {
666     // For a POD type, just emit a load of the lvalue + a copy, because our
667     // compound literal might alias the destination.
668     EmitAggLoadOfLValue(E);
669     return;
670   }
671 
672   AggValueSlot Slot = EnsureSlot(E->getType());
673 
674   // Block-scope compound literals are destroyed at the end of the enclosing
675   // scope in C.
676   bool Destruct =
677       !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
678   if (Destruct)
679     Slot.setExternallyDestructed();
680 
681   CGF.EmitAggExpr(E->getInitializer(), Slot);
682 
683   if (Destruct)
684     if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
685       CGF.pushLifetimeExtendedDestroy(
686           CGF.getCleanupKind(DtorKind), Slot.getAddress(), E->getType(),
687           CGF.getDestroyer(DtorKind), DtorKind & EHCleanup);
688 }
689 
690 /// Attempt to look through various unimportant expressions to find a
691 /// cast of the given kind.
692 static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
693   op = op->IgnoreParenNoopCasts(ctx);
694   if (auto castE = dyn_cast<CastExpr>(op)) {
695     if (castE->getCastKind() == kind)
696       return castE->getSubExpr();
697   }
698   return nullptr;
699 }
700 
701 void AggExprEmitter::VisitCastExpr(CastExpr *E) {
702   if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
703     CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
704   switch (E->getCastKind()) {
705   case CK_Dynamic: {
706     // FIXME: Can this actually happen? We have no test coverage for it.
707     assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
708     LValue LV = CGF.EmitCheckedLValue(E->getSubExpr(),
709                                       CodeGenFunction::TCK_Load);
710     // FIXME: Do we also need to handle property references here?
711     if (LV.isSimple())
712       CGF.EmitDynamicCast(LV.getAddress(CGF), cast<CXXDynamicCastExpr>(E));
713     else
714       CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast");
715 
716     if (!Dest.isIgnored())
717       CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination");
718     break;
719   }
720 
721   case CK_ToUnion: {
722     // Evaluate even if the destination is ignored.
723     if (Dest.isIgnored()) {
724       CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(),
725                       /*ignoreResult=*/true);
726       break;
727     }
728 
729     // GCC union extension
730     QualType Ty = E->getSubExpr()->getType();
731     Address CastPtr =
732       Builder.CreateElementBitCast(Dest.getAddress(), CGF.ConvertType(Ty));
733     EmitInitializationToLValue(E->getSubExpr(),
734                                CGF.MakeAddrLValue(CastPtr, Ty));
735     break;
736   }
737 
738   case CK_LValueToRValueBitCast: {
739     if (Dest.isIgnored()) {
740       CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(),
741                       /*ignoreResult=*/true);
742       break;
743     }
744 
745     LValue SourceLV = CGF.EmitLValue(E->getSubExpr());
746     Address SourceAddress =
747         Builder.CreateElementBitCast(SourceLV.getAddress(CGF), CGF.Int8Ty);
748     Address DestAddress =
749         Builder.CreateElementBitCast(Dest.getAddress(), CGF.Int8Ty);
750     llvm::Value *SizeVal = llvm::ConstantInt::get(
751         CGF.SizeTy,
752         CGF.getContext().getTypeSizeInChars(E->getType()).getQuantity());
753     Builder.CreateMemCpy(DestAddress, SourceAddress, SizeVal);
754     break;
755   }
756 
757   case CK_DerivedToBase:
758   case CK_BaseToDerived:
759   case CK_UncheckedDerivedToBase: {
760     llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
761                 "should have been unpacked before we got here");
762   }
763 
764   case CK_NonAtomicToAtomic:
765   case CK_AtomicToNonAtomic: {
766     bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
767 
768     // Determine the atomic and value types.
769     QualType atomicType = E->getSubExpr()->getType();
770     QualType valueType = E->getType();
771     if (isToAtomic) std::swap(atomicType, valueType);
772 
773     assert(atomicType->isAtomicType());
774     assert(CGF.getContext().hasSameUnqualifiedType(valueType,
775                           atomicType->castAs<AtomicType>()->getValueType()));
776 
777     // Just recurse normally if we're ignoring the result or the
778     // atomic type doesn't change representation.
779     if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(atomicType)) {
780       return Visit(E->getSubExpr());
781     }
782 
783     CastKind peepholeTarget =
784       (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
785 
786     // These two cases are reverses of each other; try to peephole them.
787     if (Expr *op =
788             findPeephole(E->getSubExpr(), peepholeTarget, CGF.getContext())) {
789       assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
790                                                      E->getType()) &&
791            "peephole significantly changed types?");
792       return Visit(op);
793     }
794 
795     // If we're converting an r-value of non-atomic type to an r-value
796     // of atomic type, just emit directly into the relevant sub-object.
797     if (isToAtomic) {
798       AggValueSlot valueDest = Dest;
799       if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(atomicType)) {
800         // Zero-initialize.  (Strictly speaking, we only need to initialize
801         // the padding at the end, but this is simpler.)
802         if (!Dest.isZeroed())
803           CGF.EmitNullInitialization(Dest.getAddress(), atomicType);
804 
805         // Build a GEP to refer to the subobject.
806         Address valueAddr =
807             CGF.Builder.CreateStructGEP(valueDest.getAddress(), 0);
808         valueDest = AggValueSlot::forAddr(valueAddr,
809                                           valueDest.getQualifiers(),
810                                           valueDest.isExternallyDestructed(),
811                                           valueDest.requiresGCollection(),
812                                           valueDest.isPotentiallyAliased(),
813                                           AggValueSlot::DoesNotOverlap,
814                                           AggValueSlot::IsZeroed);
815       }
816 
817       CGF.EmitAggExpr(E->getSubExpr(), valueDest);
818       return;
819     }
820 
821     // Otherwise, we're converting an atomic type to a non-atomic type.
822     // Make an atomic temporary, emit into that, and then copy the value out.
823     AggValueSlot atomicSlot =
824       CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp");
825     CGF.EmitAggExpr(E->getSubExpr(), atomicSlot);
826 
827     Address valueAddr = Builder.CreateStructGEP(atomicSlot.getAddress(), 0);
828     RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile());
829     return EmitFinalDestCopy(valueType, rvalue);
830   }
831   case CK_AddressSpaceConversion:
832      return Visit(E->getSubExpr());
833 
834   case CK_LValueToRValue:
835     // If we're loading from a volatile type, force the destination
836     // into existence.
837     if (E->getSubExpr()->getType().isVolatileQualified()) {
838       bool Destruct =
839           !Dest.isExternallyDestructed() &&
840           E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
841       if (Destruct)
842         Dest.setExternallyDestructed();
843       EnsureDest(E->getType());
844       Visit(E->getSubExpr());
845 
846       if (Destruct)
847         CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
848                         E->getType());
849 
850       return;
851     }
852 
853     LLVM_FALLTHROUGH;
854 
855 
856   case CK_NoOp:
857   case CK_UserDefinedConversion:
858   case CK_ConstructorConversion:
859     assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
860                                                    E->getType()) &&
861            "Implicit cast types must be compatible");
862     Visit(E->getSubExpr());
863     break;
864 
865   case CK_LValueBitCast:
866     llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
867 
868   case CK_Dependent:
869   case CK_BitCast:
870   case CK_ArrayToPointerDecay:
871   case CK_FunctionToPointerDecay:
872   case CK_NullToPointer:
873   case CK_NullToMemberPointer:
874   case CK_BaseToDerivedMemberPointer:
875   case CK_DerivedToBaseMemberPointer:
876   case CK_MemberPointerToBoolean:
877   case CK_ReinterpretMemberPointer:
878   case CK_IntegralToPointer:
879   case CK_PointerToIntegral:
880   case CK_PointerToBoolean:
881   case CK_ToVoid:
882   case CK_VectorSplat:
883   case CK_IntegralCast:
884   case CK_BooleanToSignedIntegral:
885   case CK_IntegralToBoolean:
886   case CK_IntegralToFloating:
887   case CK_FloatingToIntegral:
888   case CK_FloatingToBoolean:
889   case CK_FloatingCast:
890   case CK_CPointerToObjCPointerCast:
891   case CK_BlockPointerToObjCPointerCast:
892   case CK_AnyPointerToBlockPointerCast:
893   case CK_ObjCObjectLValueCast:
894   case CK_FloatingRealToComplex:
895   case CK_FloatingComplexToReal:
896   case CK_FloatingComplexToBoolean:
897   case CK_FloatingComplexCast:
898   case CK_FloatingComplexToIntegralComplex:
899   case CK_IntegralRealToComplex:
900   case CK_IntegralComplexToReal:
901   case CK_IntegralComplexToBoolean:
902   case CK_IntegralComplexCast:
903   case CK_IntegralComplexToFloatingComplex:
904   case CK_ARCProduceObject:
905   case CK_ARCConsumeObject:
906   case CK_ARCReclaimReturnedObject:
907   case CK_ARCExtendBlockObject:
908   case CK_CopyAndAutoreleaseBlockObject:
909   case CK_BuiltinFnToFnPtr:
910   case CK_ZeroToOCLOpaqueType:
911   case CK_MatrixCast:
912 
913   case CK_IntToOCLSampler:
914   case CK_FloatingToFixedPoint:
915   case CK_FixedPointToFloating:
916   case CK_FixedPointCast:
917   case CK_FixedPointToBoolean:
918   case CK_FixedPointToIntegral:
919   case CK_IntegralToFixedPoint:
920     llvm_unreachable("cast kind invalid for aggregate types");
921   }
922 }
923 
924 void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
925   if (E->getCallReturnType(CGF.getContext())->isReferenceType()) {
926     EmitAggLoadOfLValue(E);
927     return;
928   }
929 
930   withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
931     return CGF.EmitCallExpr(E, Slot);
932   });
933 }
934 
935 void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
936   withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
937     return CGF.EmitObjCMessageExpr(E, Slot);
938   });
939 }
940 
941 void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
942   CGF.EmitIgnoredExpr(E->getLHS());
943   Visit(E->getRHS());
944 }
945 
946 void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
947   CodeGenFunction::StmtExprEvaluation eval(CGF);
948   CGF.EmitCompoundStmt(*E->getSubStmt(), true, Dest);
949 }
950 
951 enum CompareKind {
952   CK_Less,
953   CK_Greater,
954   CK_Equal,
955 };
956 
957 static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
958                                 const BinaryOperator *E, llvm::Value *LHS,
959                                 llvm::Value *RHS, CompareKind Kind,
960                                 const char *NameSuffix = "") {
961   QualType ArgTy = E->getLHS()->getType();
962   if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
963     ArgTy = CT->getElementType();
964 
965   if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
966     assert(Kind == CK_Equal &&
967            "member pointers may only be compared for equality");
968     return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
969         CGF, LHS, RHS, MPT, /*IsInequality*/ false);
970   }
971 
972   // Compute the comparison instructions for the specified comparison kind.
973   struct CmpInstInfo {
974     const char *Name;
975     llvm::CmpInst::Predicate FCmp;
976     llvm::CmpInst::Predicate SCmp;
977     llvm::CmpInst::Predicate UCmp;
978   };
979   CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
980     using FI = llvm::FCmpInst;
981     using II = llvm::ICmpInst;
982     switch (Kind) {
983     case CK_Less:
984       return {"cmp.lt", FI::FCMP_OLT, II::ICMP_SLT, II::ICMP_ULT};
985     case CK_Greater:
986       return {"cmp.gt", FI::FCMP_OGT, II::ICMP_SGT, II::ICMP_UGT};
987     case CK_Equal:
988       return {"cmp.eq", FI::FCMP_OEQ, II::ICMP_EQ, II::ICMP_EQ};
989     }
990     llvm_unreachable("Unrecognised CompareKind enum");
991   }();
992 
993   if (ArgTy->hasFloatingRepresentation())
994     return Builder.CreateFCmp(InstInfo.FCmp, LHS, RHS,
995                               llvm::Twine(InstInfo.Name) + NameSuffix);
996   if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
997     auto Inst =
998         ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
999     return Builder.CreateICmp(Inst, LHS, RHS,
1000                               llvm::Twine(InstInfo.Name) + NameSuffix);
1001   }
1002 
1003   llvm_unreachable("unsupported aggregate binary expression should have "
1004                    "already been handled");
1005 }
1006 
1007 void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1008   using llvm::BasicBlock;
1009   using llvm::PHINode;
1010   using llvm::Value;
1011   assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1012                                       E->getRHS()->getType()));
1013   const ComparisonCategoryInfo &CmpInfo =
1014       CGF.getContext().CompCategories.getInfoForType(E->getType());
1015   assert(CmpInfo.Record->isTriviallyCopyable() &&
1016          "cannot copy non-trivially copyable aggregate");
1017 
1018   QualType ArgTy = E->getLHS()->getType();
1019 
1020   if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1021       !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1022       !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1023     return CGF.ErrorUnsupported(E, "aggregate three-way comparison");
1024   }
1025   bool IsComplex = ArgTy->isAnyComplexType();
1026 
1027   // Evaluate the operands to the expression and extract their values.
1028   auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1029     RValue RV = CGF.EmitAnyExpr(E);
1030     if (RV.isScalar())
1031       return {RV.getScalarVal(), nullptr};
1032     if (RV.isAggregate())
1033       return {RV.getAggregatePointer(), nullptr};
1034     assert(RV.isComplex());
1035     return RV.getComplexVal();
1036   };
1037   auto LHSValues = EmitOperand(E->getLHS()),
1038        RHSValues = EmitOperand(E->getRHS());
1039 
1040   auto EmitCmp = [&](CompareKind K) {
1041     Value *Cmp = EmitCompare(Builder, CGF, E, LHSValues.first, RHSValues.first,
1042                              K, IsComplex ? ".r" : "");
1043     if (!IsComplex)
1044       return Cmp;
1045     assert(K == CompareKind::CK_Equal);
1046     Value *CmpImag = EmitCompare(Builder, CGF, E, LHSValues.second,
1047                                  RHSValues.second, K, ".i");
1048     return Builder.CreateAnd(Cmp, CmpImag, "and.eq");
1049   };
1050   auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1051     return Builder.getInt(VInfo->getIntValue());
1052   };
1053 
1054   Value *Select;
1055   if (ArgTy->isNullPtrType()) {
1056     Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1057   } else if (!CmpInfo.isPartial()) {
1058     Value *SelectOne =
1059         Builder.CreateSelect(EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()),
1060                              EmitCmpRes(CmpInfo.getGreater()), "sel.lt");
1061     Select = Builder.CreateSelect(EmitCmp(CK_Equal),
1062                                   EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1063                                   SelectOne, "sel.eq");
1064   } else {
1065     Value *SelectEq = Builder.CreateSelect(
1066         EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1067         EmitCmpRes(CmpInfo.getUnordered()), "sel.eq");
1068     Value *SelectGT = Builder.CreateSelect(EmitCmp(CK_Greater),
1069                                            EmitCmpRes(CmpInfo.getGreater()),
1070                                            SelectEq, "sel.gt");
1071     Select = Builder.CreateSelect(
1072         EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), SelectGT, "sel.lt");
1073   }
1074   // Create the return value in the destination slot.
1075   EnsureDest(E->getType());
1076   LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1077 
1078   // Emit the address of the first (and only) field in the comparison category
1079   // type, and initialize it from the constant integer value selected above.
1080   LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1081       DestLV, *CmpInfo.Record->field_begin());
1082   CGF.EmitStoreThroughLValue(RValue::get(Select), FieldLV, /*IsInit*/ true);
1083 
1084   // All done! The result is in the Dest slot.
1085 }
1086 
1087 void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1088   if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1089     VisitPointerToDataMemberBinaryOperator(E);
1090   else
1091     CGF.ErrorUnsupported(E, "aggregate binary expression");
1092 }
1093 
1094 void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1095                                                     const BinaryOperator *E) {
1096   LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1097   EmitFinalDestCopy(E->getType(), LV);
1098 }
1099 
1100 /// Is the value of the given expression possibly a reference to or
1101 /// into a __block variable?
1102 static bool isBlockVarRef(const Expr *E) {
1103   // Make sure we look through parens.
1104   E = E->IgnoreParens();
1105 
1106   // Check for a direct reference to a __block variable.
1107   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {
1108     const VarDecl *var = dyn_cast<VarDecl>(DRE->getDecl());
1109     return (var && var->hasAttr<BlocksAttr>());
1110   }
1111 
1112   // More complicated stuff.
1113 
1114   // Binary operators.
1115   if (const BinaryOperator *op = dyn_cast<BinaryOperator>(E)) {
1116     // For an assignment or pointer-to-member operation, just care
1117     // about the LHS.
1118     if (op->isAssignmentOp() || op->isPtrMemOp())
1119       return isBlockVarRef(op->getLHS());
1120 
1121     // For a comma, just care about the RHS.
1122     if (op->getOpcode() == BO_Comma)
1123       return isBlockVarRef(op->getRHS());
1124 
1125     // FIXME: pointer arithmetic?
1126     return false;
1127 
1128   // Check both sides of a conditional operator.
1129   } else if (const AbstractConditionalOperator *op
1130                = dyn_cast<AbstractConditionalOperator>(E)) {
1131     return isBlockVarRef(op->getTrueExpr())
1132         || isBlockVarRef(op->getFalseExpr());
1133 
1134   // OVEs are required to support BinaryConditionalOperators.
1135   } else if (const OpaqueValueExpr *op
1136                = dyn_cast<OpaqueValueExpr>(E)) {
1137     if (const Expr *src = op->getSourceExpr())
1138       return isBlockVarRef(src);
1139 
1140   // Casts are necessary to get things like (*(int*)&var) = foo().
1141   // We don't really care about the kind of cast here, except
1142   // we don't want to look through l2r casts, because it's okay
1143   // to get the *value* in a __block variable.
1144   } else if (const CastExpr *cast = dyn_cast<CastExpr>(E)) {
1145     if (cast->getCastKind() == CK_LValueToRValue)
1146       return false;
1147     return isBlockVarRef(cast->getSubExpr());
1148 
1149   // Handle unary operators.  Again, just aggressively look through
1150   // it, ignoring the operation.
1151   } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E)) {
1152     return isBlockVarRef(uop->getSubExpr());
1153 
1154   // Look into the base of a field access.
1155   } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(E)) {
1156     return isBlockVarRef(mem->getBase());
1157 
1158   // Look into the base of a subscript.
1159   } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(E)) {
1160     return isBlockVarRef(sub->getBase());
1161   }
1162 
1163   return false;
1164 }
1165 
1166 void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1167   // For an assignment to work, the value on the right has
1168   // to be compatible with the value on the left.
1169   assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1170                                                  E->getRHS()->getType())
1171          && "Invalid assignment");
1172 
1173   // If the LHS might be a __block variable, and the RHS can
1174   // potentially cause a block copy, we need to evaluate the RHS first
1175   // so that the assignment goes the right place.
1176   // This is pretty semantically fragile.
1177   if (isBlockVarRef(E->getLHS()) &&
1178       E->getRHS()->HasSideEffects(CGF.getContext())) {
1179     // Ensure that we have a destination, and evaluate the RHS into that.
1180     EnsureDest(E->getRHS()->getType());
1181     Visit(E->getRHS());
1182 
1183     // Now emit the LHS and copy into it.
1184     LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
1185 
1186     // That copy is an atomic copy if the LHS is atomic.
1187     if (LHS.getType()->isAtomicType() ||
1188         CGF.LValueIsSuitableForInlineAtomic(LHS)) {
1189       CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
1190       return;
1191     }
1192 
1193     EmitCopy(E->getLHS()->getType(),
1194              AggValueSlot::forLValue(LHS, CGF, AggValueSlot::IsDestructed,
1195                                      needsGC(E->getLHS()->getType()),
1196                                      AggValueSlot::IsAliased,
1197                                      AggValueSlot::MayOverlap),
1198              Dest);
1199     return;
1200   }
1201 
1202   LValue LHS = CGF.EmitLValue(E->getLHS());
1203 
1204   // If we have an atomic type, evaluate into the destination and then
1205   // do an atomic copy.
1206   if (LHS.getType()->isAtomicType() ||
1207       CGF.LValueIsSuitableForInlineAtomic(LHS)) {
1208     EnsureDest(E->getRHS()->getType());
1209     Visit(E->getRHS());
1210     CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
1211     return;
1212   }
1213 
1214   // Codegen the RHS so that it stores directly into the LHS.
1215   AggValueSlot LHSSlot = AggValueSlot::forLValue(
1216       LHS, CGF, AggValueSlot::IsDestructed, needsGC(E->getLHS()->getType()),
1217       AggValueSlot::IsAliased, AggValueSlot::MayOverlap);
1218   // A non-volatile aggregate destination might have volatile member.
1219   if (!LHSSlot.isVolatile() &&
1220       CGF.hasVolatileMember(E->getLHS()->getType()))
1221     LHSSlot.setVolatile(true);
1222 
1223   CGF.EmitAggExpr(E->getRHS(), LHSSlot);
1224 
1225   // Copy into the destination if the assignment isn't ignored.
1226   EmitFinalDestCopy(E->getType(), LHS);
1227 
1228   if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1229       E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
1230     CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
1231                     E->getType());
1232 }
1233 
1234 void AggExprEmitter::
1235 VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
1236   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
1237   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
1238   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
1239 
1240   // Bind the common expression if necessary.
1241   CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1242 
1243   CodeGenFunction::ConditionalEvaluation eval(CGF);
1244   CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
1245                            CGF.getProfileCount(E));
1246 
1247   // Save whether the destination's lifetime is externally managed.
1248   bool isExternallyDestructed = Dest.isExternallyDestructed();
1249   bool destructNonTrivialCStruct =
1250       !isExternallyDestructed &&
1251       E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1252   isExternallyDestructed |= destructNonTrivialCStruct;
1253   Dest.setExternallyDestructed(isExternallyDestructed);
1254 
1255   eval.begin(CGF);
1256   CGF.EmitBlock(LHSBlock);
1257   CGF.incrementProfileCounter(E);
1258   Visit(E->getTrueExpr());
1259   eval.end(CGF);
1260 
1261   assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1262   CGF.Builder.CreateBr(ContBlock);
1263 
1264   // If the result of an agg expression is unused, then the emission
1265   // of the LHS might need to create a destination slot.  That's fine
1266   // with us, and we can safely emit the RHS into the same slot, but
1267   // we shouldn't claim that it's already being destructed.
1268   Dest.setExternallyDestructed(isExternallyDestructed);
1269 
1270   eval.begin(CGF);
1271   CGF.EmitBlock(RHSBlock);
1272   Visit(E->getFalseExpr());
1273   eval.end(CGF);
1274 
1275   if (destructNonTrivialCStruct)
1276     CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
1277                     E->getType());
1278 
1279   CGF.EmitBlock(ContBlock);
1280 }
1281 
1282 void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1283   Visit(CE->getChosenSubExpr());
1284 }
1285 
1286 void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1287   Address ArgValue = Address::invalid();
1288   Address ArgPtr = CGF.EmitVAArg(VE, ArgValue);
1289 
1290   // If EmitVAArg fails, emit an error.
1291   if (!ArgPtr.isValid()) {
1292     CGF.ErrorUnsupported(VE, "aggregate va_arg expression");
1293     return;
1294   }
1295 
1296   EmitFinalDestCopy(VE->getType(), CGF.MakeAddrLValue(ArgPtr, VE->getType()));
1297 }
1298 
1299 void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1300   // Ensure that we have a slot, but if we already do, remember
1301   // whether it was externally destructed.
1302   bool wasExternallyDestructed = Dest.isExternallyDestructed();
1303   EnsureDest(E->getType());
1304 
1305   // We're going to push a destructor if there isn't already one.
1306   Dest.setExternallyDestructed();
1307 
1308   Visit(E->getSubExpr());
1309 
1310   // Push that destructor we promised.
1311   if (!wasExternallyDestructed)
1312     CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddress());
1313 }
1314 
1315 void
1316 AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1317   AggValueSlot Slot = EnsureSlot(E->getType());
1318   CGF.EmitCXXConstructExpr(E, Slot);
1319 }
1320 
1321 void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1322     const CXXInheritedCtorInitExpr *E) {
1323   AggValueSlot Slot = EnsureSlot(E->getType());
1324   CGF.EmitInheritedCXXConstructorCall(
1325       E->getConstructor(), E->constructsVBase(), Slot.getAddress(),
1326       E->inheritedFromVBase(), E);
1327 }
1328 
1329 void
1330 AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1331   AggValueSlot Slot = EnsureSlot(E->getType());
1332   LValue SlotLV = CGF.MakeAddrLValue(Slot.getAddress(), E->getType());
1333 
1334   // We'll need to enter cleanup scopes in case any of the element
1335   // initializers throws an exception.
1336   SmallVector<EHScopeStack::stable_iterator, 16> Cleanups;
1337   llvm::Instruction *CleanupDominator = nullptr;
1338 
1339   CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1340   for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
1341                                                e = E->capture_init_end();
1342        i != e; ++i, ++CurField) {
1343     // Emit initialization
1344     LValue LV = CGF.EmitLValueForFieldInitialization(SlotLV, *CurField);
1345     if (CurField->hasCapturedVLAType()) {
1346       CGF.EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);
1347       continue;
1348     }
1349 
1350     EmitInitializationToLValue(*i, LV);
1351 
1352     // Push a destructor if necessary.
1353     if (QualType::DestructionKind DtorKind =
1354             CurField->getType().isDestructedType()) {
1355       assert(LV.isSimple());
1356       if (CGF.needsEHCleanup(DtorKind)) {
1357         if (!CleanupDominator)
1358           CleanupDominator = CGF.Builder.CreateAlignedLoad(
1359               CGF.Int8Ty,
1360               llvm::Constant::getNullValue(CGF.Int8PtrTy),
1361               CharUnits::One()); // placeholder
1362 
1363         CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), CurField->getType(),
1364                         CGF.getDestroyer(DtorKind), false);
1365         Cleanups.push_back(CGF.EHStack.stable_begin());
1366       }
1367     }
1368   }
1369 
1370   // Deactivate all the partial cleanups in reverse order, which
1371   // generally means popping them.
1372   for (unsigned i = Cleanups.size(); i != 0; --i)
1373     CGF.DeactivateCleanupBlock(Cleanups[i-1], CleanupDominator);
1374 
1375   // Destroy the placeholder if we made one.
1376   if (CleanupDominator)
1377     CleanupDominator->eraseFromParent();
1378 }
1379 
1380 void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1381   CodeGenFunction::RunCleanupsScope cleanups(CGF);
1382   Visit(E->getSubExpr());
1383 }
1384 
1385 void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1386   QualType T = E->getType();
1387   AggValueSlot Slot = EnsureSlot(T);
1388   EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));
1389 }
1390 
1391 void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1392   QualType T = E->getType();
1393   AggValueSlot Slot = EnsureSlot(T);
1394   EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));
1395 }
1396 
1397 /// Determine whether the given cast kind is known to always convert values
1398 /// with all zero bits in their value representation to values with all zero
1399 /// bits in their value representation.
1400 static bool castPreservesZero(const CastExpr *CE) {
1401   switch (CE->getCastKind()) {
1402     // No-ops.
1403   case CK_NoOp:
1404   case CK_UserDefinedConversion:
1405   case CK_ConstructorConversion:
1406   case CK_BitCast:
1407   case CK_ToUnion:
1408   case CK_ToVoid:
1409     // Conversions between (possibly-complex) integral, (possibly-complex)
1410     // floating-point, and bool.
1411   case CK_BooleanToSignedIntegral:
1412   case CK_FloatingCast:
1413   case CK_FloatingComplexCast:
1414   case CK_FloatingComplexToBoolean:
1415   case CK_FloatingComplexToIntegralComplex:
1416   case CK_FloatingComplexToReal:
1417   case CK_FloatingRealToComplex:
1418   case CK_FloatingToBoolean:
1419   case CK_FloatingToIntegral:
1420   case CK_IntegralCast:
1421   case CK_IntegralComplexCast:
1422   case CK_IntegralComplexToBoolean:
1423   case CK_IntegralComplexToFloatingComplex:
1424   case CK_IntegralComplexToReal:
1425   case CK_IntegralRealToComplex:
1426   case CK_IntegralToBoolean:
1427   case CK_IntegralToFloating:
1428     // Reinterpreting integers as pointers and vice versa.
1429   case CK_IntegralToPointer:
1430   case CK_PointerToIntegral:
1431     // Language extensions.
1432   case CK_VectorSplat:
1433   case CK_MatrixCast:
1434   case CK_NonAtomicToAtomic:
1435   case CK_AtomicToNonAtomic:
1436     return true;
1437 
1438   case CK_BaseToDerivedMemberPointer:
1439   case CK_DerivedToBaseMemberPointer:
1440   case CK_MemberPointerToBoolean:
1441   case CK_NullToMemberPointer:
1442   case CK_ReinterpretMemberPointer:
1443     // FIXME: ABI-dependent.
1444     return false;
1445 
1446   case CK_AnyPointerToBlockPointerCast:
1447   case CK_BlockPointerToObjCPointerCast:
1448   case CK_CPointerToObjCPointerCast:
1449   case CK_ObjCObjectLValueCast:
1450   case CK_IntToOCLSampler:
1451   case CK_ZeroToOCLOpaqueType:
1452     // FIXME: Check these.
1453     return false;
1454 
1455   case CK_FixedPointCast:
1456   case CK_FixedPointToBoolean:
1457   case CK_FixedPointToFloating:
1458   case CK_FixedPointToIntegral:
1459   case CK_FloatingToFixedPoint:
1460   case CK_IntegralToFixedPoint:
1461     // FIXME: Do all fixed-point types represent zero as all 0 bits?
1462     return false;
1463 
1464   case CK_AddressSpaceConversion:
1465   case CK_BaseToDerived:
1466   case CK_DerivedToBase:
1467   case CK_Dynamic:
1468   case CK_NullToPointer:
1469   case CK_PointerToBoolean:
1470     // FIXME: Preserves zeroes only if zero pointers and null pointers have the
1471     // same representation in all involved address spaces.
1472     return false;
1473 
1474   case CK_ARCConsumeObject:
1475   case CK_ARCExtendBlockObject:
1476   case CK_ARCProduceObject:
1477   case CK_ARCReclaimReturnedObject:
1478   case CK_CopyAndAutoreleaseBlockObject:
1479   case CK_ArrayToPointerDecay:
1480   case CK_FunctionToPointerDecay:
1481   case CK_BuiltinFnToFnPtr:
1482   case CK_Dependent:
1483   case CK_LValueBitCast:
1484   case CK_LValueToRValue:
1485   case CK_LValueToRValueBitCast:
1486   case CK_UncheckedDerivedToBase:
1487     return false;
1488   }
1489   llvm_unreachable("Unhandled clang::CastKind enum");
1490 }
1491 
1492 /// isSimpleZero - If emitting this value will obviously just cause a store of
1493 /// zero to memory, return true.  This can return false if uncertain, so it just
1494 /// handles simple cases.
1495 static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1496   E = E->IgnoreParens();
1497   while (auto *CE = dyn_cast<CastExpr>(E)) {
1498     if (!castPreservesZero(CE))
1499       break;
1500     E = CE->getSubExpr()->IgnoreParens();
1501   }
1502 
1503   // 0
1504   if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(E))
1505     return IL->getValue() == 0;
1506   // +0.0
1507   if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(E))
1508     return FL->getValue().isPosZero();
1509   // int()
1510   if ((isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) &&
1511       CGF.getTypes().isZeroInitializable(E->getType()))
1512     return true;
1513   // (int*)0 - Null pointer expressions.
1514   if (const CastExpr *ICE = dyn_cast<CastExpr>(E))
1515     return ICE->getCastKind() == CK_NullToPointer &&
1516            CGF.getTypes().isPointerZeroInitializable(E->getType()) &&
1517            !E->HasSideEffects(CGF.getContext());
1518   // '\0'
1519   if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(E))
1520     return CL->getValue() == 0;
1521 
1522   // Otherwise, hard case: conservatively return false.
1523   return false;
1524 }
1525 
1526 
1527 void
1528 AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1529   QualType type = LV.getType();
1530   // FIXME: Ignore result?
1531   // FIXME: Are initializers affected by volatile?
1532   if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1533     // Storing "i32 0" to a zero'd memory location is a noop.
1534     return;
1535   } else if (isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) {
1536     return EmitNullInitializationToLValue(LV);
1537   } else if (isa<NoInitExpr>(E)) {
1538     // Do nothing.
1539     return;
1540   } else if (type->isReferenceType()) {
1541     RValue RV = CGF.EmitReferenceBindingToExpr(E);
1542     return CGF.EmitStoreThroughLValue(RV, LV);
1543   }
1544 
1545   switch (CGF.getEvaluationKind(type)) {
1546   case TEK_Complex:
1547     CGF.EmitComplexExprIntoLValue(E, LV, /*isInit*/ true);
1548     return;
1549   case TEK_Aggregate:
1550     CGF.EmitAggExpr(
1551         E, AggValueSlot::forLValue(LV, CGF, AggValueSlot::IsDestructed,
1552                                    AggValueSlot::DoesNotNeedGCBarriers,
1553                                    AggValueSlot::IsNotAliased,
1554                                    AggValueSlot::MayOverlap, Dest.isZeroed()));
1555     return;
1556   case TEK_Scalar:
1557     if (LV.isSimple()) {
1558       CGF.EmitScalarInit(E, /*D=*/nullptr, LV, /*Captured=*/false);
1559     } else {
1560       CGF.EmitStoreThroughLValue(RValue::get(CGF.EmitScalarExpr(E)), LV);
1561     }
1562     return;
1563   }
1564   llvm_unreachable("bad evaluation kind");
1565 }
1566 
1567 void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1568   QualType type = lv.getType();
1569 
1570   // If the destination slot is already zeroed out before the aggregate is
1571   // copied into it, we don't have to emit any zeros here.
1572   if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(type))
1573     return;
1574 
1575   if (CGF.hasScalarEvaluationKind(type)) {
1576     // For non-aggregates, we can store the appropriate null constant.
1577     llvm::Value *null = CGF.CGM.EmitNullConstant(type);
1578     // Note that the following is not equivalent to
1579     // EmitStoreThroughBitfieldLValue for ARC types.
1580     if (lv.isBitField()) {
1581       CGF.EmitStoreThroughBitfieldLValue(RValue::get(null), lv);
1582     } else {
1583       assert(lv.isSimple());
1584       CGF.EmitStoreOfScalar(null, lv, /* isInitialization */ true);
1585     }
1586   } else {
1587     // There's a potential optimization opportunity in combining
1588     // memsets; that would be easy for arrays, but relatively
1589     // difficult for structures with the current code.
1590     CGF.EmitNullInitialization(lv.getAddress(CGF), lv.getType());
1591   }
1592 }
1593 
1594 void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1595 #if 0
1596   // FIXME: Assess perf here?  Figure out what cases are worth optimizing here
1597   // (Length of globals? Chunks of zeroed-out space?).
1598   //
1599   // If we can, prefer a copy from a global; this is a lot less code for long
1600   // globals, and it's easier for the current optimizers to analyze.
1601   if (llvm::Constant* C = CGF.CGM.EmitConstantExpr(E, E->getType(), &CGF)) {
1602     llvm::GlobalVariable* GV =
1603     new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1604                              llvm::GlobalValue::InternalLinkage, C, "");
1605     EmitFinalDestCopy(E->getType(), CGF.MakeAddrLValue(GV, E->getType()));
1606     return;
1607   }
1608 #endif
1609   if (E->hadArrayRangeDesignator())
1610     CGF.ErrorUnsupported(E, "GNU array range designator extension");
1611 
1612   if (E->isTransparent())
1613     return Visit(E->getInit(0));
1614 
1615   AggValueSlot Dest = EnsureSlot(E->getType());
1616 
1617   LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1618 
1619   // Handle initialization of an array.
1620   if (E->getType()->isArrayType()) {
1621     auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType());
1622     EmitArrayInit(Dest.getAddress(), AType, E->getType(), E);
1623     return;
1624   }
1625 
1626   assert(E->getType()->isRecordType() && "Only support structs/unions here!");
1627 
1628   // Do struct initialization; this code just sets each individual member
1629   // to the approprate value.  This makes bitfield support automatic;
1630   // the disadvantage is that the generated code is more difficult for
1631   // the optimizer, especially with bitfields.
1632   unsigned NumInitElements = E->getNumInits();
1633   RecordDecl *record = E->getType()->castAs<RecordType>()->getDecl();
1634 
1635   // We'll need to enter cleanup scopes in case any of the element
1636   // initializers throws an exception.
1637   SmallVector<EHScopeStack::stable_iterator, 16> cleanups;
1638   llvm::Instruction *cleanupDominator = nullptr;
1639   auto addCleanup = [&](const EHScopeStack::stable_iterator &cleanup) {
1640     cleanups.push_back(cleanup);
1641     if (!cleanupDominator) // create placeholder once needed
1642       cleanupDominator = CGF.Builder.CreateAlignedLoad(
1643           CGF.Int8Ty, llvm::Constant::getNullValue(CGF.Int8PtrTy),
1644           CharUnits::One());
1645   };
1646 
1647   unsigned curInitIndex = 0;
1648 
1649   // Emit initialization of base classes.
1650   if (auto *CXXRD = dyn_cast<CXXRecordDecl>(record)) {
1651     assert(E->getNumInits() >= CXXRD->getNumBases() &&
1652            "missing initializer for base class");
1653     for (auto &Base : CXXRD->bases()) {
1654       assert(!Base.isVirtual() && "should not see vbases here");
1655       auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1656       Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
1657           Dest.getAddress(), CXXRD, BaseRD,
1658           /*isBaseVirtual*/ false);
1659       AggValueSlot AggSlot = AggValueSlot::forAddr(
1660           V, Qualifiers(),
1661           AggValueSlot::IsDestructed,
1662           AggValueSlot::DoesNotNeedGCBarriers,
1663           AggValueSlot::IsNotAliased,
1664           CGF.getOverlapForBaseInit(CXXRD, BaseRD, Base.isVirtual()));
1665       CGF.EmitAggExpr(E->getInit(curInitIndex++), AggSlot);
1666 
1667       if (QualType::DestructionKind dtorKind =
1668               Base.getType().isDestructedType()) {
1669         CGF.pushDestroy(dtorKind, V, Base.getType());
1670         addCleanup(CGF.EHStack.stable_begin());
1671       }
1672     }
1673   }
1674 
1675   // Prepare a 'this' for CXXDefaultInitExprs.
1676   CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1677 
1678   if (record->isUnion()) {
1679     // Only initialize one field of a union. The field itself is
1680     // specified by the initializer list.
1681     if (!E->getInitializedFieldInUnion()) {
1682       // Empty union; we have nothing to do.
1683 
1684 #ifndef NDEBUG
1685       // Make sure that it's really an empty and not a failure of
1686       // semantic analysis.
1687       for (const auto *Field : record->fields())
1688         assert(Field->isUnnamedBitfield() && "Only unnamed bitfields allowed");
1689 #endif
1690       return;
1691     }
1692 
1693     // FIXME: volatility
1694     FieldDecl *Field = E->getInitializedFieldInUnion();
1695 
1696     LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestLV, Field);
1697     if (NumInitElements) {
1698       // Store the initializer into the field
1699       EmitInitializationToLValue(E->getInit(0), FieldLoc);
1700     } else {
1701       // Default-initialize to null.
1702       EmitNullInitializationToLValue(FieldLoc);
1703     }
1704 
1705     return;
1706   }
1707 
1708   // Here we iterate over the fields; this makes it simpler to both
1709   // default-initialize fields and skip over unnamed fields.
1710   for (const auto *field : record->fields()) {
1711     // We're done once we hit the flexible array member.
1712     if (field->getType()->isIncompleteArrayType())
1713       break;
1714 
1715     // Always skip anonymous bitfields.
1716     if (field->isUnnamedBitfield())
1717       continue;
1718 
1719     // We're done if we reach the end of the explicit initializers, we
1720     // have a zeroed object, and the rest of the fields are
1721     // zero-initializable.
1722     if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1723         CGF.getTypes().isZeroInitializable(E->getType()))
1724       break;
1725 
1726 
1727     LValue LV = CGF.EmitLValueForFieldInitialization(DestLV, field);
1728     // We never generate write-barries for initialized fields.
1729     LV.setNonGC(true);
1730 
1731     if (curInitIndex < NumInitElements) {
1732       // Store the initializer into the field.
1733       EmitInitializationToLValue(E->getInit(curInitIndex++), LV);
1734     } else {
1735       // We're out of initializers; default-initialize to null
1736       EmitNullInitializationToLValue(LV);
1737     }
1738 
1739     // Push a destructor if necessary.
1740     // FIXME: if we have an array of structures, all explicitly
1741     // initialized, we can end up pushing a linear number of cleanups.
1742     bool pushedCleanup = false;
1743     if (QualType::DestructionKind dtorKind
1744           = field->getType().isDestructedType()) {
1745       assert(LV.isSimple());
1746       if (CGF.needsEHCleanup(dtorKind)) {
1747         CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), field->getType(),
1748                         CGF.getDestroyer(dtorKind), false);
1749         addCleanup(CGF.EHStack.stable_begin());
1750         pushedCleanup = true;
1751       }
1752     }
1753 
1754     // If the GEP didn't get used because of a dead zero init or something
1755     // else, clean it up for -O0 builds and general tidiness.
1756     if (!pushedCleanup && LV.isSimple())
1757       if (llvm::GetElementPtrInst *GEP =
1758               dyn_cast<llvm::GetElementPtrInst>(LV.getPointer(CGF)))
1759         if (GEP->use_empty())
1760           GEP->eraseFromParent();
1761   }
1762 
1763   // Deactivate all the partial cleanups in reverse order, which
1764   // generally means popping them.
1765   assert((cleanupDominator || cleanups.empty()) &&
1766          "Missing cleanupDominator before deactivating cleanup blocks");
1767   for (unsigned i = cleanups.size(); i != 0; --i)
1768     CGF.DeactivateCleanupBlock(cleanups[i-1], cleanupDominator);
1769 
1770   // Destroy the placeholder if we made one.
1771   if (cleanupDominator)
1772     cleanupDominator->eraseFromParent();
1773 }
1774 
1775 void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
1776                                             llvm::Value *outerBegin) {
1777   // Emit the common subexpression.
1778   CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
1779 
1780   Address destPtr = EnsureSlot(E->getType()).getAddress();
1781   uint64_t numElements = E->getArraySize().getZExtValue();
1782 
1783   if (!numElements)
1784     return;
1785 
1786   // destPtr is an array*. Construct an elementType* by drilling down a level.
1787   llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
1788   llvm::Value *indices[] = {zero, zero};
1789   llvm::Value *begin = Builder.CreateInBoundsGEP(
1790       destPtr.getElementType(), destPtr.getPointer(), indices,
1791       "arrayinit.begin");
1792 
1793   // Prepare to special-case multidimensional array initialization: we avoid
1794   // emitting multiple destructor loops in that case.
1795   if (!outerBegin)
1796     outerBegin = begin;
1797   ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(E->getSubExpr());
1798 
1799   QualType elementType =
1800       CGF.getContext().getAsArrayType(E->getType())->getElementType();
1801   CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
1802   CharUnits elementAlign =
1803       destPtr.getAlignment().alignmentOfArrayElement(elementSize);
1804   llvm::Type *llvmElementType = CGF.ConvertTypeForMem(elementType);
1805 
1806   llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1807   llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body");
1808 
1809   // Jump into the body.
1810   CGF.EmitBlock(bodyBB);
1811   llvm::PHINode *index =
1812       Builder.CreatePHI(zero->getType(), 2, "arrayinit.index");
1813   index->addIncoming(zero, entryBB);
1814   llvm::Value *element =
1815       Builder.CreateInBoundsGEP(llvmElementType, begin, index);
1816 
1817   // Prepare for a cleanup.
1818   QualType::DestructionKind dtorKind = elementType.isDestructedType();
1819   EHScopeStack::stable_iterator cleanup;
1820   if (CGF.needsEHCleanup(dtorKind) && !InnerLoop) {
1821     if (outerBegin->getType() != element->getType())
1822       outerBegin = Builder.CreateBitCast(outerBegin, element->getType());
1823     CGF.pushRegularPartialArrayCleanup(outerBegin, element, elementType,
1824                                        elementAlign,
1825                                        CGF.getDestroyer(dtorKind));
1826     cleanup = CGF.EHStack.stable_begin();
1827   } else {
1828     dtorKind = QualType::DK_none;
1829   }
1830 
1831   // Emit the actual filler expression.
1832   {
1833     // Temporaries created in an array initialization loop are destroyed
1834     // at the end of each iteration.
1835     CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
1836     CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
1837     LValue elementLV = CGF.MakeAddrLValue(
1838         Address(element, llvmElementType, elementAlign), elementType);
1839 
1840     if (InnerLoop) {
1841       // If the subexpression is an ArrayInitLoopExpr, share its cleanup.
1842       auto elementSlot = AggValueSlot::forLValue(
1843           elementLV, CGF, AggValueSlot::IsDestructed,
1844           AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
1845           AggValueSlot::DoesNotOverlap);
1846       AggExprEmitter(CGF, elementSlot, false)
1847           .VisitArrayInitLoopExpr(InnerLoop, outerBegin);
1848     } else
1849       EmitInitializationToLValue(E->getSubExpr(), elementLV);
1850   }
1851 
1852   // Move on to the next element.
1853   llvm::Value *nextIndex = Builder.CreateNUWAdd(
1854       index, llvm::ConstantInt::get(CGF.SizeTy, 1), "arrayinit.next");
1855   index->addIncoming(nextIndex, Builder.GetInsertBlock());
1856 
1857   // Leave the loop if we're done.
1858   llvm::Value *done = Builder.CreateICmpEQ(
1859       nextIndex, llvm::ConstantInt::get(CGF.SizeTy, numElements),
1860       "arrayinit.done");
1861   llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end");
1862   Builder.CreateCondBr(done, endBB, bodyBB);
1863 
1864   CGF.EmitBlock(endBB);
1865 
1866   // Leave the partial-array cleanup if we entered one.
1867   if (dtorKind)
1868     CGF.DeactivateCleanupBlock(cleanup, index);
1869 }
1870 
1871 void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) {
1872   AggValueSlot Dest = EnsureSlot(E->getType());
1873 
1874   LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1875   EmitInitializationToLValue(E->getBase(), DestLV);
1876   VisitInitListExpr(E->getUpdater());
1877 }
1878 
1879 //===----------------------------------------------------------------------===//
1880 //                        Entry Points into this File
1881 //===----------------------------------------------------------------------===//
1882 
1883 /// GetNumNonZeroBytesInInit - Get an approximate count of the number of
1884 /// non-zero bytes that will be stored when outputting the initializer for the
1885 /// specified initializer expression.
1886 static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
1887   if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(E))
1888     E = MTE->getSubExpr();
1889   E = E->IgnoreParenNoopCasts(CGF.getContext());
1890 
1891   // 0 and 0.0 won't require any non-zero stores!
1892   if (isSimpleZero(E, CGF)) return CharUnits::Zero();
1893 
1894   // If this is an initlist expr, sum up the size of sizes of the (present)
1895   // elements.  If this is something weird, assume the whole thing is non-zero.
1896   const InitListExpr *ILE = dyn_cast<InitListExpr>(E);
1897   while (ILE && ILE->isTransparent())
1898     ILE = dyn_cast<InitListExpr>(ILE->getInit(0));
1899   if (!ILE || !CGF.getTypes().isZeroInitializable(ILE->getType()))
1900     return CGF.getContext().getTypeSizeInChars(E->getType());
1901 
1902   // InitListExprs for structs have to be handled carefully.  If there are
1903   // reference members, we need to consider the size of the reference, not the
1904   // referencee.  InitListExprs for unions and arrays can't have references.
1905   if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
1906     if (!RT->isUnionType()) {
1907       RecordDecl *SD = RT->getDecl();
1908       CharUnits NumNonZeroBytes = CharUnits::Zero();
1909 
1910       unsigned ILEElement = 0;
1911       if (auto *CXXRD = dyn_cast<CXXRecordDecl>(SD))
1912         while (ILEElement != CXXRD->getNumBases())
1913           NumNonZeroBytes +=
1914               GetNumNonZeroBytesInInit(ILE->getInit(ILEElement++), CGF);
1915       for (const auto *Field : SD->fields()) {
1916         // We're done once we hit the flexible array member or run out of
1917         // InitListExpr elements.
1918         if (Field->getType()->isIncompleteArrayType() ||
1919             ILEElement == ILE->getNumInits())
1920           break;
1921         if (Field->isUnnamedBitfield())
1922           continue;
1923 
1924         const Expr *E = ILE->getInit(ILEElement++);
1925 
1926         // Reference values are always non-null and have the width of a pointer.
1927         if (Field->getType()->isReferenceType())
1928           NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
1929               CGF.getTarget().getPointerWidth(0));
1930         else
1931           NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
1932       }
1933 
1934       return NumNonZeroBytes;
1935     }
1936   }
1937 
1938   // FIXME: This overestimates the number of non-zero bytes for bit-fields.
1939   CharUnits NumNonZeroBytes = CharUnits::Zero();
1940   for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
1941     NumNonZeroBytes += GetNumNonZeroBytesInInit(ILE->getInit(i), CGF);
1942   return NumNonZeroBytes;
1943 }
1944 
1945 /// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
1946 /// zeros in it, emit a memset and avoid storing the individual zeros.
1947 ///
1948 static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
1949                                      CodeGenFunction &CGF) {
1950   // If the slot is already known to be zeroed, nothing to do.  Don't mess with
1951   // volatile stores.
1952   if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
1953     return;
1954 
1955   // C++ objects with a user-declared constructor don't need zero'ing.
1956   if (CGF.getLangOpts().CPlusPlus)
1957     if (const RecordType *RT = CGF.getContext()
1958                        .getBaseElementType(E->getType())->getAs<RecordType>()) {
1959       const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
1960       if (RD->hasUserDeclaredConstructor())
1961         return;
1962     }
1963 
1964   // If the type is 16-bytes or smaller, prefer individual stores over memset.
1965   CharUnits Size = Slot.getPreferredSize(CGF.getContext(), E->getType());
1966   if (Size <= CharUnits::fromQuantity(16))
1967     return;
1968 
1969   // Check to see if over 3/4 of the initializer are known to be zero.  If so,
1970   // we prefer to emit memset + individual stores for the rest.
1971   CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
1972   if (NumNonZeroBytes*4 > Size)
1973     return;
1974 
1975   // Okay, it seems like a good idea to use an initial memset, emit the call.
1976   llvm::Constant *SizeVal = CGF.Builder.getInt64(Size.getQuantity());
1977 
1978   Address Loc = Slot.getAddress();
1979   Loc = CGF.Builder.CreateElementBitCast(Loc, CGF.Int8Ty);
1980   CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, false);
1981 
1982   // Tell the AggExprEmitter that the slot is known zero.
1983   Slot.setZeroed();
1984 }
1985 
1986 
1987 
1988 
1989 /// EmitAggExpr - Emit the computation of the specified expression of aggregate
1990 /// type.  The result is computed into DestPtr.  Note that if DestPtr is null,
1991 /// the value of the aggregate expression is not needed.  If VolatileDest is
1992 /// true, DestPtr cannot be 0.
1993 void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
1994   assert(E && hasAggregateEvaluationKind(E->getType()) &&
1995          "Invalid aggregate expression to emit");
1996   assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
1997          "slot has bits but no address");
1998 
1999   // Optimize the slot if possible.
2000   CheckAggExprForMemSetUse(Slot, E, *this);
2001 
2002   AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E));
2003 }
2004 
2005 LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
2006   assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
2007   Address Temp = CreateMemTemp(E->getType());
2008   LValue LV = MakeAddrLValue(Temp, E->getType());
2009   EmitAggExpr(E, AggValueSlot::forLValue(
2010                      LV, *this, AggValueSlot::IsNotDestructed,
2011                      AggValueSlot::DoesNotNeedGCBarriers,
2012                      AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap));
2013   return LV;
2014 }
2015 
2016 AggValueSlot::Overlap_t
2017 CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {
2018   if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2019     return AggValueSlot::DoesNotOverlap;
2020 
2021   // If the field lies entirely within the enclosing class's nvsize, its tail
2022   // padding cannot overlap any already-initialized object. (The only subobjects
2023   // with greater addresses that might already be initialized are vbases.)
2024   const RecordDecl *ClassRD = FD->getParent();
2025   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(ClassRD);
2026   if (Layout.getFieldOffset(FD->getFieldIndex()) +
2027           getContext().getTypeSize(FD->getType()) <=
2028       (uint64_t)getContext().toBits(Layout.getNonVirtualSize()))
2029     return AggValueSlot::DoesNotOverlap;
2030 
2031   // The tail padding may contain values we need to preserve.
2032   return AggValueSlot::MayOverlap;
2033 }
2034 
2035 AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
2036     const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2037   // If the most-derived object is a field declared with [[no_unique_address]],
2038   // the tail padding of any virtual base could be reused for other subobjects
2039   // of that field's class.
2040   if (IsVirtual)
2041     return AggValueSlot::MayOverlap;
2042 
2043   // If the base class is laid out entirely within the nvsize of the derived
2044   // class, its tail padding cannot yet be initialized, so we can issue
2045   // stores at the full width of the base class.
2046   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2047   if (Layout.getBaseClassOffset(BaseRD) +
2048           getContext().getASTRecordLayout(BaseRD).getSize() <=
2049       Layout.getNonVirtualSize())
2050     return AggValueSlot::DoesNotOverlap;
2051 
2052   // The tail padding may contain values we need to preserve.
2053   return AggValueSlot::MayOverlap;
2054 }
2055 
2056 void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
2057                                         AggValueSlot::Overlap_t MayOverlap,
2058                                         bool isVolatile) {
2059   assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2060 
2061   Address DestPtr = Dest.getAddress(*this);
2062   Address SrcPtr = Src.getAddress(*this);
2063 
2064   if (getLangOpts().CPlusPlus) {
2065     if (const RecordType *RT = Ty->getAs<RecordType>()) {
2066       CXXRecordDecl *Record = cast<CXXRecordDecl>(RT->getDecl());
2067       assert((Record->hasTrivialCopyConstructor() ||
2068               Record->hasTrivialCopyAssignment() ||
2069               Record->hasTrivialMoveConstructor() ||
2070               Record->hasTrivialMoveAssignment() ||
2071               Record->hasAttr<TrivialABIAttr>() || Record->isUnion()) &&
2072              "Trying to aggregate-copy a type without a trivial copy/move "
2073              "constructor or assignment operator");
2074       // Ignore empty classes in C++.
2075       if (Record->isEmpty())
2076         return;
2077     }
2078   }
2079 
2080   if (getLangOpts().CUDAIsDevice) {
2081     if (Ty->isCUDADeviceBuiltinSurfaceType()) {
2082       if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(*this, Dest,
2083                                                                   Src))
2084         return;
2085     } else if (Ty->isCUDADeviceBuiltinTextureType()) {
2086       if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(*this, Dest,
2087                                                                   Src))
2088         return;
2089     }
2090   }
2091 
2092   // Aggregate assignment turns into llvm.memcpy.  This is almost valid per
2093   // C99 6.5.16.1p3, which states "If the value being stored in an object is
2094   // read from another object that overlaps in anyway the storage of the first
2095   // object, then the overlap shall be exact and the two objects shall have
2096   // qualified or unqualified versions of a compatible type."
2097   //
2098   // memcpy is not defined if the source and destination pointers are exactly
2099   // equal, but other compilers do this optimization, and almost every memcpy
2100   // implementation handles this case safely.  If there is a libc that does not
2101   // safely handle this, we can add a target hook.
2102 
2103   // Get data size info for this aggregate. Don't copy the tail padding if this
2104   // might be a potentially-overlapping subobject, since the tail padding might
2105   // be occupied by a different object. Otherwise, copying it is fine.
2106   TypeInfoChars TypeInfo;
2107   if (MayOverlap)
2108     TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty);
2109   else
2110     TypeInfo = getContext().getTypeInfoInChars(Ty);
2111 
2112   llvm::Value *SizeVal = nullptr;
2113   if (TypeInfo.Width.isZero()) {
2114     // But note that getTypeInfo returns 0 for a VLA.
2115     if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2116             getContext().getAsArrayType(Ty))) {
2117       QualType BaseEltTy;
2118       SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr);
2119       TypeInfo = getContext().getTypeInfoInChars(BaseEltTy);
2120       assert(!TypeInfo.Width.isZero());
2121       SizeVal = Builder.CreateNUWMul(
2122           SizeVal,
2123           llvm::ConstantInt::get(SizeTy, TypeInfo.Width.getQuantity()));
2124     }
2125   }
2126   if (!SizeVal) {
2127     SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.Width.getQuantity());
2128   }
2129 
2130   // FIXME: If we have a volatile struct, the optimizer can remove what might
2131   // appear to be `extra' memory ops:
2132   //
2133   // volatile struct { int i; } a, b;
2134   //
2135   // int main() {
2136   //   a = b;
2137   //   a = b;
2138   // }
2139   //
2140   // we need to use a different call here.  We use isVolatile to indicate when
2141   // either the source or the destination is volatile.
2142 
2143   DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
2144   SrcPtr = Builder.CreateElementBitCast(SrcPtr, Int8Ty);
2145 
2146   // Don't do any of the memmove_collectable tests if GC isn't set.
2147   if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2148     // fall through
2149   } else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
2150     RecordDecl *Record = RecordTy->getDecl();
2151     if (Record->hasObjectMember()) {
2152       CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
2153                                                     SizeVal);
2154       return;
2155     }
2156   } else if (Ty->isArrayType()) {
2157     QualType BaseType = getContext().getBaseElementType(Ty);
2158     if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) {
2159       if (RecordTy->getDecl()->hasObjectMember()) {
2160         CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
2161                                                       SizeVal);
2162         return;
2163       }
2164     }
2165   }
2166 
2167   auto Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile);
2168 
2169   // Determine the metadata to describe the position of any padding in this
2170   // memcpy, as well as the TBAA tags for the members of the struct, in case
2171   // the optimizer wishes to expand it in to scalar memory operations.
2172   if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty))
2173     Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag);
2174 
2175   if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2176     TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2177         Dest.getTBAAInfo(), Src.getTBAAInfo());
2178     CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2179   }
2180 }
2181