xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGExprAgg.cpp (revision ec0ea6efa1ad229d75c394c1a9b9cac33af2b1d3)
1 //===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Aggregate Expr nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCXXABI.h"
14 #include "CGObjCRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "ConstantEmitter.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/ASTContext.h"
20 #include "clang/AST/Attr.h"
21 #include "clang/AST/DeclCXX.h"
22 #include "clang/AST/DeclTemplate.h"
23 #include "clang/AST/StmtVisitor.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/Function.h"
26 #include "llvm/IR/GlobalVariable.h"
27 #include "llvm/IR/IntrinsicInst.h"
28 #include "llvm/IR/Intrinsics.h"
29 using namespace clang;
30 using namespace CodeGen;
31 
32 //===----------------------------------------------------------------------===//
33 //                        Aggregate Expression Emitter
34 //===----------------------------------------------------------------------===//
35 
36 namespace  {
37 class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
38   CodeGenFunction &CGF;
39   CGBuilderTy &Builder;
40   AggValueSlot Dest;
41   bool IsResultUnused;
42 
43   AggValueSlot EnsureSlot(QualType T) {
44     if (!Dest.isIgnored()) return Dest;
45     return CGF.CreateAggTemp(T, "agg.tmp.ensured");
46   }
47   void EnsureDest(QualType T) {
48     if (!Dest.isIgnored()) return;
49     Dest = CGF.CreateAggTemp(T, "agg.tmp.ensured");
50   }
51 
52   // Calls `Fn` with a valid return value slot, potentially creating a temporary
53   // to do so. If a temporary is created, an appropriate copy into `Dest` will
54   // be emitted, as will lifetime markers.
55   //
56   // The given function should take a ReturnValueSlot, and return an RValue that
57   // points to said slot.
58   void withReturnValueSlot(const Expr *E,
59                            llvm::function_ref<RValue(ReturnValueSlot)> Fn);
60 
61 public:
62   AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
63     : CGF(cgf), Builder(CGF.Builder), Dest(Dest),
64     IsResultUnused(IsResultUnused) { }
65 
66   //===--------------------------------------------------------------------===//
67   //                               Utilities
68   //===--------------------------------------------------------------------===//
69 
70   /// EmitAggLoadOfLValue - Given an expression with aggregate type that
71   /// represents a value lvalue, this method emits the address of the lvalue,
72   /// then loads the result into DestPtr.
73   void EmitAggLoadOfLValue(const Expr *E);
74 
75   enum ExprValueKind {
76     EVK_RValue,
77     EVK_NonRValue
78   };
79 
80   /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
81   /// SrcIsRValue is true if source comes from an RValue.
82   void EmitFinalDestCopy(QualType type, const LValue &src,
83                          ExprValueKind SrcValueKind = EVK_NonRValue);
84   void EmitFinalDestCopy(QualType type, RValue src);
85   void EmitCopy(QualType type, const AggValueSlot &dest,
86                 const AggValueSlot &src);
87 
88   void EmitMoveFromReturnSlot(const Expr *E, RValue Src);
89 
90   void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
91                      QualType ArrayQTy, InitListExpr *E);
92 
93   AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
94     if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
95       return AggValueSlot::NeedsGCBarriers;
96     return AggValueSlot::DoesNotNeedGCBarriers;
97   }
98 
99   bool TypeRequiresGCollection(QualType T);
100 
101   //===--------------------------------------------------------------------===//
102   //                            Visitor Methods
103   //===--------------------------------------------------------------------===//
104 
105   void Visit(Expr *E) {
106     ApplyDebugLocation DL(CGF, E);
107     StmtVisitor<AggExprEmitter>::Visit(E);
108   }
109 
110   void VisitStmt(Stmt *S) {
111     CGF.ErrorUnsupported(S, "aggregate expression");
112   }
113   void VisitParenExpr(ParenExpr *PE) { Visit(PE->getSubExpr()); }
114   void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
115     Visit(GE->getResultExpr());
116   }
117   void VisitCoawaitExpr(CoawaitExpr *E) {
118     CGF.EmitCoawaitExpr(*E, Dest, IsResultUnused);
119   }
120   void VisitCoyieldExpr(CoyieldExpr *E) {
121     CGF.EmitCoyieldExpr(*E, Dest, IsResultUnused);
122   }
123   void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); }
124   void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); }
125   void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
126     return Visit(E->getReplacement());
127   }
128 
129   void VisitConstantExpr(ConstantExpr *E) {
130     if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) {
131       CGF.EmitAggregateStore(Result, Dest.getAddress(),
132                              E->getType().isVolatileQualified());
133       return;
134     }
135     return Visit(E->getSubExpr());
136   }
137 
138   // l-values.
139   void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
140   void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(ME); }
141   void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
142   void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
143   void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
144   void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
145     EmitAggLoadOfLValue(E);
146   }
147   void VisitPredefinedExpr(const PredefinedExpr *E) {
148     EmitAggLoadOfLValue(E);
149   }
150 
151   // Operators.
152   void VisitCastExpr(CastExpr *E);
153   void VisitCallExpr(const CallExpr *E);
154   void VisitStmtExpr(const StmtExpr *E);
155   void VisitBinaryOperator(const BinaryOperator *BO);
156   void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
157   void VisitBinAssign(const BinaryOperator *E);
158   void VisitBinComma(const BinaryOperator *E);
159   void VisitBinCmp(const BinaryOperator *E);
160   void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
161     Visit(E->getSemanticForm());
162   }
163 
164   void VisitObjCMessageExpr(ObjCMessageExpr *E);
165   void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
166     EmitAggLoadOfLValue(E);
167   }
168 
169   void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
170   void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
171   void VisitChooseExpr(const ChooseExpr *CE);
172   void VisitInitListExpr(InitListExpr *E);
173   void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
174                               llvm::Value *outerBegin = nullptr);
175   void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
176   void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing.
177   void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
178     CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
179     Visit(DAE->getExpr());
180   }
181   void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
182     CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
183     Visit(DIE->getExpr());
184   }
185   void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
186   void VisitCXXConstructExpr(const CXXConstructExpr *E);
187   void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
188   void VisitLambdaExpr(LambdaExpr *E);
189   void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
190   void VisitExprWithCleanups(ExprWithCleanups *E);
191   void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
192   void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
193   void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
194   void VisitOpaqueValueExpr(OpaqueValueExpr *E);
195 
196   void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
197     if (E->isGLValue()) {
198       LValue LV = CGF.EmitPseudoObjectLValue(E);
199       return EmitFinalDestCopy(E->getType(), LV);
200     }
201 
202     CGF.EmitPseudoObjectRValue(E, EnsureSlot(E->getType()));
203   }
204 
205   void VisitVAArgExpr(VAArgExpr *E);
206 
207   void EmitInitializationToLValue(Expr *E, LValue Address);
208   void EmitNullInitializationToLValue(LValue Address);
209   //  case Expr::ChooseExprClass:
210   void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
211   void VisitAtomicExpr(AtomicExpr *E) {
212     RValue Res = CGF.EmitAtomicExpr(E);
213     EmitFinalDestCopy(E->getType(), Res);
214   }
215 };
216 }  // end anonymous namespace.
217 
218 //===----------------------------------------------------------------------===//
219 //                                Utilities
220 //===----------------------------------------------------------------------===//
221 
222 /// EmitAggLoadOfLValue - Given an expression with aggregate type that
223 /// represents a value lvalue, this method emits the address of the lvalue,
224 /// then loads the result into DestPtr.
225 void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
226   LValue LV = CGF.EmitLValue(E);
227 
228   // If the type of the l-value is atomic, then do an atomic load.
229   if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) {
230     CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest);
231     return;
232   }
233 
234   EmitFinalDestCopy(E->getType(), LV);
235 }
236 
237 /// True if the given aggregate type requires special GC API calls.
238 bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
239   // Only record types have members that might require garbage collection.
240   const RecordType *RecordTy = T->getAs<RecordType>();
241   if (!RecordTy) return false;
242 
243   // Don't mess with non-trivial C++ types.
244   RecordDecl *Record = RecordTy->getDecl();
245   if (isa<CXXRecordDecl>(Record) &&
246       (cast<CXXRecordDecl>(Record)->hasNonTrivialCopyConstructor() ||
247        !cast<CXXRecordDecl>(Record)->hasTrivialDestructor()))
248     return false;
249 
250   // Check whether the type has an object member.
251   return Record->hasObjectMember();
252 }
253 
254 void AggExprEmitter::withReturnValueSlot(
255     const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
256   QualType RetTy = E->getType();
257   bool RequiresDestruction =
258       !Dest.isExternallyDestructed() &&
259       RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
260 
261   // If it makes no observable difference, save a memcpy + temporary.
262   //
263   // We need to always provide our own temporary if destruction is required.
264   // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
265   // its lifetime before we have the chance to emit a proper destructor call.
266   bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
267                  (RequiresDestruction && !Dest.getAddress().isValid());
268 
269   Address RetAddr = Address::invalid();
270   Address RetAllocaAddr = Address::invalid();
271 
272   EHScopeStack::stable_iterator LifetimeEndBlock;
273   llvm::Value *LifetimeSizePtr = nullptr;
274   llvm::IntrinsicInst *LifetimeStartInst = nullptr;
275   if (!UseTemp) {
276     RetAddr = Dest.getAddress();
277   } else {
278     RetAddr = CGF.CreateMemTemp(RetTy, "tmp", &RetAllocaAddr);
279     llvm::TypeSize Size =
280         CGF.CGM.getDataLayout().getTypeAllocSize(CGF.ConvertTypeForMem(RetTy));
281     LifetimeSizePtr = CGF.EmitLifetimeStart(Size, RetAllocaAddr.getPointer());
282     if (LifetimeSizePtr) {
283       LifetimeStartInst =
284           cast<llvm::IntrinsicInst>(std::prev(Builder.GetInsertPoint()));
285       assert(LifetimeStartInst->getIntrinsicID() ==
286                  llvm::Intrinsic::lifetime_start &&
287              "Last insertion wasn't a lifetime.start?");
288 
289       CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
290           NormalEHLifetimeMarker, RetAllocaAddr, LifetimeSizePtr);
291       LifetimeEndBlock = CGF.EHStack.stable_begin();
292     }
293   }
294 
295   RValue Src =
296       EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
297                                Dest.isExternallyDestructed()));
298 
299   if (!UseTemp)
300     return;
301 
302   assert(Dest.getPointer() != Src.getAggregatePointer());
303   EmitFinalDestCopy(E->getType(), Src);
304 
305   if (!RequiresDestruction && LifetimeStartInst) {
306     // If there's no dtor to run, the copy was the last use of our temporary.
307     // Since we're not guaranteed to be in an ExprWithCleanups, clean up
308     // eagerly.
309     CGF.DeactivateCleanupBlock(LifetimeEndBlock, LifetimeStartInst);
310     CGF.EmitLifetimeEnd(LifetimeSizePtr, RetAllocaAddr.getPointer());
311   }
312 }
313 
314 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
315 void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
316   assert(src.isAggregate() && "value must be aggregate value!");
317   LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type);
318   EmitFinalDestCopy(type, srcLV, EVK_RValue);
319 }
320 
321 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
322 void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src,
323                                        ExprValueKind SrcValueKind) {
324   // If Dest is ignored, then we're evaluating an aggregate expression
325   // in a context that doesn't care about the result.  Note that loads
326   // from volatile l-values force the existence of a non-ignored
327   // destination.
328   if (Dest.isIgnored())
329     return;
330 
331   // Copy non-trivial C structs here.
332   LValue DstLV = CGF.MakeAddrLValue(
333       Dest.getAddress(), Dest.isVolatile() ? type.withVolatile() : type);
334 
335   if (SrcValueKind == EVK_RValue) {
336     if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
337       if (Dest.isPotentiallyAliased())
338         CGF.callCStructMoveAssignmentOperator(DstLV, src);
339       else
340         CGF.callCStructMoveConstructor(DstLV, src);
341       return;
342     }
343   } else {
344     if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
345       if (Dest.isPotentiallyAliased())
346         CGF.callCStructCopyAssignmentOperator(DstLV, src);
347       else
348         CGF.callCStructCopyConstructor(DstLV, src);
349       return;
350     }
351   }
352 
353   AggValueSlot srcAgg = AggValueSlot::forLValue(
354       src, CGF, AggValueSlot::IsDestructed, needsGC(type),
355       AggValueSlot::IsAliased, AggValueSlot::MayOverlap);
356   EmitCopy(type, Dest, srcAgg);
357 }
358 
359 /// Perform a copy from the source into the destination.
360 ///
361 /// \param type - the type of the aggregate being copied; qualifiers are
362 ///   ignored
363 void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
364                               const AggValueSlot &src) {
365   if (dest.requiresGCollection()) {
366     CharUnits sz = dest.getPreferredSize(CGF.getContext(), type);
367     llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity());
368     CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF,
369                                                       dest.getAddress(),
370                                                       src.getAddress(),
371                                                       size);
372     return;
373   }
374 
375   // If the result of the assignment is used, copy the LHS there also.
376   // It's volatile if either side is.  Use the minimum alignment of
377   // the two sides.
378   LValue DestLV = CGF.MakeAddrLValue(dest.getAddress(), type);
379   LValue SrcLV = CGF.MakeAddrLValue(src.getAddress(), type);
380   CGF.EmitAggregateCopy(DestLV, SrcLV, type, dest.mayOverlap(),
381                         dest.isVolatile() || src.isVolatile());
382 }
383 
384 /// Emit the initializer for a std::initializer_list initialized with a
385 /// real initializer list.
386 void
387 AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
388   // Emit an array containing the elements.  The array is externally destructed
389   // if the std::initializer_list object is.
390   ASTContext &Ctx = CGF.getContext();
391   LValue Array = CGF.EmitLValue(E->getSubExpr());
392   assert(Array.isSimple() && "initializer_list array not a simple lvalue");
393   Address ArrayPtr = Array.getAddress(CGF);
394 
395   const ConstantArrayType *ArrayType =
396       Ctx.getAsConstantArrayType(E->getSubExpr()->getType());
397   assert(ArrayType && "std::initializer_list constructed from non-array");
398 
399   // FIXME: Perform the checks on the field types in SemaInit.
400   RecordDecl *Record = E->getType()->castAs<RecordType>()->getDecl();
401   RecordDecl::field_iterator Field = Record->field_begin();
402   if (Field == Record->field_end()) {
403     CGF.ErrorUnsupported(E, "weird std::initializer_list");
404     return;
405   }
406 
407   // Start pointer.
408   if (!Field->getType()->isPointerType() ||
409       !Ctx.hasSameType(Field->getType()->getPointeeType(),
410                        ArrayType->getElementType())) {
411     CGF.ErrorUnsupported(E, "weird std::initializer_list");
412     return;
413   }
414 
415   AggValueSlot Dest = EnsureSlot(E->getType());
416   LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
417   LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field);
418   llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0);
419   llvm::Value *IdxStart[] = { Zero, Zero };
420   llvm::Value *ArrayStart = Builder.CreateInBoundsGEP(
421       ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxStart, "arraystart");
422   CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start);
423   ++Field;
424 
425   if (Field == Record->field_end()) {
426     CGF.ErrorUnsupported(E, "weird std::initializer_list");
427     return;
428   }
429 
430   llvm::Value *Size = Builder.getInt(ArrayType->getSize());
431   LValue EndOrLength = CGF.EmitLValueForFieldInitialization(DestLV, *Field);
432   if (Field->getType()->isPointerType() &&
433       Ctx.hasSameType(Field->getType()->getPointeeType(),
434                       ArrayType->getElementType())) {
435     // End pointer.
436     llvm::Value *IdxEnd[] = { Zero, Size };
437     llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
438         ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxEnd, "arrayend");
439     CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength);
440   } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) {
441     // Length.
442     CGF.EmitStoreThroughLValue(RValue::get(Size), EndOrLength);
443   } else {
444     CGF.ErrorUnsupported(E, "weird std::initializer_list");
445     return;
446   }
447 }
448 
449 /// Determine if E is a trivial array filler, that is, one that is
450 /// equivalent to zero-initialization.
451 static bool isTrivialFiller(Expr *E) {
452   if (!E)
453     return true;
454 
455   if (isa<ImplicitValueInitExpr>(E))
456     return true;
457 
458   if (auto *ILE = dyn_cast<InitListExpr>(E)) {
459     if (ILE->getNumInits())
460       return false;
461     return isTrivialFiller(ILE->getArrayFiller());
462   }
463 
464   if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(E))
465     return Cons->getConstructor()->isDefaultConstructor() &&
466            Cons->getConstructor()->isTrivial();
467 
468   // FIXME: Are there other cases where we can avoid emitting an initializer?
469   return false;
470 }
471 
472 /// Emit initialization of an array from an initializer list.
473 void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
474                                    QualType ArrayQTy, InitListExpr *E) {
475   uint64_t NumInitElements = E->getNumInits();
476 
477   uint64_t NumArrayElements = AType->getNumElements();
478   assert(NumInitElements <= NumArrayElements);
479 
480   QualType elementType =
481       CGF.getContext().getAsArrayType(ArrayQTy)->getElementType();
482 
483   // DestPtr is an array*.  Construct an elementType* by drilling
484   // down a level.
485   llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
486   llvm::Value *indices[] = { zero, zero };
487   llvm::Value *begin = Builder.CreateInBoundsGEP(
488       DestPtr.getElementType(), DestPtr.getPointer(), indices,
489       "arrayinit.begin");
490 
491   CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
492   CharUnits elementAlign =
493     DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
494   llvm::Type *llvmElementType = begin->getType()->getPointerElementType();
495 
496   // Consider initializing the array by copying from a global. For this to be
497   // more efficient than per-element initialization, the size of the elements
498   // with explicit initializers should be large enough.
499   if (NumInitElements * elementSize.getQuantity() > 16 &&
500       elementType.isTriviallyCopyableType(CGF.getContext())) {
501     CodeGen::CodeGenModule &CGM = CGF.CGM;
502     ConstantEmitter Emitter(CGF);
503     LangAS AS = ArrayQTy.getAddressSpace();
504     if (llvm::Constant *C = Emitter.tryEmitForInitializer(E, AS, ArrayQTy)) {
505       auto GV = new llvm::GlobalVariable(
506           CGM.getModule(), C->getType(),
507           CGM.isTypeConstant(ArrayQTy, /* ExcludeCtorDtor= */ true),
508           llvm::GlobalValue::PrivateLinkage, C, "constinit",
509           /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
510           CGM.getContext().getTargetAddressSpace(AS));
511       Emitter.finalize(GV);
512       CharUnits Align = CGM.getContext().getTypeAlignInChars(ArrayQTy);
513       GV->setAlignment(Align.getAsAlign());
514       EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GV, ArrayQTy, Align));
515       return;
516     }
517   }
518 
519   // Exception safety requires us to destroy all the
520   // already-constructed members if an initializer throws.
521   // For that, we'll need an EH cleanup.
522   QualType::DestructionKind dtorKind = elementType.isDestructedType();
523   Address endOfInit = Address::invalid();
524   EHScopeStack::stable_iterator cleanup;
525   llvm::Instruction *cleanupDominator = nullptr;
526   if (CGF.needsEHCleanup(dtorKind)) {
527     // In principle we could tell the cleanup where we are more
528     // directly, but the control flow can get so varied here that it
529     // would actually be quite complex.  Therefore we go through an
530     // alloca.
531     endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(),
532                                      "arrayinit.endOfInit");
533     cleanupDominator = Builder.CreateStore(begin, endOfInit);
534     CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType,
535                                          elementAlign,
536                                          CGF.getDestroyer(dtorKind));
537     cleanup = CGF.EHStack.stable_begin();
538 
539   // Otherwise, remember that we didn't need a cleanup.
540   } else {
541     dtorKind = QualType::DK_none;
542   }
543 
544   llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1);
545 
546   // The 'current element to initialize'.  The invariants on this
547   // variable are complicated.  Essentially, after each iteration of
548   // the loop, it points to the last initialized element, except
549   // that it points to the beginning of the array before any
550   // elements have been initialized.
551   llvm::Value *element = begin;
552 
553   // Emit the explicit initializers.
554   for (uint64_t i = 0; i != NumInitElements; ++i) {
555     // Advance to the next element.
556     if (i > 0) {
557       element = Builder.CreateInBoundsGEP(
558           llvmElementType, element, one, "arrayinit.element");
559 
560       // Tell the cleanup that it needs to destroy up to this
561       // element.  TODO: some of these stores can be trivially
562       // observed to be unnecessary.
563       if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
564     }
565 
566     LValue elementLV =
567       CGF.MakeAddrLValue(Address(element, elementAlign), elementType);
568     EmitInitializationToLValue(E->getInit(i), elementLV);
569   }
570 
571   // Check whether there's a non-trivial array-fill expression.
572   Expr *filler = E->getArrayFiller();
573   bool hasTrivialFiller = isTrivialFiller(filler);
574 
575   // Any remaining elements need to be zero-initialized, possibly
576   // using the filler expression.  We can skip this if the we're
577   // emitting to zeroed memory.
578   if (NumInitElements != NumArrayElements &&
579       !(Dest.isZeroed() && hasTrivialFiller &&
580         CGF.getTypes().isZeroInitializable(elementType))) {
581 
582     // Use an actual loop.  This is basically
583     //   do { *array++ = filler; } while (array != end);
584 
585     // Advance to the start of the rest of the array.
586     if (NumInitElements) {
587       element = Builder.CreateInBoundsGEP(
588           llvmElementType, element, one, "arrayinit.start");
589       if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);
590     }
591 
592     // Compute the end of the array.
593     llvm::Value *end = Builder.CreateInBoundsGEP(
594         llvmElementType, begin,
595         llvm::ConstantInt::get(CGF.SizeTy, NumArrayElements), "arrayinit.end");
596 
597     llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
598     llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body");
599 
600     // Jump into the body.
601     CGF.EmitBlock(bodyBB);
602     llvm::PHINode *currentElement =
603       Builder.CreatePHI(element->getType(), 2, "arrayinit.cur");
604     currentElement->addIncoming(element, entryBB);
605 
606     // Emit the actual filler expression.
607     {
608       // C++1z [class.temporary]p5:
609       //   when a default constructor is called to initialize an element of
610       //   an array with no corresponding initializer [...] the destruction of
611       //   every temporary created in a default argument is sequenced before
612       //   the construction of the next array element, if any
613       CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
614       LValue elementLV =
615         CGF.MakeAddrLValue(Address(currentElement, elementAlign), elementType);
616       if (filler)
617         EmitInitializationToLValue(filler, elementLV);
618       else
619         EmitNullInitializationToLValue(elementLV);
620     }
621 
622     // Move on to the next element.
623     llvm::Value *nextElement = Builder.CreateInBoundsGEP(
624         llvmElementType, currentElement, one, "arrayinit.next");
625 
626     // Tell the EH cleanup that we finished with the last element.
627     if (endOfInit.isValid()) Builder.CreateStore(nextElement, endOfInit);
628 
629     // Leave the loop if we're done.
630     llvm::Value *done = Builder.CreateICmpEQ(nextElement, end,
631                                              "arrayinit.done");
632     llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end");
633     Builder.CreateCondBr(done, endBB, bodyBB);
634     currentElement->addIncoming(nextElement, Builder.GetInsertBlock());
635 
636     CGF.EmitBlock(endBB);
637   }
638 
639   // Leave the partial-array cleanup if we entered one.
640   if (dtorKind) CGF.DeactivateCleanupBlock(cleanup, cleanupDominator);
641 }
642 
643 //===----------------------------------------------------------------------===//
644 //                            Visitor Methods
645 //===----------------------------------------------------------------------===//
646 
647 void AggExprEmitter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E){
648   Visit(E->getSubExpr());
649 }
650 
651 void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
652   // If this is a unique OVE, just visit its source expression.
653   if (e->isUnique())
654     Visit(e->getSourceExpr());
655   else
656     EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e));
657 }
658 
659 void
660 AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
661   if (Dest.isPotentiallyAliased() &&
662       E->getType().isPODType(CGF.getContext())) {
663     // For a POD type, just emit a load of the lvalue + a copy, because our
664     // compound literal might alias the destination.
665     EmitAggLoadOfLValue(E);
666     return;
667   }
668 
669   AggValueSlot Slot = EnsureSlot(E->getType());
670 
671   // Block-scope compound literals are destroyed at the end of the enclosing
672   // scope in C.
673   bool Destruct =
674       !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
675   if (Destruct)
676     Slot.setExternallyDestructed();
677 
678   CGF.EmitAggExpr(E->getInitializer(), Slot);
679 
680   if (Destruct)
681     if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
682       CGF.pushLifetimeExtendedDestroy(
683           CGF.getCleanupKind(DtorKind), Slot.getAddress(), E->getType(),
684           CGF.getDestroyer(DtorKind), DtorKind & EHCleanup);
685 }
686 
687 /// Attempt to look through various unimportant expressions to find a
688 /// cast of the given kind.
689 static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
690   op = op->IgnoreParenNoopCasts(ctx);
691   if (auto castE = dyn_cast<CastExpr>(op)) {
692     if (castE->getCastKind() == kind)
693       return castE->getSubExpr();
694   }
695   return nullptr;
696 }
697 
698 void AggExprEmitter::VisitCastExpr(CastExpr *E) {
699   if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
700     CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
701   switch (E->getCastKind()) {
702   case CK_Dynamic: {
703     // FIXME: Can this actually happen? We have no test coverage for it.
704     assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
705     LValue LV = CGF.EmitCheckedLValue(E->getSubExpr(),
706                                       CodeGenFunction::TCK_Load);
707     // FIXME: Do we also need to handle property references here?
708     if (LV.isSimple())
709       CGF.EmitDynamicCast(LV.getAddress(CGF), cast<CXXDynamicCastExpr>(E));
710     else
711       CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast");
712 
713     if (!Dest.isIgnored())
714       CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination");
715     break;
716   }
717 
718   case CK_ToUnion: {
719     // Evaluate even if the destination is ignored.
720     if (Dest.isIgnored()) {
721       CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(),
722                       /*ignoreResult=*/true);
723       break;
724     }
725 
726     // GCC union extension
727     QualType Ty = E->getSubExpr()->getType();
728     Address CastPtr =
729       Builder.CreateElementBitCast(Dest.getAddress(), CGF.ConvertType(Ty));
730     EmitInitializationToLValue(E->getSubExpr(),
731                                CGF.MakeAddrLValue(CastPtr, Ty));
732     break;
733   }
734 
735   case CK_LValueToRValueBitCast: {
736     if (Dest.isIgnored()) {
737       CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(),
738                       /*ignoreResult=*/true);
739       break;
740     }
741 
742     LValue SourceLV = CGF.EmitLValue(E->getSubExpr());
743     Address SourceAddress =
744         Builder.CreateElementBitCast(SourceLV.getAddress(CGF), CGF.Int8Ty);
745     Address DestAddress =
746         Builder.CreateElementBitCast(Dest.getAddress(), CGF.Int8Ty);
747     llvm::Value *SizeVal = llvm::ConstantInt::get(
748         CGF.SizeTy,
749         CGF.getContext().getTypeSizeInChars(E->getType()).getQuantity());
750     Builder.CreateMemCpy(DestAddress, SourceAddress, SizeVal);
751     break;
752   }
753 
754   case CK_DerivedToBase:
755   case CK_BaseToDerived:
756   case CK_UncheckedDerivedToBase: {
757     llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
758                 "should have been unpacked before we got here");
759   }
760 
761   case CK_NonAtomicToAtomic:
762   case CK_AtomicToNonAtomic: {
763     bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
764 
765     // Determine the atomic and value types.
766     QualType atomicType = E->getSubExpr()->getType();
767     QualType valueType = E->getType();
768     if (isToAtomic) std::swap(atomicType, valueType);
769 
770     assert(atomicType->isAtomicType());
771     assert(CGF.getContext().hasSameUnqualifiedType(valueType,
772                           atomicType->castAs<AtomicType>()->getValueType()));
773 
774     // Just recurse normally if we're ignoring the result or the
775     // atomic type doesn't change representation.
776     if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(atomicType)) {
777       return Visit(E->getSubExpr());
778     }
779 
780     CastKind peepholeTarget =
781       (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
782 
783     // These two cases are reverses of each other; try to peephole them.
784     if (Expr *op =
785             findPeephole(E->getSubExpr(), peepholeTarget, CGF.getContext())) {
786       assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
787                                                      E->getType()) &&
788            "peephole significantly changed types?");
789       return Visit(op);
790     }
791 
792     // If we're converting an r-value of non-atomic type to an r-value
793     // of atomic type, just emit directly into the relevant sub-object.
794     if (isToAtomic) {
795       AggValueSlot valueDest = Dest;
796       if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(atomicType)) {
797         // Zero-initialize.  (Strictly speaking, we only need to initialize
798         // the padding at the end, but this is simpler.)
799         if (!Dest.isZeroed())
800           CGF.EmitNullInitialization(Dest.getAddress(), atomicType);
801 
802         // Build a GEP to refer to the subobject.
803         Address valueAddr =
804             CGF.Builder.CreateStructGEP(valueDest.getAddress(), 0);
805         valueDest = AggValueSlot::forAddr(valueAddr,
806                                           valueDest.getQualifiers(),
807                                           valueDest.isExternallyDestructed(),
808                                           valueDest.requiresGCollection(),
809                                           valueDest.isPotentiallyAliased(),
810                                           AggValueSlot::DoesNotOverlap,
811                                           AggValueSlot::IsZeroed);
812       }
813 
814       CGF.EmitAggExpr(E->getSubExpr(), valueDest);
815       return;
816     }
817 
818     // Otherwise, we're converting an atomic type to a non-atomic type.
819     // Make an atomic temporary, emit into that, and then copy the value out.
820     AggValueSlot atomicSlot =
821       CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp");
822     CGF.EmitAggExpr(E->getSubExpr(), atomicSlot);
823 
824     Address valueAddr = Builder.CreateStructGEP(atomicSlot.getAddress(), 0);
825     RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile());
826     return EmitFinalDestCopy(valueType, rvalue);
827   }
828   case CK_AddressSpaceConversion:
829      return Visit(E->getSubExpr());
830 
831   case CK_LValueToRValue:
832     // If we're loading from a volatile type, force the destination
833     // into existence.
834     if (E->getSubExpr()->getType().isVolatileQualified()) {
835       bool Destruct =
836           !Dest.isExternallyDestructed() &&
837           E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
838       if (Destruct)
839         Dest.setExternallyDestructed();
840       EnsureDest(E->getType());
841       Visit(E->getSubExpr());
842 
843       if (Destruct)
844         CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
845                         E->getType());
846 
847       return;
848     }
849 
850     LLVM_FALLTHROUGH;
851 
852 
853   case CK_NoOp:
854   case CK_UserDefinedConversion:
855   case CK_ConstructorConversion:
856     assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
857                                                    E->getType()) &&
858            "Implicit cast types must be compatible");
859     Visit(E->getSubExpr());
860     break;
861 
862   case CK_LValueBitCast:
863     llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
864 
865   case CK_Dependent:
866   case CK_BitCast:
867   case CK_ArrayToPointerDecay:
868   case CK_FunctionToPointerDecay:
869   case CK_NullToPointer:
870   case CK_NullToMemberPointer:
871   case CK_BaseToDerivedMemberPointer:
872   case CK_DerivedToBaseMemberPointer:
873   case CK_MemberPointerToBoolean:
874   case CK_ReinterpretMemberPointer:
875   case CK_IntegralToPointer:
876   case CK_PointerToIntegral:
877   case CK_PointerToBoolean:
878   case CK_ToVoid:
879   case CK_VectorSplat:
880   case CK_IntegralCast:
881   case CK_BooleanToSignedIntegral:
882   case CK_IntegralToBoolean:
883   case CK_IntegralToFloating:
884   case CK_FloatingToIntegral:
885   case CK_FloatingToBoolean:
886   case CK_FloatingCast:
887   case CK_CPointerToObjCPointerCast:
888   case CK_BlockPointerToObjCPointerCast:
889   case CK_AnyPointerToBlockPointerCast:
890   case CK_ObjCObjectLValueCast:
891   case CK_FloatingRealToComplex:
892   case CK_FloatingComplexToReal:
893   case CK_FloatingComplexToBoolean:
894   case CK_FloatingComplexCast:
895   case CK_FloatingComplexToIntegralComplex:
896   case CK_IntegralRealToComplex:
897   case CK_IntegralComplexToReal:
898   case CK_IntegralComplexToBoolean:
899   case CK_IntegralComplexCast:
900   case CK_IntegralComplexToFloatingComplex:
901   case CK_ARCProduceObject:
902   case CK_ARCConsumeObject:
903   case CK_ARCReclaimReturnedObject:
904   case CK_ARCExtendBlockObject:
905   case CK_CopyAndAutoreleaseBlockObject:
906   case CK_BuiltinFnToFnPtr:
907   case CK_ZeroToOCLOpaqueType:
908   case CK_MatrixCast:
909 
910   case CK_IntToOCLSampler:
911   case CK_FloatingToFixedPoint:
912   case CK_FixedPointToFloating:
913   case CK_FixedPointCast:
914   case CK_FixedPointToBoolean:
915   case CK_FixedPointToIntegral:
916   case CK_IntegralToFixedPoint:
917     llvm_unreachable("cast kind invalid for aggregate types");
918   }
919 }
920 
921 void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
922   if (E->getCallReturnType(CGF.getContext())->isReferenceType()) {
923     EmitAggLoadOfLValue(E);
924     return;
925   }
926 
927   withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
928     return CGF.EmitCallExpr(E, Slot);
929   });
930 }
931 
932 void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
933   withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
934     return CGF.EmitObjCMessageExpr(E, Slot);
935   });
936 }
937 
938 void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
939   CGF.EmitIgnoredExpr(E->getLHS());
940   Visit(E->getRHS());
941 }
942 
943 void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
944   CodeGenFunction::StmtExprEvaluation eval(CGF);
945   CGF.EmitCompoundStmt(*E->getSubStmt(), true, Dest);
946 }
947 
948 enum CompareKind {
949   CK_Less,
950   CK_Greater,
951   CK_Equal,
952 };
953 
954 static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
955                                 const BinaryOperator *E, llvm::Value *LHS,
956                                 llvm::Value *RHS, CompareKind Kind,
957                                 const char *NameSuffix = "") {
958   QualType ArgTy = E->getLHS()->getType();
959   if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
960     ArgTy = CT->getElementType();
961 
962   if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
963     assert(Kind == CK_Equal &&
964            "member pointers may only be compared for equality");
965     return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
966         CGF, LHS, RHS, MPT, /*IsInequality*/ false);
967   }
968 
969   // Compute the comparison instructions for the specified comparison kind.
970   struct CmpInstInfo {
971     const char *Name;
972     llvm::CmpInst::Predicate FCmp;
973     llvm::CmpInst::Predicate SCmp;
974     llvm::CmpInst::Predicate UCmp;
975   };
976   CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
977     using FI = llvm::FCmpInst;
978     using II = llvm::ICmpInst;
979     switch (Kind) {
980     case CK_Less:
981       return {"cmp.lt", FI::FCMP_OLT, II::ICMP_SLT, II::ICMP_ULT};
982     case CK_Greater:
983       return {"cmp.gt", FI::FCMP_OGT, II::ICMP_SGT, II::ICMP_UGT};
984     case CK_Equal:
985       return {"cmp.eq", FI::FCMP_OEQ, II::ICMP_EQ, II::ICMP_EQ};
986     }
987     llvm_unreachable("Unrecognised CompareKind enum");
988   }();
989 
990   if (ArgTy->hasFloatingRepresentation())
991     return Builder.CreateFCmp(InstInfo.FCmp, LHS, RHS,
992                               llvm::Twine(InstInfo.Name) + NameSuffix);
993   if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
994     auto Inst =
995         ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
996     return Builder.CreateICmp(Inst, LHS, RHS,
997                               llvm::Twine(InstInfo.Name) + NameSuffix);
998   }
999 
1000   llvm_unreachable("unsupported aggregate binary expression should have "
1001                    "already been handled");
1002 }
1003 
1004 void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1005   using llvm::BasicBlock;
1006   using llvm::PHINode;
1007   using llvm::Value;
1008   assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1009                                       E->getRHS()->getType()));
1010   const ComparisonCategoryInfo &CmpInfo =
1011       CGF.getContext().CompCategories.getInfoForType(E->getType());
1012   assert(CmpInfo.Record->isTriviallyCopyable() &&
1013          "cannot copy non-trivially copyable aggregate");
1014 
1015   QualType ArgTy = E->getLHS()->getType();
1016 
1017   if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1018       !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1019       !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1020     return CGF.ErrorUnsupported(E, "aggregate three-way comparison");
1021   }
1022   bool IsComplex = ArgTy->isAnyComplexType();
1023 
1024   // Evaluate the operands to the expression and extract their values.
1025   auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1026     RValue RV = CGF.EmitAnyExpr(E);
1027     if (RV.isScalar())
1028       return {RV.getScalarVal(), nullptr};
1029     if (RV.isAggregate())
1030       return {RV.getAggregatePointer(), nullptr};
1031     assert(RV.isComplex());
1032     return RV.getComplexVal();
1033   };
1034   auto LHSValues = EmitOperand(E->getLHS()),
1035        RHSValues = EmitOperand(E->getRHS());
1036 
1037   auto EmitCmp = [&](CompareKind K) {
1038     Value *Cmp = EmitCompare(Builder, CGF, E, LHSValues.first, RHSValues.first,
1039                              K, IsComplex ? ".r" : "");
1040     if (!IsComplex)
1041       return Cmp;
1042     assert(K == CompareKind::CK_Equal);
1043     Value *CmpImag = EmitCompare(Builder, CGF, E, LHSValues.second,
1044                                  RHSValues.second, K, ".i");
1045     return Builder.CreateAnd(Cmp, CmpImag, "and.eq");
1046   };
1047   auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1048     return Builder.getInt(VInfo->getIntValue());
1049   };
1050 
1051   Value *Select;
1052   if (ArgTy->isNullPtrType()) {
1053     Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1054   } else if (!CmpInfo.isPartial()) {
1055     Value *SelectOne =
1056         Builder.CreateSelect(EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()),
1057                              EmitCmpRes(CmpInfo.getGreater()), "sel.lt");
1058     Select = Builder.CreateSelect(EmitCmp(CK_Equal),
1059                                   EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1060                                   SelectOne, "sel.eq");
1061   } else {
1062     Value *SelectEq = Builder.CreateSelect(
1063         EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1064         EmitCmpRes(CmpInfo.getUnordered()), "sel.eq");
1065     Value *SelectGT = Builder.CreateSelect(EmitCmp(CK_Greater),
1066                                            EmitCmpRes(CmpInfo.getGreater()),
1067                                            SelectEq, "sel.gt");
1068     Select = Builder.CreateSelect(
1069         EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), SelectGT, "sel.lt");
1070   }
1071   // Create the return value in the destination slot.
1072   EnsureDest(E->getType());
1073   LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1074 
1075   // Emit the address of the first (and only) field in the comparison category
1076   // type, and initialize it from the constant integer value selected above.
1077   LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1078       DestLV, *CmpInfo.Record->field_begin());
1079   CGF.EmitStoreThroughLValue(RValue::get(Select), FieldLV, /*IsInit*/ true);
1080 
1081   // All done! The result is in the Dest slot.
1082 }
1083 
1084 void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1085   if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1086     VisitPointerToDataMemberBinaryOperator(E);
1087   else
1088     CGF.ErrorUnsupported(E, "aggregate binary expression");
1089 }
1090 
1091 void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1092                                                     const BinaryOperator *E) {
1093   LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1094   EmitFinalDestCopy(E->getType(), LV);
1095 }
1096 
1097 /// Is the value of the given expression possibly a reference to or
1098 /// into a __block variable?
1099 static bool isBlockVarRef(const Expr *E) {
1100   // Make sure we look through parens.
1101   E = E->IgnoreParens();
1102 
1103   // Check for a direct reference to a __block variable.
1104   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {
1105     const VarDecl *var = dyn_cast<VarDecl>(DRE->getDecl());
1106     return (var && var->hasAttr<BlocksAttr>());
1107   }
1108 
1109   // More complicated stuff.
1110 
1111   // Binary operators.
1112   if (const BinaryOperator *op = dyn_cast<BinaryOperator>(E)) {
1113     // For an assignment or pointer-to-member operation, just care
1114     // about the LHS.
1115     if (op->isAssignmentOp() || op->isPtrMemOp())
1116       return isBlockVarRef(op->getLHS());
1117 
1118     // For a comma, just care about the RHS.
1119     if (op->getOpcode() == BO_Comma)
1120       return isBlockVarRef(op->getRHS());
1121 
1122     // FIXME: pointer arithmetic?
1123     return false;
1124 
1125   // Check both sides of a conditional operator.
1126   } else if (const AbstractConditionalOperator *op
1127                = dyn_cast<AbstractConditionalOperator>(E)) {
1128     return isBlockVarRef(op->getTrueExpr())
1129         || isBlockVarRef(op->getFalseExpr());
1130 
1131   // OVEs are required to support BinaryConditionalOperators.
1132   } else if (const OpaqueValueExpr *op
1133                = dyn_cast<OpaqueValueExpr>(E)) {
1134     if (const Expr *src = op->getSourceExpr())
1135       return isBlockVarRef(src);
1136 
1137   // Casts are necessary to get things like (*(int*)&var) = foo().
1138   // We don't really care about the kind of cast here, except
1139   // we don't want to look through l2r casts, because it's okay
1140   // to get the *value* in a __block variable.
1141   } else if (const CastExpr *cast = dyn_cast<CastExpr>(E)) {
1142     if (cast->getCastKind() == CK_LValueToRValue)
1143       return false;
1144     return isBlockVarRef(cast->getSubExpr());
1145 
1146   // Handle unary operators.  Again, just aggressively look through
1147   // it, ignoring the operation.
1148   } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E)) {
1149     return isBlockVarRef(uop->getSubExpr());
1150 
1151   // Look into the base of a field access.
1152   } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(E)) {
1153     return isBlockVarRef(mem->getBase());
1154 
1155   // Look into the base of a subscript.
1156   } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(E)) {
1157     return isBlockVarRef(sub->getBase());
1158   }
1159 
1160   return false;
1161 }
1162 
1163 void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1164   // For an assignment to work, the value on the right has
1165   // to be compatible with the value on the left.
1166   assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1167                                                  E->getRHS()->getType())
1168          && "Invalid assignment");
1169 
1170   // If the LHS might be a __block variable, and the RHS can
1171   // potentially cause a block copy, we need to evaluate the RHS first
1172   // so that the assignment goes the right place.
1173   // This is pretty semantically fragile.
1174   if (isBlockVarRef(E->getLHS()) &&
1175       E->getRHS()->HasSideEffects(CGF.getContext())) {
1176     // Ensure that we have a destination, and evaluate the RHS into that.
1177     EnsureDest(E->getRHS()->getType());
1178     Visit(E->getRHS());
1179 
1180     // Now emit the LHS and copy into it.
1181     LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
1182 
1183     // That copy is an atomic copy if the LHS is atomic.
1184     if (LHS.getType()->isAtomicType() ||
1185         CGF.LValueIsSuitableForInlineAtomic(LHS)) {
1186       CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
1187       return;
1188     }
1189 
1190     EmitCopy(E->getLHS()->getType(),
1191              AggValueSlot::forLValue(LHS, CGF, AggValueSlot::IsDestructed,
1192                                      needsGC(E->getLHS()->getType()),
1193                                      AggValueSlot::IsAliased,
1194                                      AggValueSlot::MayOverlap),
1195              Dest);
1196     return;
1197   }
1198 
1199   LValue LHS = CGF.EmitLValue(E->getLHS());
1200 
1201   // If we have an atomic type, evaluate into the destination and then
1202   // do an atomic copy.
1203   if (LHS.getType()->isAtomicType() ||
1204       CGF.LValueIsSuitableForInlineAtomic(LHS)) {
1205     EnsureDest(E->getRHS()->getType());
1206     Visit(E->getRHS());
1207     CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);
1208     return;
1209   }
1210 
1211   // Codegen the RHS so that it stores directly into the LHS.
1212   AggValueSlot LHSSlot = AggValueSlot::forLValue(
1213       LHS, CGF, AggValueSlot::IsDestructed, needsGC(E->getLHS()->getType()),
1214       AggValueSlot::IsAliased, AggValueSlot::MayOverlap);
1215   // A non-volatile aggregate destination might have volatile member.
1216   if (!LHSSlot.isVolatile() &&
1217       CGF.hasVolatileMember(E->getLHS()->getType()))
1218     LHSSlot.setVolatile(true);
1219 
1220   CGF.EmitAggExpr(E->getRHS(), LHSSlot);
1221 
1222   // Copy into the destination if the assignment isn't ignored.
1223   EmitFinalDestCopy(E->getType(), LHS);
1224 
1225   if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1226       E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
1227     CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
1228                     E->getType());
1229 }
1230 
1231 void AggExprEmitter::
1232 VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
1233   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
1234   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
1235   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
1236 
1237   // Bind the common expression if necessary.
1238   CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1239 
1240   CodeGenFunction::ConditionalEvaluation eval(CGF);
1241   CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
1242                            CGF.getProfileCount(E));
1243 
1244   // Save whether the destination's lifetime is externally managed.
1245   bool isExternallyDestructed = Dest.isExternallyDestructed();
1246   bool destructNonTrivialCStruct =
1247       !isExternallyDestructed &&
1248       E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1249   isExternallyDestructed |= destructNonTrivialCStruct;
1250   Dest.setExternallyDestructed(isExternallyDestructed);
1251 
1252   eval.begin(CGF);
1253   CGF.EmitBlock(LHSBlock);
1254   CGF.incrementProfileCounter(E);
1255   Visit(E->getTrueExpr());
1256   eval.end(CGF);
1257 
1258   assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1259   CGF.Builder.CreateBr(ContBlock);
1260 
1261   // If the result of an agg expression is unused, then the emission
1262   // of the LHS might need to create a destination slot.  That's fine
1263   // with us, and we can safely emit the RHS into the same slot, but
1264   // we shouldn't claim that it's already being destructed.
1265   Dest.setExternallyDestructed(isExternallyDestructed);
1266 
1267   eval.begin(CGF);
1268   CGF.EmitBlock(RHSBlock);
1269   Visit(E->getFalseExpr());
1270   eval.end(CGF);
1271 
1272   if (destructNonTrivialCStruct)
1273     CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
1274                     E->getType());
1275 
1276   CGF.EmitBlock(ContBlock);
1277 }
1278 
1279 void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1280   Visit(CE->getChosenSubExpr());
1281 }
1282 
1283 void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1284   Address ArgValue = Address::invalid();
1285   Address ArgPtr = CGF.EmitVAArg(VE, ArgValue);
1286 
1287   // If EmitVAArg fails, emit an error.
1288   if (!ArgPtr.isValid()) {
1289     CGF.ErrorUnsupported(VE, "aggregate va_arg expression");
1290     return;
1291   }
1292 
1293   EmitFinalDestCopy(VE->getType(), CGF.MakeAddrLValue(ArgPtr, VE->getType()));
1294 }
1295 
1296 void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1297   // Ensure that we have a slot, but if we already do, remember
1298   // whether it was externally destructed.
1299   bool wasExternallyDestructed = Dest.isExternallyDestructed();
1300   EnsureDest(E->getType());
1301 
1302   // We're going to push a destructor if there isn't already one.
1303   Dest.setExternallyDestructed();
1304 
1305   Visit(E->getSubExpr());
1306 
1307   // Push that destructor we promised.
1308   if (!wasExternallyDestructed)
1309     CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddress());
1310 }
1311 
1312 void
1313 AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1314   AggValueSlot Slot = EnsureSlot(E->getType());
1315   CGF.EmitCXXConstructExpr(E, Slot);
1316 }
1317 
1318 void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1319     const CXXInheritedCtorInitExpr *E) {
1320   AggValueSlot Slot = EnsureSlot(E->getType());
1321   CGF.EmitInheritedCXXConstructorCall(
1322       E->getConstructor(), E->constructsVBase(), Slot.getAddress(),
1323       E->inheritedFromVBase(), E);
1324 }
1325 
1326 void
1327 AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1328   AggValueSlot Slot = EnsureSlot(E->getType());
1329   LValue SlotLV = CGF.MakeAddrLValue(Slot.getAddress(), E->getType());
1330 
1331   // We'll need to enter cleanup scopes in case any of the element
1332   // initializers throws an exception.
1333   SmallVector<EHScopeStack::stable_iterator, 16> Cleanups;
1334   llvm::Instruction *CleanupDominator = nullptr;
1335 
1336   CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1337   for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
1338                                                e = E->capture_init_end();
1339        i != e; ++i, ++CurField) {
1340     // Emit initialization
1341     LValue LV = CGF.EmitLValueForFieldInitialization(SlotLV, *CurField);
1342     if (CurField->hasCapturedVLAType()) {
1343       CGF.EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);
1344       continue;
1345     }
1346 
1347     EmitInitializationToLValue(*i, LV);
1348 
1349     // Push a destructor if necessary.
1350     if (QualType::DestructionKind DtorKind =
1351             CurField->getType().isDestructedType()) {
1352       assert(LV.isSimple());
1353       if (CGF.needsEHCleanup(DtorKind)) {
1354         if (!CleanupDominator)
1355           CleanupDominator = CGF.Builder.CreateAlignedLoad(
1356               CGF.Int8Ty,
1357               llvm::Constant::getNullValue(CGF.Int8PtrTy),
1358               CharUnits::One()); // placeholder
1359 
1360         CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), CurField->getType(),
1361                         CGF.getDestroyer(DtorKind), false);
1362         Cleanups.push_back(CGF.EHStack.stable_begin());
1363       }
1364     }
1365   }
1366 
1367   // Deactivate all the partial cleanups in reverse order, which
1368   // generally means popping them.
1369   for (unsigned i = Cleanups.size(); i != 0; --i)
1370     CGF.DeactivateCleanupBlock(Cleanups[i-1], CleanupDominator);
1371 
1372   // Destroy the placeholder if we made one.
1373   if (CleanupDominator)
1374     CleanupDominator->eraseFromParent();
1375 }
1376 
1377 void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1378   CodeGenFunction::RunCleanupsScope cleanups(CGF);
1379   Visit(E->getSubExpr());
1380 }
1381 
1382 void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1383   QualType T = E->getType();
1384   AggValueSlot Slot = EnsureSlot(T);
1385   EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));
1386 }
1387 
1388 void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1389   QualType T = E->getType();
1390   AggValueSlot Slot = EnsureSlot(T);
1391   EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));
1392 }
1393 
1394 /// Determine whether the given cast kind is known to always convert values
1395 /// with all zero bits in their value representation to values with all zero
1396 /// bits in their value representation.
1397 static bool castPreservesZero(const CastExpr *CE) {
1398   switch (CE->getCastKind()) {
1399     // No-ops.
1400   case CK_NoOp:
1401   case CK_UserDefinedConversion:
1402   case CK_ConstructorConversion:
1403   case CK_BitCast:
1404   case CK_ToUnion:
1405   case CK_ToVoid:
1406     // Conversions between (possibly-complex) integral, (possibly-complex)
1407     // floating-point, and bool.
1408   case CK_BooleanToSignedIntegral:
1409   case CK_FloatingCast:
1410   case CK_FloatingComplexCast:
1411   case CK_FloatingComplexToBoolean:
1412   case CK_FloatingComplexToIntegralComplex:
1413   case CK_FloatingComplexToReal:
1414   case CK_FloatingRealToComplex:
1415   case CK_FloatingToBoolean:
1416   case CK_FloatingToIntegral:
1417   case CK_IntegralCast:
1418   case CK_IntegralComplexCast:
1419   case CK_IntegralComplexToBoolean:
1420   case CK_IntegralComplexToFloatingComplex:
1421   case CK_IntegralComplexToReal:
1422   case CK_IntegralRealToComplex:
1423   case CK_IntegralToBoolean:
1424   case CK_IntegralToFloating:
1425     // Reinterpreting integers as pointers and vice versa.
1426   case CK_IntegralToPointer:
1427   case CK_PointerToIntegral:
1428     // Language extensions.
1429   case CK_VectorSplat:
1430   case CK_MatrixCast:
1431   case CK_NonAtomicToAtomic:
1432   case CK_AtomicToNonAtomic:
1433     return true;
1434 
1435   case CK_BaseToDerivedMemberPointer:
1436   case CK_DerivedToBaseMemberPointer:
1437   case CK_MemberPointerToBoolean:
1438   case CK_NullToMemberPointer:
1439   case CK_ReinterpretMemberPointer:
1440     // FIXME: ABI-dependent.
1441     return false;
1442 
1443   case CK_AnyPointerToBlockPointerCast:
1444   case CK_BlockPointerToObjCPointerCast:
1445   case CK_CPointerToObjCPointerCast:
1446   case CK_ObjCObjectLValueCast:
1447   case CK_IntToOCLSampler:
1448   case CK_ZeroToOCLOpaqueType:
1449     // FIXME: Check these.
1450     return false;
1451 
1452   case CK_FixedPointCast:
1453   case CK_FixedPointToBoolean:
1454   case CK_FixedPointToFloating:
1455   case CK_FixedPointToIntegral:
1456   case CK_FloatingToFixedPoint:
1457   case CK_IntegralToFixedPoint:
1458     // FIXME: Do all fixed-point types represent zero as all 0 bits?
1459     return false;
1460 
1461   case CK_AddressSpaceConversion:
1462   case CK_BaseToDerived:
1463   case CK_DerivedToBase:
1464   case CK_Dynamic:
1465   case CK_NullToPointer:
1466   case CK_PointerToBoolean:
1467     // FIXME: Preserves zeroes only if zero pointers and null pointers have the
1468     // same representation in all involved address spaces.
1469     return false;
1470 
1471   case CK_ARCConsumeObject:
1472   case CK_ARCExtendBlockObject:
1473   case CK_ARCProduceObject:
1474   case CK_ARCReclaimReturnedObject:
1475   case CK_CopyAndAutoreleaseBlockObject:
1476   case CK_ArrayToPointerDecay:
1477   case CK_FunctionToPointerDecay:
1478   case CK_BuiltinFnToFnPtr:
1479   case CK_Dependent:
1480   case CK_LValueBitCast:
1481   case CK_LValueToRValue:
1482   case CK_LValueToRValueBitCast:
1483   case CK_UncheckedDerivedToBase:
1484     return false;
1485   }
1486   llvm_unreachable("Unhandled clang::CastKind enum");
1487 }
1488 
1489 /// isSimpleZero - If emitting this value will obviously just cause a store of
1490 /// zero to memory, return true.  This can return false if uncertain, so it just
1491 /// handles simple cases.
1492 static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1493   E = E->IgnoreParens();
1494   while (auto *CE = dyn_cast<CastExpr>(E)) {
1495     if (!castPreservesZero(CE))
1496       break;
1497     E = CE->getSubExpr()->IgnoreParens();
1498   }
1499 
1500   // 0
1501   if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(E))
1502     return IL->getValue() == 0;
1503   // +0.0
1504   if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(E))
1505     return FL->getValue().isPosZero();
1506   // int()
1507   if ((isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) &&
1508       CGF.getTypes().isZeroInitializable(E->getType()))
1509     return true;
1510   // (int*)0 - Null pointer expressions.
1511   if (const CastExpr *ICE = dyn_cast<CastExpr>(E))
1512     return ICE->getCastKind() == CK_NullToPointer &&
1513            CGF.getTypes().isPointerZeroInitializable(E->getType()) &&
1514            !E->HasSideEffects(CGF.getContext());
1515   // '\0'
1516   if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(E))
1517     return CL->getValue() == 0;
1518 
1519   // Otherwise, hard case: conservatively return false.
1520   return false;
1521 }
1522 
1523 
1524 void
1525 AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1526   QualType type = LV.getType();
1527   // FIXME: Ignore result?
1528   // FIXME: Are initializers affected by volatile?
1529   if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1530     // Storing "i32 0" to a zero'd memory location is a noop.
1531     return;
1532   } else if (isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) {
1533     return EmitNullInitializationToLValue(LV);
1534   } else if (isa<NoInitExpr>(E)) {
1535     // Do nothing.
1536     return;
1537   } else if (type->isReferenceType()) {
1538     RValue RV = CGF.EmitReferenceBindingToExpr(E);
1539     return CGF.EmitStoreThroughLValue(RV, LV);
1540   }
1541 
1542   switch (CGF.getEvaluationKind(type)) {
1543   case TEK_Complex:
1544     CGF.EmitComplexExprIntoLValue(E, LV, /*isInit*/ true);
1545     return;
1546   case TEK_Aggregate:
1547     CGF.EmitAggExpr(
1548         E, AggValueSlot::forLValue(LV, CGF, AggValueSlot::IsDestructed,
1549                                    AggValueSlot::DoesNotNeedGCBarriers,
1550                                    AggValueSlot::IsNotAliased,
1551                                    AggValueSlot::MayOverlap, Dest.isZeroed()));
1552     return;
1553   case TEK_Scalar:
1554     if (LV.isSimple()) {
1555       CGF.EmitScalarInit(E, /*D=*/nullptr, LV, /*Captured=*/false);
1556     } else {
1557       CGF.EmitStoreThroughLValue(RValue::get(CGF.EmitScalarExpr(E)), LV);
1558     }
1559     return;
1560   }
1561   llvm_unreachable("bad evaluation kind");
1562 }
1563 
1564 void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1565   QualType type = lv.getType();
1566 
1567   // If the destination slot is already zeroed out before the aggregate is
1568   // copied into it, we don't have to emit any zeros here.
1569   if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(type))
1570     return;
1571 
1572   if (CGF.hasScalarEvaluationKind(type)) {
1573     // For non-aggregates, we can store the appropriate null constant.
1574     llvm::Value *null = CGF.CGM.EmitNullConstant(type);
1575     // Note that the following is not equivalent to
1576     // EmitStoreThroughBitfieldLValue for ARC types.
1577     if (lv.isBitField()) {
1578       CGF.EmitStoreThroughBitfieldLValue(RValue::get(null), lv);
1579     } else {
1580       assert(lv.isSimple());
1581       CGF.EmitStoreOfScalar(null, lv, /* isInitialization */ true);
1582     }
1583   } else {
1584     // There's a potential optimization opportunity in combining
1585     // memsets; that would be easy for arrays, but relatively
1586     // difficult for structures with the current code.
1587     CGF.EmitNullInitialization(lv.getAddress(CGF), lv.getType());
1588   }
1589 }
1590 
1591 void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1592 #if 0
1593   // FIXME: Assess perf here?  Figure out what cases are worth optimizing here
1594   // (Length of globals? Chunks of zeroed-out space?).
1595   //
1596   // If we can, prefer a copy from a global; this is a lot less code for long
1597   // globals, and it's easier for the current optimizers to analyze.
1598   if (llvm::Constant* C = CGF.CGM.EmitConstantExpr(E, E->getType(), &CGF)) {
1599     llvm::GlobalVariable* GV =
1600     new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1601                              llvm::GlobalValue::InternalLinkage, C, "");
1602     EmitFinalDestCopy(E->getType(), CGF.MakeAddrLValue(GV, E->getType()));
1603     return;
1604   }
1605 #endif
1606   if (E->hadArrayRangeDesignator())
1607     CGF.ErrorUnsupported(E, "GNU array range designator extension");
1608 
1609   if (E->isTransparent())
1610     return Visit(E->getInit(0));
1611 
1612   AggValueSlot Dest = EnsureSlot(E->getType());
1613 
1614   LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1615 
1616   // Handle initialization of an array.
1617   if (E->getType()->isArrayType()) {
1618     auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType());
1619     EmitArrayInit(Dest.getAddress(), AType, E->getType(), E);
1620     return;
1621   }
1622 
1623   assert(E->getType()->isRecordType() && "Only support structs/unions here!");
1624 
1625   // Do struct initialization; this code just sets each individual member
1626   // to the approprate value.  This makes bitfield support automatic;
1627   // the disadvantage is that the generated code is more difficult for
1628   // the optimizer, especially with bitfields.
1629   unsigned NumInitElements = E->getNumInits();
1630   RecordDecl *record = E->getType()->castAs<RecordType>()->getDecl();
1631 
1632   // We'll need to enter cleanup scopes in case any of the element
1633   // initializers throws an exception.
1634   SmallVector<EHScopeStack::stable_iterator, 16> cleanups;
1635   llvm::Instruction *cleanupDominator = nullptr;
1636   auto addCleanup = [&](const EHScopeStack::stable_iterator &cleanup) {
1637     cleanups.push_back(cleanup);
1638     if (!cleanupDominator) // create placeholder once needed
1639       cleanupDominator = CGF.Builder.CreateAlignedLoad(
1640           CGF.Int8Ty, llvm::Constant::getNullValue(CGF.Int8PtrTy),
1641           CharUnits::One());
1642   };
1643 
1644   unsigned curInitIndex = 0;
1645 
1646   // Emit initialization of base classes.
1647   if (auto *CXXRD = dyn_cast<CXXRecordDecl>(record)) {
1648     assert(E->getNumInits() >= CXXRD->getNumBases() &&
1649            "missing initializer for base class");
1650     for (auto &Base : CXXRD->bases()) {
1651       assert(!Base.isVirtual() && "should not see vbases here");
1652       auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1653       Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
1654           Dest.getAddress(), CXXRD, BaseRD,
1655           /*isBaseVirtual*/ false);
1656       AggValueSlot AggSlot = AggValueSlot::forAddr(
1657           V, Qualifiers(),
1658           AggValueSlot::IsDestructed,
1659           AggValueSlot::DoesNotNeedGCBarriers,
1660           AggValueSlot::IsNotAliased,
1661           CGF.getOverlapForBaseInit(CXXRD, BaseRD, Base.isVirtual()));
1662       CGF.EmitAggExpr(E->getInit(curInitIndex++), AggSlot);
1663 
1664       if (QualType::DestructionKind dtorKind =
1665               Base.getType().isDestructedType()) {
1666         CGF.pushDestroy(dtorKind, V, Base.getType());
1667         addCleanup(CGF.EHStack.stable_begin());
1668       }
1669     }
1670   }
1671 
1672   // Prepare a 'this' for CXXDefaultInitExprs.
1673   CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1674 
1675   if (record->isUnion()) {
1676     // Only initialize one field of a union. The field itself is
1677     // specified by the initializer list.
1678     if (!E->getInitializedFieldInUnion()) {
1679       // Empty union; we have nothing to do.
1680 
1681 #ifndef NDEBUG
1682       // Make sure that it's really an empty and not a failure of
1683       // semantic analysis.
1684       for (const auto *Field : record->fields())
1685         assert(Field->isUnnamedBitfield() && "Only unnamed bitfields allowed");
1686 #endif
1687       return;
1688     }
1689 
1690     // FIXME: volatility
1691     FieldDecl *Field = E->getInitializedFieldInUnion();
1692 
1693     LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestLV, Field);
1694     if (NumInitElements) {
1695       // Store the initializer into the field
1696       EmitInitializationToLValue(E->getInit(0), FieldLoc);
1697     } else {
1698       // Default-initialize to null.
1699       EmitNullInitializationToLValue(FieldLoc);
1700     }
1701 
1702     return;
1703   }
1704 
1705   // Here we iterate over the fields; this makes it simpler to both
1706   // default-initialize fields and skip over unnamed fields.
1707   for (const auto *field : record->fields()) {
1708     // We're done once we hit the flexible array member.
1709     if (field->getType()->isIncompleteArrayType())
1710       break;
1711 
1712     // Always skip anonymous bitfields.
1713     if (field->isUnnamedBitfield())
1714       continue;
1715 
1716     // We're done if we reach the end of the explicit initializers, we
1717     // have a zeroed object, and the rest of the fields are
1718     // zero-initializable.
1719     if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1720         CGF.getTypes().isZeroInitializable(E->getType()))
1721       break;
1722 
1723 
1724     LValue LV = CGF.EmitLValueForFieldInitialization(DestLV, field);
1725     // We never generate write-barries for initialized fields.
1726     LV.setNonGC(true);
1727 
1728     if (curInitIndex < NumInitElements) {
1729       // Store the initializer into the field.
1730       EmitInitializationToLValue(E->getInit(curInitIndex++), LV);
1731     } else {
1732       // We're out of initializers; default-initialize to null
1733       EmitNullInitializationToLValue(LV);
1734     }
1735 
1736     // Push a destructor if necessary.
1737     // FIXME: if we have an array of structures, all explicitly
1738     // initialized, we can end up pushing a linear number of cleanups.
1739     bool pushedCleanup = false;
1740     if (QualType::DestructionKind dtorKind
1741           = field->getType().isDestructedType()) {
1742       assert(LV.isSimple());
1743       if (CGF.needsEHCleanup(dtorKind)) {
1744         CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), field->getType(),
1745                         CGF.getDestroyer(dtorKind), false);
1746         addCleanup(CGF.EHStack.stable_begin());
1747         pushedCleanup = true;
1748       }
1749     }
1750 
1751     // If the GEP didn't get used because of a dead zero init or something
1752     // else, clean it up for -O0 builds and general tidiness.
1753     if (!pushedCleanup && LV.isSimple())
1754       if (llvm::GetElementPtrInst *GEP =
1755               dyn_cast<llvm::GetElementPtrInst>(LV.getPointer(CGF)))
1756         if (GEP->use_empty())
1757           GEP->eraseFromParent();
1758   }
1759 
1760   // Deactivate all the partial cleanups in reverse order, which
1761   // generally means popping them.
1762   assert((cleanupDominator || cleanups.empty()) &&
1763          "Missing cleanupDominator before deactivating cleanup blocks");
1764   for (unsigned i = cleanups.size(); i != 0; --i)
1765     CGF.DeactivateCleanupBlock(cleanups[i-1], cleanupDominator);
1766 
1767   // Destroy the placeholder if we made one.
1768   if (cleanupDominator)
1769     cleanupDominator->eraseFromParent();
1770 }
1771 
1772 void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
1773                                             llvm::Value *outerBegin) {
1774   // Emit the common subexpression.
1775   CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
1776 
1777   Address destPtr = EnsureSlot(E->getType()).getAddress();
1778   uint64_t numElements = E->getArraySize().getZExtValue();
1779 
1780   if (!numElements)
1781     return;
1782 
1783   // destPtr is an array*. Construct an elementType* by drilling down a level.
1784   llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
1785   llvm::Value *indices[] = {zero, zero};
1786   llvm::Value *begin = Builder.CreateInBoundsGEP(
1787       destPtr.getElementType(), destPtr.getPointer(), indices,
1788       "arrayinit.begin");
1789 
1790   // Prepare to special-case multidimensional array initialization: we avoid
1791   // emitting multiple destructor loops in that case.
1792   if (!outerBegin)
1793     outerBegin = begin;
1794   ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(E->getSubExpr());
1795 
1796   QualType elementType =
1797       CGF.getContext().getAsArrayType(E->getType())->getElementType();
1798   CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
1799   CharUnits elementAlign =
1800       destPtr.getAlignment().alignmentOfArrayElement(elementSize);
1801 
1802   llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1803   llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body");
1804 
1805   // Jump into the body.
1806   CGF.EmitBlock(bodyBB);
1807   llvm::PHINode *index =
1808       Builder.CreatePHI(zero->getType(), 2, "arrayinit.index");
1809   index->addIncoming(zero, entryBB);
1810   llvm::Value *element = Builder.CreateInBoundsGEP(
1811       begin->getType()->getPointerElementType(), begin, index);
1812 
1813   // Prepare for a cleanup.
1814   QualType::DestructionKind dtorKind = elementType.isDestructedType();
1815   EHScopeStack::stable_iterator cleanup;
1816   if (CGF.needsEHCleanup(dtorKind) && !InnerLoop) {
1817     if (outerBegin->getType() != element->getType())
1818       outerBegin = Builder.CreateBitCast(outerBegin, element->getType());
1819     CGF.pushRegularPartialArrayCleanup(outerBegin, element, elementType,
1820                                        elementAlign,
1821                                        CGF.getDestroyer(dtorKind));
1822     cleanup = CGF.EHStack.stable_begin();
1823   } else {
1824     dtorKind = QualType::DK_none;
1825   }
1826 
1827   // Emit the actual filler expression.
1828   {
1829     // Temporaries created in an array initialization loop are destroyed
1830     // at the end of each iteration.
1831     CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
1832     CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
1833     LValue elementLV =
1834         CGF.MakeAddrLValue(Address(element, elementAlign), elementType);
1835 
1836     if (InnerLoop) {
1837       // If the subexpression is an ArrayInitLoopExpr, share its cleanup.
1838       auto elementSlot = AggValueSlot::forLValue(
1839           elementLV, CGF, AggValueSlot::IsDestructed,
1840           AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
1841           AggValueSlot::DoesNotOverlap);
1842       AggExprEmitter(CGF, elementSlot, false)
1843           .VisitArrayInitLoopExpr(InnerLoop, outerBegin);
1844     } else
1845       EmitInitializationToLValue(E->getSubExpr(), elementLV);
1846   }
1847 
1848   // Move on to the next element.
1849   llvm::Value *nextIndex = Builder.CreateNUWAdd(
1850       index, llvm::ConstantInt::get(CGF.SizeTy, 1), "arrayinit.next");
1851   index->addIncoming(nextIndex, Builder.GetInsertBlock());
1852 
1853   // Leave the loop if we're done.
1854   llvm::Value *done = Builder.CreateICmpEQ(
1855       nextIndex, llvm::ConstantInt::get(CGF.SizeTy, numElements),
1856       "arrayinit.done");
1857   llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end");
1858   Builder.CreateCondBr(done, endBB, bodyBB);
1859 
1860   CGF.EmitBlock(endBB);
1861 
1862   // Leave the partial-array cleanup if we entered one.
1863   if (dtorKind)
1864     CGF.DeactivateCleanupBlock(cleanup, index);
1865 }
1866 
1867 void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) {
1868   AggValueSlot Dest = EnsureSlot(E->getType());
1869 
1870   LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1871   EmitInitializationToLValue(E->getBase(), DestLV);
1872   VisitInitListExpr(E->getUpdater());
1873 }
1874 
1875 //===----------------------------------------------------------------------===//
1876 //                        Entry Points into this File
1877 //===----------------------------------------------------------------------===//
1878 
1879 /// GetNumNonZeroBytesInInit - Get an approximate count of the number of
1880 /// non-zero bytes that will be stored when outputting the initializer for the
1881 /// specified initializer expression.
1882 static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
1883   if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(E))
1884     E = MTE->getSubExpr();
1885   E = E->IgnoreParenNoopCasts(CGF.getContext());
1886 
1887   // 0 and 0.0 won't require any non-zero stores!
1888   if (isSimpleZero(E, CGF)) return CharUnits::Zero();
1889 
1890   // If this is an initlist expr, sum up the size of sizes of the (present)
1891   // elements.  If this is something weird, assume the whole thing is non-zero.
1892   const InitListExpr *ILE = dyn_cast<InitListExpr>(E);
1893   while (ILE && ILE->isTransparent())
1894     ILE = dyn_cast<InitListExpr>(ILE->getInit(0));
1895   if (!ILE || !CGF.getTypes().isZeroInitializable(ILE->getType()))
1896     return CGF.getContext().getTypeSizeInChars(E->getType());
1897 
1898   // InitListExprs for structs have to be handled carefully.  If there are
1899   // reference members, we need to consider the size of the reference, not the
1900   // referencee.  InitListExprs for unions and arrays can't have references.
1901   if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
1902     if (!RT->isUnionType()) {
1903       RecordDecl *SD = RT->getDecl();
1904       CharUnits NumNonZeroBytes = CharUnits::Zero();
1905 
1906       unsigned ILEElement = 0;
1907       if (auto *CXXRD = dyn_cast<CXXRecordDecl>(SD))
1908         while (ILEElement != CXXRD->getNumBases())
1909           NumNonZeroBytes +=
1910               GetNumNonZeroBytesInInit(ILE->getInit(ILEElement++), CGF);
1911       for (const auto *Field : SD->fields()) {
1912         // We're done once we hit the flexible array member or run out of
1913         // InitListExpr elements.
1914         if (Field->getType()->isIncompleteArrayType() ||
1915             ILEElement == ILE->getNumInits())
1916           break;
1917         if (Field->isUnnamedBitfield())
1918           continue;
1919 
1920         const Expr *E = ILE->getInit(ILEElement++);
1921 
1922         // Reference values are always non-null and have the width of a pointer.
1923         if (Field->getType()->isReferenceType())
1924           NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
1925               CGF.getTarget().getPointerWidth(0));
1926         else
1927           NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
1928       }
1929 
1930       return NumNonZeroBytes;
1931     }
1932   }
1933 
1934   // FIXME: This overestimates the number of non-zero bytes for bit-fields.
1935   CharUnits NumNonZeroBytes = CharUnits::Zero();
1936   for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
1937     NumNonZeroBytes += GetNumNonZeroBytesInInit(ILE->getInit(i), CGF);
1938   return NumNonZeroBytes;
1939 }
1940 
1941 /// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
1942 /// zeros in it, emit a memset and avoid storing the individual zeros.
1943 ///
1944 static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
1945                                      CodeGenFunction &CGF) {
1946   // If the slot is already known to be zeroed, nothing to do.  Don't mess with
1947   // volatile stores.
1948   if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
1949     return;
1950 
1951   // C++ objects with a user-declared constructor don't need zero'ing.
1952   if (CGF.getLangOpts().CPlusPlus)
1953     if (const RecordType *RT = CGF.getContext()
1954                        .getBaseElementType(E->getType())->getAs<RecordType>()) {
1955       const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
1956       if (RD->hasUserDeclaredConstructor())
1957         return;
1958     }
1959 
1960   // If the type is 16-bytes or smaller, prefer individual stores over memset.
1961   CharUnits Size = Slot.getPreferredSize(CGF.getContext(), E->getType());
1962   if (Size <= CharUnits::fromQuantity(16))
1963     return;
1964 
1965   // Check to see if over 3/4 of the initializer are known to be zero.  If so,
1966   // we prefer to emit memset + individual stores for the rest.
1967   CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
1968   if (NumNonZeroBytes*4 > Size)
1969     return;
1970 
1971   // Okay, it seems like a good idea to use an initial memset, emit the call.
1972   llvm::Constant *SizeVal = CGF.Builder.getInt64(Size.getQuantity());
1973 
1974   Address Loc = Slot.getAddress();
1975   Loc = CGF.Builder.CreateElementBitCast(Loc, CGF.Int8Ty);
1976   CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, false);
1977 
1978   // Tell the AggExprEmitter that the slot is known zero.
1979   Slot.setZeroed();
1980 }
1981 
1982 
1983 
1984 
1985 /// EmitAggExpr - Emit the computation of the specified expression of aggregate
1986 /// type.  The result is computed into DestPtr.  Note that if DestPtr is null,
1987 /// the value of the aggregate expression is not needed.  If VolatileDest is
1988 /// true, DestPtr cannot be 0.
1989 void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
1990   assert(E && hasAggregateEvaluationKind(E->getType()) &&
1991          "Invalid aggregate expression to emit");
1992   assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
1993          "slot has bits but no address");
1994 
1995   // Optimize the slot if possible.
1996   CheckAggExprForMemSetUse(Slot, E, *this);
1997 
1998   AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E));
1999 }
2000 
2001 LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
2002   assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
2003   Address Temp = CreateMemTemp(E->getType());
2004   LValue LV = MakeAddrLValue(Temp, E->getType());
2005   EmitAggExpr(E, AggValueSlot::forLValue(
2006                      LV, *this, AggValueSlot::IsNotDestructed,
2007                      AggValueSlot::DoesNotNeedGCBarriers,
2008                      AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap));
2009   return LV;
2010 }
2011 
2012 AggValueSlot::Overlap_t
2013 CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {
2014   if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2015     return AggValueSlot::DoesNotOverlap;
2016 
2017   // If the field lies entirely within the enclosing class's nvsize, its tail
2018   // padding cannot overlap any already-initialized object. (The only subobjects
2019   // with greater addresses that might already be initialized are vbases.)
2020   const RecordDecl *ClassRD = FD->getParent();
2021   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(ClassRD);
2022   if (Layout.getFieldOffset(FD->getFieldIndex()) +
2023           getContext().getTypeSize(FD->getType()) <=
2024       (uint64_t)getContext().toBits(Layout.getNonVirtualSize()))
2025     return AggValueSlot::DoesNotOverlap;
2026 
2027   // The tail padding may contain values we need to preserve.
2028   return AggValueSlot::MayOverlap;
2029 }
2030 
2031 AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
2032     const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2033   // If the most-derived object is a field declared with [[no_unique_address]],
2034   // the tail padding of any virtual base could be reused for other subobjects
2035   // of that field's class.
2036   if (IsVirtual)
2037     return AggValueSlot::MayOverlap;
2038 
2039   // If the base class is laid out entirely within the nvsize of the derived
2040   // class, its tail padding cannot yet be initialized, so we can issue
2041   // stores at the full width of the base class.
2042   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2043   if (Layout.getBaseClassOffset(BaseRD) +
2044           getContext().getASTRecordLayout(BaseRD).getSize() <=
2045       Layout.getNonVirtualSize())
2046     return AggValueSlot::DoesNotOverlap;
2047 
2048   // The tail padding may contain values we need to preserve.
2049   return AggValueSlot::MayOverlap;
2050 }
2051 
2052 void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
2053                                         AggValueSlot::Overlap_t MayOverlap,
2054                                         bool isVolatile) {
2055   assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2056 
2057   Address DestPtr = Dest.getAddress(*this);
2058   Address SrcPtr = Src.getAddress(*this);
2059 
2060   if (getLangOpts().CPlusPlus) {
2061     if (const RecordType *RT = Ty->getAs<RecordType>()) {
2062       CXXRecordDecl *Record = cast<CXXRecordDecl>(RT->getDecl());
2063       assert((Record->hasTrivialCopyConstructor() ||
2064               Record->hasTrivialCopyAssignment() ||
2065               Record->hasTrivialMoveConstructor() ||
2066               Record->hasTrivialMoveAssignment() ||
2067               Record->hasAttr<TrivialABIAttr>() || Record->isUnion()) &&
2068              "Trying to aggregate-copy a type without a trivial copy/move "
2069              "constructor or assignment operator");
2070       // Ignore empty classes in C++.
2071       if (Record->isEmpty())
2072         return;
2073     }
2074   }
2075 
2076   if (getLangOpts().CUDAIsDevice) {
2077     if (Ty->isCUDADeviceBuiltinSurfaceType()) {
2078       if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(*this, Dest,
2079                                                                   Src))
2080         return;
2081     } else if (Ty->isCUDADeviceBuiltinTextureType()) {
2082       if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(*this, Dest,
2083                                                                   Src))
2084         return;
2085     }
2086   }
2087 
2088   // Aggregate assignment turns into llvm.memcpy.  This is almost valid per
2089   // C99 6.5.16.1p3, which states "If the value being stored in an object is
2090   // read from another object that overlaps in anyway the storage of the first
2091   // object, then the overlap shall be exact and the two objects shall have
2092   // qualified or unqualified versions of a compatible type."
2093   //
2094   // memcpy is not defined if the source and destination pointers are exactly
2095   // equal, but other compilers do this optimization, and almost every memcpy
2096   // implementation handles this case safely.  If there is a libc that does not
2097   // safely handle this, we can add a target hook.
2098 
2099   // Get data size info for this aggregate. Don't copy the tail padding if this
2100   // might be a potentially-overlapping subobject, since the tail padding might
2101   // be occupied by a different object. Otherwise, copying it is fine.
2102   TypeInfoChars TypeInfo;
2103   if (MayOverlap)
2104     TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty);
2105   else
2106     TypeInfo = getContext().getTypeInfoInChars(Ty);
2107 
2108   llvm::Value *SizeVal = nullptr;
2109   if (TypeInfo.Width.isZero()) {
2110     // But note that getTypeInfo returns 0 for a VLA.
2111     if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2112             getContext().getAsArrayType(Ty))) {
2113       QualType BaseEltTy;
2114       SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr);
2115       TypeInfo = getContext().getTypeInfoInChars(BaseEltTy);
2116       assert(!TypeInfo.Width.isZero());
2117       SizeVal = Builder.CreateNUWMul(
2118           SizeVal,
2119           llvm::ConstantInt::get(SizeTy, TypeInfo.Width.getQuantity()));
2120     }
2121   }
2122   if (!SizeVal) {
2123     SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.Width.getQuantity());
2124   }
2125 
2126   // FIXME: If we have a volatile struct, the optimizer can remove what might
2127   // appear to be `extra' memory ops:
2128   //
2129   // volatile struct { int i; } a, b;
2130   //
2131   // int main() {
2132   //   a = b;
2133   //   a = b;
2134   // }
2135   //
2136   // we need to use a different call here.  We use isVolatile to indicate when
2137   // either the source or the destination is volatile.
2138 
2139   DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
2140   SrcPtr = Builder.CreateElementBitCast(SrcPtr, Int8Ty);
2141 
2142   // Don't do any of the memmove_collectable tests if GC isn't set.
2143   if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2144     // fall through
2145   } else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
2146     RecordDecl *Record = RecordTy->getDecl();
2147     if (Record->hasObjectMember()) {
2148       CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
2149                                                     SizeVal);
2150       return;
2151     }
2152   } else if (Ty->isArrayType()) {
2153     QualType BaseType = getContext().getBaseElementType(Ty);
2154     if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) {
2155       if (RecordTy->getDecl()->hasObjectMember()) {
2156         CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
2157                                                       SizeVal);
2158         return;
2159       }
2160     }
2161   }
2162 
2163   auto Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile);
2164 
2165   // Determine the metadata to describe the position of any padding in this
2166   // memcpy, as well as the TBAA tags for the members of the struct, in case
2167   // the optimizer wishes to expand it in to scalar memory operations.
2168   if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty))
2169     Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag);
2170 
2171   if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2172     TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2173         Dest.getTBAAInfo(), Src.getTBAAInfo());
2174     CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2175   }
2176 }
2177