xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp (revision 02e9120893770924227138ba49df1edb3896112a) 
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
13 #include "clang/Basic/CodeGenOptions.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 #include <optional>
21 using namespace clang::CodeGen;
22 using namespace llvm;
23 
24 MDNode *
25 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
26   LLVMContext &Ctx = Header->getContext();
27   SmallVector<Metadata *, 4> NewLoopProperties;
28   NewLoopProperties.push_back(nullptr);
29   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
30 
31   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
32   LoopID->replaceOperandWith(0, LoopID);
33   return LoopID;
34 }
35 
36 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37                                            ArrayRef<Metadata *> LoopProperties,
38                                            bool &HasUserTransforms) {
39   LLVMContext &Ctx = Header->getContext();
40 
41   std::optional<bool> Enabled;
42   if (Attrs.PipelineDisabled)
43     Enabled = false;
44   else if (Attrs.PipelineInitiationInterval != 0)
45     Enabled = true;
46 
47   if (Enabled != true) {
48     SmallVector<Metadata *, 4> NewLoopProperties;
49     if (Enabled == false) {
50       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
51       NewLoopProperties.push_back(
52           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
53                             ConstantAsMetadata::get(ConstantInt::get(
54                                 llvm::Type::getInt1Ty(Ctx), 1))}));
55       LoopProperties = NewLoopProperties;
56     }
57     return createLoopPropertiesMetadata(LoopProperties);
58   }
59 
60   SmallVector<Metadata *, 4> Args;
61   Args.push_back(nullptr);
62   Args.append(LoopProperties.begin(), LoopProperties.end());
63 
64   if (Attrs.PipelineInitiationInterval > 0) {
65     Metadata *Vals[] = {
66         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
67         ConstantAsMetadata::get(ConstantInt::get(
68             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
69     Args.push_back(MDNode::get(Ctx, Vals));
70   }
71 
72   // No follow-up: This is the last transformation.
73 
74   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
75   LoopID->replaceOperandWith(0, LoopID);
76   HasUserTransforms = true;
77   return LoopID;
78 }
79 
80 MDNode *
81 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
82                                       ArrayRef<Metadata *> LoopProperties,
83                                       bool &HasUserTransforms) {
84   LLVMContext &Ctx = Header->getContext();
85 
86   std::optional<bool> Enabled;
87   if (Attrs.UnrollEnable == LoopAttributes::Disable)
88     Enabled = false;
89   else if (Attrs.UnrollEnable == LoopAttributes::Full)
90     Enabled = std::nullopt;
91   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
92            Attrs.UnrollCount != 0)
93     Enabled = true;
94 
95   if (Enabled != true) {
96     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
97     // if unrolling is disabled.
98     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
99   }
100 
101   SmallVector<Metadata *, 4> FollowupLoopProperties;
102 
103   // Apply all loop properties to the unrolled loop.
104   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
105 
106   // Don't unroll an already unrolled loop.
107   FollowupLoopProperties.push_back(
108       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
109 
110   bool FollowupHasTransforms = false;
111   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
112                                               FollowupHasTransforms);
113 
114   SmallVector<Metadata *, 4> Args;
115   Args.push_back(nullptr);
116   Args.append(LoopProperties.begin(), LoopProperties.end());
117 
118   // Setting unroll.count
119   if (Attrs.UnrollCount > 0) {
120     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121                         ConstantAsMetadata::get(ConstantInt::get(
122                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123     Args.push_back(MDNode::get(Ctx, Vals));
124   }
125 
126   // Setting unroll.full or unroll.disable
127   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129     Args.push_back(MDNode::get(Ctx, Vals));
130   }
131 
132   if (FollowupHasTransforms)
133     Args.push_back(MDNode::get(
134         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135 
136   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137   LoopID->replaceOperandWith(0, LoopID);
138   HasUserTransforms = true;
139   return LoopID;
140 }
141 
142 MDNode *
143 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144                                      ArrayRef<Metadata *> LoopProperties,
145                                      bool &HasUserTransforms) {
146   LLVMContext &Ctx = Header->getContext();
147 
148   std::optional<bool> Enabled;
149   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
150     Enabled = false;
151   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152            Attrs.UnrollAndJamCount != 0)
153     Enabled = true;
154 
155   if (Enabled != true) {
156     SmallVector<Metadata *, 4> NewLoopProperties;
157     if (Enabled == false) {
158       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159       NewLoopProperties.push_back(MDNode::get(
160           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161       LoopProperties = NewLoopProperties;
162     }
163     return createPartialUnrollMetadata(Attrs, LoopProperties,
164                                        HasUserTransforms);
165   }
166 
167   SmallVector<Metadata *, 4> FollowupLoopProperties;
168   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169   FollowupLoopProperties.push_back(
170       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171 
172   bool FollowupHasTransforms = false;
173   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174                                                  FollowupHasTransforms);
175 
176   SmallVector<Metadata *, 4> Args;
177   Args.push_back(nullptr);
178   Args.append(LoopProperties.begin(), LoopProperties.end());
179 
180   // Setting unroll_and_jam.count
181   if (Attrs.UnrollAndJamCount > 0) {
182     Metadata *Vals[] = {
183         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
184         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
185                                                  Attrs.UnrollAndJamCount))};
186     Args.push_back(MDNode::get(Ctx, Vals));
187   }
188 
189   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
190     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
191     Args.push_back(MDNode::get(Ctx, Vals));
192   }
193 
194   if (FollowupHasTransforms)
195     Args.push_back(MDNode::get(
196         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
197               Followup}));
198 
199   if (UnrollAndJamInnerFollowup)
200     Args.push_back(MDNode::get(
201         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
202               UnrollAndJamInnerFollowup}));
203 
204   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
205   LoopID->replaceOperandWith(0, LoopID);
206   HasUserTransforms = true;
207   return LoopID;
208 }
209 
210 MDNode *
211 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
212                                       ArrayRef<Metadata *> LoopProperties,
213                                       bool &HasUserTransforms) {
214   LLVMContext &Ctx = Header->getContext();
215 
216   std::optional<bool> Enabled;
217   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
218     Enabled = false;
219   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
220            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
221            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
222            Attrs.VectorizeScalable != LoopAttributes::Unspecified)
223     Enabled = true;
224 
225   if (Enabled != true) {
226     SmallVector<Metadata *, 4> NewLoopProperties;
227     if (Enabled == false) {
228       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229       NewLoopProperties.push_back(
230           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
231                             ConstantAsMetadata::get(ConstantInt::get(
232                                 llvm::Type::getInt1Ty(Ctx), 0))}));
233       LoopProperties = NewLoopProperties;
234     }
235     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
236   }
237 
238   // Apply all loop properties to the vectorized loop.
239   SmallVector<Metadata *, 4> FollowupLoopProperties;
240   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
241 
242   // Don't vectorize an already vectorized loop.
243   FollowupLoopProperties.push_back(
244       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
245 
246   bool FollowupHasTransforms = false;
247   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248                                                 FollowupHasTransforms);
249 
250   SmallVector<Metadata *, 4> Args;
251   Args.push_back(nullptr);
252   Args.append(LoopProperties.begin(), LoopProperties.end());
253 
254   // Setting vectorize.predicate when it has been specified and vectorization
255   // has not been disabled.
256   bool IsVectorPredicateEnabled = false;
257   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
258     IsVectorPredicateEnabled =
259         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
260 
261     Metadata *Vals[] = {
262         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
263         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
264                                                  IsVectorPredicateEnabled))};
265     Args.push_back(MDNode::get(Ctx, Vals));
266   }
267 
268   // Setting vectorize.width
269   if (Attrs.VectorizeWidth > 0) {
270     Metadata *Vals[] = {
271         MDString::get(Ctx, "llvm.loop.vectorize.width"),
272         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
273                                                  Attrs.VectorizeWidth))};
274 
275     Args.push_back(MDNode::get(Ctx, Vals));
276   }
277 
278   if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
279     bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
280     Metadata *Vals[] = {
281         MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
282         ConstantAsMetadata::get(
283             ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
284     Args.push_back(MDNode::get(Ctx, Vals));
285   }
286 
287   // Setting interleave.count
288   if (Attrs.InterleaveCount > 0) {
289     Metadata *Vals[] = {
290         MDString::get(Ctx, "llvm.loop.interleave.count"),
291         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
292                                                  Attrs.InterleaveCount))};
293     Args.push_back(MDNode::get(Ctx, Vals));
294   }
295 
296   // vectorize.enable is set if:
297   // 1) loop hint vectorize.enable is set, or
298   // 2) it is implied when vectorize.predicate is set, or
299   // 3) it is implied when vectorize.width is set to a value > 1
300   // 4) it is implied when vectorize.scalable.enable is true
301   // 5) it is implied when vectorize.width is unset (0) and the user
302   //    explicitly requested fixed-width vectorization, i.e.
303   //    vectorize.scalable.enable is false.
304   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
305       (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
306       Attrs.VectorizeWidth > 1 ||
307       Attrs.VectorizeScalable == LoopAttributes::Enable ||
308       (Attrs.VectorizeScalable == LoopAttributes::Disable &&
309        Attrs.VectorizeWidth != 1)) {
310     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
311     Args.push_back(
312         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
313                           ConstantAsMetadata::get(ConstantInt::get(
314                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
315   }
316 
317   if (FollowupHasTransforms)
318     Args.push_back(MDNode::get(
319         Ctx,
320         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
321 
322   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
323   LoopID->replaceOperandWith(0, LoopID);
324   HasUserTransforms = true;
325   return LoopID;
326 }
327 
328 MDNode *
329 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
330                                        ArrayRef<Metadata *> LoopProperties,
331                                        bool &HasUserTransforms) {
332   LLVMContext &Ctx = Header->getContext();
333 
334   std::optional<bool> Enabled;
335   if (Attrs.DistributeEnable == LoopAttributes::Disable)
336     Enabled = false;
337   if (Attrs.DistributeEnable == LoopAttributes::Enable)
338     Enabled = true;
339 
340   if (Enabled != true) {
341     SmallVector<Metadata *, 4> NewLoopProperties;
342     if (Enabled == false) {
343       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
344       NewLoopProperties.push_back(
345           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
346                             ConstantAsMetadata::get(ConstantInt::get(
347                                 llvm::Type::getInt1Ty(Ctx), 0))}));
348       LoopProperties = NewLoopProperties;
349     }
350     return createLoopVectorizeMetadata(Attrs, LoopProperties,
351                                        HasUserTransforms);
352   }
353 
354   bool FollowupHasTransforms = false;
355   MDNode *Followup =
356       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
357 
358   SmallVector<Metadata *, 4> Args;
359   Args.push_back(nullptr);
360   Args.append(LoopProperties.begin(), LoopProperties.end());
361 
362   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
363                       ConstantAsMetadata::get(ConstantInt::get(
364                           llvm::Type::getInt1Ty(Ctx),
365                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
366   Args.push_back(MDNode::get(Ctx, Vals));
367 
368   if (FollowupHasTransforms)
369     Args.push_back(MDNode::get(
370         Ctx,
371         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
372 
373   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
374   LoopID->replaceOperandWith(0, LoopID);
375   HasUserTransforms = true;
376   return LoopID;
377 }
378 
379 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
380                                            ArrayRef<Metadata *> LoopProperties,
381                                            bool &HasUserTransforms) {
382   LLVMContext &Ctx = Header->getContext();
383 
384   std::optional<bool> Enabled;
385   if (Attrs.UnrollEnable == LoopAttributes::Disable)
386     Enabled = false;
387   else if (Attrs.UnrollEnable == LoopAttributes::Full)
388     Enabled = true;
389 
390   if (Enabled != true) {
391     SmallVector<Metadata *, 4> NewLoopProperties;
392     if (Enabled == false) {
393       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
394       NewLoopProperties.push_back(
395           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
396       LoopProperties = NewLoopProperties;
397     }
398     return createLoopDistributeMetadata(Attrs, LoopProperties,
399                                         HasUserTransforms);
400   }
401 
402   SmallVector<Metadata *, 4> Args;
403   Args.push_back(nullptr);
404   Args.append(LoopProperties.begin(), LoopProperties.end());
405   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
406 
407   // No follow-up: there is no loop after full unrolling.
408   // TODO: Warn if there are transformations after full unrolling.
409 
410   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
411   LoopID->replaceOperandWith(0, LoopID);
412   HasUserTransforms = true;
413   return LoopID;
414 }
415 
416 MDNode *LoopInfo::createMetadata(
417     const LoopAttributes &Attrs,
418     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
419     bool &HasUserTransforms) {
420   SmallVector<Metadata *, 3> LoopProperties;
421 
422   // If we have a valid start debug location for the loop, add it.
423   if (StartLoc) {
424     LoopProperties.push_back(StartLoc.getAsMDNode());
425 
426     // If we also have a valid end debug location for the loop, add it.
427     if (EndLoc)
428       LoopProperties.push_back(EndLoc.getAsMDNode());
429   }
430 
431   LLVMContext &Ctx = Header->getContext();
432   if (Attrs.MustProgress)
433     LoopProperties.push_back(
434         MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
435 
436   assert(!!AccGroup == Attrs.IsParallel &&
437          "There must be an access group iff the loop is parallel");
438   if (Attrs.IsParallel) {
439     LoopProperties.push_back(MDNode::get(
440         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
441   }
442 
443   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
444                         AdditionalLoopProperties.end());
445   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
446 }
447 
448 LoopAttributes::LoopAttributes(bool IsParallel)
449     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
450       UnrollEnable(LoopAttributes::Unspecified),
451       UnrollAndJamEnable(LoopAttributes::Unspecified),
452       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
453       VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
454       UnrollCount(0), UnrollAndJamCount(0),
455       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
456       PipelineInitiationInterval(0), MustProgress(false) {}
457 
458 void LoopAttributes::clear() {
459   IsParallel = false;
460   VectorizeWidth = 0;
461   VectorizeScalable = LoopAttributes::Unspecified;
462   InterleaveCount = 0;
463   UnrollCount = 0;
464   UnrollAndJamCount = 0;
465   VectorizeEnable = LoopAttributes::Unspecified;
466   UnrollEnable = LoopAttributes::Unspecified;
467   UnrollAndJamEnable = LoopAttributes::Unspecified;
468   VectorizePredicateEnable = LoopAttributes::Unspecified;
469   DistributeEnable = LoopAttributes::Unspecified;
470   PipelineDisabled = false;
471   PipelineInitiationInterval = 0;
472   MustProgress = false;
473 }
474 
475 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
476                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
477                    LoopInfo *Parent)
478     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
479       Parent(Parent) {
480 
481   if (Attrs.IsParallel) {
482     // Create an access group for this loop.
483     LLVMContext &Ctx = Header->getContext();
484     AccGroup = MDNode::getDistinct(Ctx, {});
485   }
486 
487   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
488       Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
489       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
490       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
491       Attrs.PipelineInitiationInterval == 0 &&
492       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
493       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
494       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
495       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
496       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
497       !EndLoc && !Attrs.MustProgress)
498     return;
499 
500   TempLoopID = MDNode::getTemporary(Header->getContext(), std::nullopt);
501 }
502 
503 void LoopInfo::finish() {
504   // We did not annotate the loop body instructions because there are no
505   // attributes for this loop.
506   if (!TempLoopID)
507     return;
508 
509   MDNode *LoopID;
510   LoopAttributes CurLoopAttr = Attrs;
511   LLVMContext &Ctx = Header->getContext();
512 
513   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
514                  Parent->Attrs.UnrollAndJamCount != 0)) {
515     // Parent unroll-and-jams this loop.
516     // Split the transformations in those that happens before the unroll-and-jam
517     // and those after.
518 
519     LoopAttributes BeforeJam, AfterJam;
520 
521     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
522 
523     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
524     BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
525     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
526     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
527     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
528     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
529 
530     switch (Attrs.UnrollEnable) {
531     case LoopAttributes::Unspecified:
532     case LoopAttributes::Disable:
533       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
534       AfterJam.UnrollEnable = Attrs.UnrollEnable;
535       break;
536     case LoopAttributes::Full:
537       BeforeJam.UnrollEnable = LoopAttributes::Full;
538       break;
539     case LoopAttributes::Enable:
540       AfterJam.UnrollEnable = LoopAttributes::Enable;
541       break;
542     }
543 
544     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
545     AfterJam.UnrollCount = Attrs.UnrollCount;
546     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
547     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
548 
549     // If this loop is subject of an unroll-and-jam by the parent loop, and has
550     // an unroll-and-jam annotation itself, we have to decide whether to first
551     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
552     // UnrollAndJam pass processes loops from inner to outer, so we apply the
553     // inner first.
554     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
555     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
556 
557     // Set the inner followup metadata to process by the outer loop. Only
558     // consider the first inner loop.
559     if (!Parent->UnrollAndJamInnerFollowup) {
560       // Splitting the attributes into a BeforeJam and an AfterJam part will
561       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
562       // to be forwarded to the AfterJam part. We detect the situation here and
563       // add it manually.
564       SmallVector<Metadata *, 1> BeforeLoopProperties;
565       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
566           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
567           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
568           BeforeJam.VectorizeScalable == LoopAttributes::Enable)
569         BeforeLoopProperties.push_back(
570             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
571 
572       bool InnerFollowupHasTransform = false;
573       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
574                                              InnerFollowupHasTransform);
575       if (InnerFollowupHasTransform)
576         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
577     }
578 
579     CurLoopAttr = BeforeJam;
580   }
581 
582   bool HasUserTransforms = false;
583   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
584   TempLoopID->replaceAllUsesWith(LoopID);
585 }
586 
587 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
588                          const llvm::DebugLoc &EndLoc) {
589   Active.emplace_back(
590       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
591                    Active.empty() ? nullptr : Active.back().get()));
592   // Clear the attributes so nested loops do not inherit them.
593   StagedAttrs.clear();
594 }
595 
596 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
597                          const clang::CodeGenOptions &CGOpts,
598                          ArrayRef<const clang::Attr *> Attrs,
599                          const llvm::DebugLoc &StartLoc,
600                          const llvm::DebugLoc &EndLoc, bool MustProgress) {
601   // Identify loop hint attributes from Attrs.
602   for (const auto *Attr : Attrs) {
603     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
604     const OpenCLUnrollHintAttr *OpenCLHint =
605         dyn_cast<OpenCLUnrollHintAttr>(Attr);
606 
607     // Skip non loop hint attributes
608     if (!LH && !OpenCLHint) {
609       continue;
610     }
611 
612     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
613     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
614     unsigned ValueInt = 1;
615     // Translate opencl_unroll_hint attribute argument to
616     // equivalent LoopHintAttr enums.
617     // OpenCL v2.0 s6.11.5:
618     // 0 - enable unroll (no argument).
619     // 1 - disable unroll.
620     // other positive integer n - unroll by n.
621     if (OpenCLHint) {
622       ValueInt = OpenCLHint->getUnrollHint();
623       if (ValueInt == 0) {
624         State = LoopHintAttr::Enable;
625       } else if (ValueInt != 1) {
626         Option = LoopHintAttr::UnrollCount;
627         State = LoopHintAttr::Numeric;
628       }
629     } else if (LH) {
630       auto *ValueExpr = LH->getValue();
631       if (ValueExpr) {
632         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
633         ValueInt = ValueAPS.getSExtValue();
634       }
635 
636       Option = LH->getOption();
637       State = LH->getState();
638     }
639     switch (State) {
640     case LoopHintAttr::Disable:
641       switch (Option) {
642       case LoopHintAttr::Vectorize:
643         // Disable vectorization by specifying a width of 1.
644         setVectorizeWidth(1);
645         setVectorizeScalable(LoopAttributes::Unspecified);
646         break;
647       case LoopHintAttr::Interleave:
648         // Disable interleaving by speciyfing a count of 1.
649         setInterleaveCount(1);
650         break;
651       case LoopHintAttr::Unroll:
652         setUnrollState(LoopAttributes::Disable);
653         break;
654       case LoopHintAttr::UnrollAndJam:
655         setUnrollAndJamState(LoopAttributes::Disable);
656         break;
657       case LoopHintAttr::VectorizePredicate:
658         setVectorizePredicateState(LoopAttributes::Disable);
659         break;
660       case LoopHintAttr::Distribute:
661         setDistributeState(false);
662         break;
663       case LoopHintAttr::PipelineDisabled:
664         setPipelineDisabled(true);
665         break;
666       case LoopHintAttr::UnrollCount:
667       case LoopHintAttr::UnrollAndJamCount:
668       case LoopHintAttr::VectorizeWidth:
669       case LoopHintAttr::InterleaveCount:
670       case LoopHintAttr::PipelineInitiationInterval:
671         llvm_unreachable("Options cannot be disabled.");
672         break;
673       }
674       break;
675     case LoopHintAttr::Enable:
676       switch (Option) {
677       case LoopHintAttr::Vectorize:
678       case LoopHintAttr::Interleave:
679         setVectorizeEnable(true);
680         break;
681       case LoopHintAttr::Unroll:
682         setUnrollState(LoopAttributes::Enable);
683         break;
684       case LoopHintAttr::UnrollAndJam:
685         setUnrollAndJamState(LoopAttributes::Enable);
686         break;
687       case LoopHintAttr::VectorizePredicate:
688         setVectorizePredicateState(LoopAttributes::Enable);
689         break;
690       case LoopHintAttr::Distribute:
691         setDistributeState(true);
692         break;
693       case LoopHintAttr::UnrollCount:
694       case LoopHintAttr::UnrollAndJamCount:
695       case LoopHintAttr::VectorizeWidth:
696       case LoopHintAttr::InterleaveCount:
697       case LoopHintAttr::PipelineDisabled:
698       case LoopHintAttr::PipelineInitiationInterval:
699         llvm_unreachable("Options cannot enabled.");
700         break;
701       }
702       break;
703     case LoopHintAttr::AssumeSafety:
704       switch (Option) {
705       case LoopHintAttr::Vectorize:
706       case LoopHintAttr::Interleave:
707         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
708         setParallel(true);
709         setVectorizeEnable(true);
710         break;
711       case LoopHintAttr::Unroll:
712       case LoopHintAttr::UnrollAndJam:
713       case LoopHintAttr::VectorizePredicate:
714       case LoopHintAttr::UnrollCount:
715       case LoopHintAttr::UnrollAndJamCount:
716       case LoopHintAttr::VectorizeWidth:
717       case LoopHintAttr::InterleaveCount:
718       case LoopHintAttr::Distribute:
719       case LoopHintAttr::PipelineDisabled:
720       case LoopHintAttr::PipelineInitiationInterval:
721         llvm_unreachable("Options cannot be used to assume mem safety.");
722         break;
723       }
724       break;
725     case LoopHintAttr::Full:
726       switch (Option) {
727       case LoopHintAttr::Unroll:
728         setUnrollState(LoopAttributes::Full);
729         break;
730       case LoopHintAttr::UnrollAndJam:
731         setUnrollAndJamState(LoopAttributes::Full);
732         break;
733       case LoopHintAttr::Vectorize:
734       case LoopHintAttr::Interleave:
735       case LoopHintAttr::UnrollCount:
736       case LoopHintAttr::UnrollAndJamCount:
737       case LoopHintAttr::VectorizeWidth:
738       case LoopHintAttr::InterleaveCount:
739       case LoopHintAttr::Distribute:
740       case LoopHintAttr::PipelineDisabled:
741       case LoopHintAttr::PipelineInitiationInterval:
742       case LoopHintAttr::VectorizePredicate:
743         llvm_unreachable("Options cannot be used with 'full' hint.");
744         break;
745       }
746       break;
747     case LoopHintAttr::FixedWidth:
748     case LoopHintAttr::ScalableWidth:
749       switch (Option) {
750       case LoopHintAttr::VectorizeWidth:
751         setVectorizeScalable(State == LoopHintAttr::ScalableWidth
752                                  ? LoopAttributes::Enable
753                                  : LoopAttributes::Disable);
754         if (LH->getValue())
755           setVectorizeWidth(ValueInt);
756         break;
757       default:
758         llvm_unreachable("Options cannot be used with 'scalable' hint.");
759         break;
760       }
761       break;
762     case LoopHintAttr::Numeric:
763       switch (Option) {
764       case LoopHintAttr::InterleaveCount:
765         setInterleaveCount(ValueInt);
766         break;
767       case LoopHintAttr::UnrollCount:
768         setUnrollCount(ValueInt);
769         break;
770       case LoopHintAttr::UnrollAndJamCount:
771         setUnrollAndJamCount(ValueInt);
772         break;
773       case LoopHintAttr::PipelineInitiationInterval:
774         setPipelineInitiationInterval(ValueInt);
775         break;
776       case LoopHintAttr::Unroll:
777       case LoopHintAttr::UnrollAndJam:
778       case LoopHintAttr::VectorizePredicate:
779       case LoopHintAttr::Vectorize:
780       case LoopHintAttr::VectorizeWidth:
781       case LoopHintAttr::Interleave:
782       case LoopHintAttr::Distribute:
783       case LoopHintAttr::PipelineDisabled:
784         llvm_unreachable("Options cannot be assigned a value.");
785         break;
786       }
787       break;
788     }
789   }
790 
791   setMustProgress(MustProgress);
792 
793   if (CGOpts.OptimizationLevel > 0)
794     // Disable unrolling for the loop, if unrolling is disabled (via
795     // -fno-unroll-loops) and no pragmas override the decision.
796     if (!CGOpts.UnrollLoops &&
797         (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
798          StagedAttrs.UnrollCount == 0))
799       setUnrollState(LoopAttributes::Disable);
800 
801   /// Stage the attributes.
802   push(Header, StartLoc, EndLoc);
803 }
804 
805 void LoopInfoStack::pop() {
806   assert(!Active.empty() && "No active loops to pop");
807   Active.back()->finish();
808   Active.pop_back();
809 }
810 
811 void LoopInfoStack::InsertHelper(Instruction *I) const {
812   if (I->mayReadOrWriteMemory()) {
813     SmallVector<Metadata *, 4> AccessGroups;
814     for (const auto &AL : Active) {
815       // Here we assume that every loop that has an access group is parallel.
816       if (MDNode *Group = AL->getAccessGroup())
817         AccessGroups.push_back(Group);
818     }
819     MDNode *UnionMD = nullptr;
820     if (AccessGroups.size() == 1)
821       UnionMD = cast<MDNode>(AccessGroups[0]);
822     else if (AccessGroups.size() >= 2)
823       UnionMD = MDNode::get(I->getContext(), AccessGroups);
824     I->setMetadata("llvm.access.group", UnionMD);
825   }
826 
827   if (!hasInfo())
828     return;
829 
830   const LoopInfo &L = getInfo();
831   if (!L.getLoopID())
832     return;
833 
834   if (I->isTerminator()) {
835     for (BasicBlock *Succ : successors(I))
836       if (Succ == L.getHeader()) {
837         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
838         break;
839       }
840     return;
841   }
842 }
843