xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp (revision cfd6422a5217410fbd66f7a7a8a64d9d85e61229) 
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
13 #include "clang/Basic/CodeGenOptions.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 using namespace clang::CodeGen;
21 using namespace llvm;
22 
23 MDNode *
24 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25   LLVMContext &Ctx = Header->getContext();
26   SmallVector<Metadata *, 4> NewLoopProperties;
27   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
28   NewLoopProperties.push_back(TempNode.get());
29   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
30 
31   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
32   LoopID->replaceOperandWith(0, LoopID);
33   return LoopID;
34 }
35 
36 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37                                            ArrayRef<Metadata *> LoopProperties,
38                                            bool &HasUserTransforms) {
39   LLVMContext &Ctx = Header->getContext();
40 
41   Optional<bool> Enabled;
42   if (Attrs.PipelineDisabled)
43     Enabled = false;
44   else if (Attrs.PipelineInitiationInterval != 0)
45     Enabled = true;
46 
47   if (Enabled != true) {
48     SmallVector<Metadata *, 4> NewLoopProperties;
49     if (Enabled == false) {
50       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
51       NewLoopProperties.push_back(
52           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
53                             ConstantAsMetadata::get(ConstantInt::get(
54                                 llvm::Type::getInt1Ty(Ctx), 1))}));
55       LoopProperties = NewLoopProperties;
56     }
57     return createLoopPropertiesMetadata(LoopProperties);
58   }
59 
60   SmallVector<Metadata *, 4> Args;
61   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
62   Args.push_back(TempNode.get());
63   Args.append(LoopProperties.begin(), LoopProperties.end());
64 
65   if (Attrs.PipelineInitiationInterval > 0) {
66     Metadata *Vals[] = {
67         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
68         ConstantAsMetadata::get(ConstantInt::get(
69             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
70     Args.push_back(MDNode::get(Ctx, Vals));
71   }
72 
73   // No follow-up: This is the last transformation.
74 
75   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
76   LoopID->replaceOperandWith(0, LoopID);
77   HasUserTransforms = true;
78   return LoopID;
79 }
80 
81 MDNode *
82 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
83                                       ArrayRef<Metadata *> LoopProperties,
84                                       bool &HasUserTransforms) {
85   LLVMContext &Ctx = Header->getContext();
86 
87   Optional<bool> Enabled;
88   if (Attrs.UnrollEnable == LoopAttributes::Disable)
89     Enabled = false;
90   else if (Attrs.UnrollEnable == LoopAttributes::Full)
91     Enabled = None;
92   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
93            Attrs.UnrollCount != 0)
94     Enabled = true;
95 
96   if (Enabled != true) {
97     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
98     // if unrolling is disabled.
99     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
100   }
101 
102   SmallVector<Metadata *, 4> FollowupLoopProperties;
103 
104   // Apply all loop properties to the unrolled loop.
105   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
106 
107   // Don't unroll an already unrolled loop.
108   FollowupLoopProperties.push_back(
109       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
110 
111   bool FollowupHasTransforms = false;
112   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
113                                               FollowupHasTransforms);
114 
115   SmallVector<Metadata *, 4> Args;
116   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
117   Args.push_back(TempNode.get());
118   Args.append(LoopProperties.begin(), LoopProperties.end());
119 
120   // Setting unroll.count
121   if (Attrs.UnrollCount > 0) {
122     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
123                         ConstantAsMetadata::get(ConstantInt::get(
124                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
125     Args.push_back(MDNode::get(Ctx, Vals));
126   }
127 
128   // Setting unroll.full or unroll.disable
129   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
130     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
131     Args.push_back(MDNode::get(Ctx, Vals));
132   }
133 
134   if (FollowupHasTransforms)
135     Args.push_back(MDNode::get(
136         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
137 
138   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
139   LoopID->replaceOperandWith(0, LoopID);
140   HasUserTransforms = true;
141   return LoopID;
142 }
143 
144 MDNode *
145 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
146                                      ArrayRef<Metadata *> LoopProperties,
147                                      bool &HasUserTransforms) {
148   LLVMContext &Ctx = Header->getContext();
149 
150   Optional<bool> Enabled;
151   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
152     Enabled = false;
153   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
154            Attrs.UnrollAndJamCount != 0)
155     Enabled = true;
156 
157   if (Enabled != true) {
158     SmallVector<Metadata *, 4> NewLoopProperties;
159     if (Enabled == false) {
160       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
161       NewLoopProperties.push_back(MDNode::get(
162           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
163       LoopProperties = NewLoopProperties;
164     }
165     return createPartialUnrollMetadata(Attrs, LoopProperties,
166                                        HasUserTransforms);
167   }
168 
169   SmallVector<Metadata *, 4> FollowupLoopProperties;
170   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
171   FollowupLoopProperties.push_back(
172       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
173 
174   bool FollowupHasTransforms = false;
175   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
176                                                  FollowupHasTransforms);
177 
178   SmallVector<Metadata *, 4> Args;
179   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
180   Args.push_back(TempNode.get());
181   Args.append(LoopProperties.begin(), LoopProperties.end());
182 
183   // Setting unroll_and_jam.count
184   if (Attrs.UnrollAndJamCount > 0) {
185     Metadata *Vals[] = {
186         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
187         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
188                                                  Attrs.UnrollAndJamCount))};
189     Args.push_back(MDNode::get(Ctx, Vals));
190   }
191 
192   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
193     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
194     Args.push_back(MDNode::get(Ctx, Vals));
195   }
196 
197   if (FollowupHasTransforms)
198     Args.push_back(MDNode::get(
199         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
200               Followup}));
201 
202   if (UnrollAndJamInnerFollowup)
203     Args.push_back(MDNode::get(
204         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
205               UnrollAndJamInnerFollowup}));
206 
207   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
208   LoopID->replaceOperandWith(0, LoopID);
209   HasUserTransforms = true;
210   return LoopID;
211 }
212 
213 MDNode *
214 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
215                                       ArrayRef<Metadata *> LoopProperties,
216                                       bool &HasUserTransforms) {
217   LLVMContext &Ctx = Header->getContext();
218 
219   Optional<bool> Enabled;
220   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
221     Enabled = false;
222   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
223            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
224            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
225     Enabled = true;
226 
227   if (Enabled != true) {
228     SmallVector<Metadata *, 4> NewLoopProperties;
229     if (Enabled == false) {
230       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
231       NewLoopProperties.push_back(
232           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
233                             ConstantAsMetadata::get(ConstantInt::get(
234                                 llvm::Type::getInt1Ty(Ctx), 0))}));
235       LoopProperties = NewLoopProperties;
236     }
237     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
238   }
239 
240   // Apply all loop properties to the vectorized loop.
241   SmallVector<Metadata *, 4> FollowupLoopProperties;
242   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
243 
244   // Don't vectorize an already vectorized loop.
245   FollowupLoopProperties.push_back(
246       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
247 
248   bool FollowupHasTransforms = false;
249   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
250                                                 FollowupHasTransforms);
251 
252   SmallVector<Metadata *, 4> Args;
253   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
254   Args.push_back(TempNode.get());
255   Args.append(LoopProperties.begin(), LoopProperties.end());
256 
257   // Setting vectorize.predicate
258   bool IsVectorPredicateEnabled = false;
259   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
260       Attrs.VectorizeEnable != LoopAttributes::Disable &&
261       Attrs.VectorizeWidth < 1) {
262 
263     IsVectorPredicateEnabled =
264         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
265 
266     Metadata *Vals[] = {
267         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
268         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
269                                                  IsVectorPredicateEnabled))};
270     Args.push_back(MDNode::get(Ctx, Vals));
271   }
272 
273   // Setting vectorize.width
274   if (Attrs.VectorizeWidth > 0) {
275     Metadata *Vals[] = {
276         MDString::get(Ctx, "llvm.loop.vectorize.width"),
277         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
278                                                  Attrs.VectorizeWidth))};
279     Args.push_back(MDNode::get(Ctx, Vals));
280   }
281 
282   // Setting interleave.count
283   if (Attrs.InterleaveCount > 0) {
284     Metadata *Vals[] = {
285         MDString::get(Ctx, "llvm.loop.interleave.count"),
286         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
287                                                  Attrs.InterleaveCount))};
288     Args.push_back(MDNode::get(Ctx, Vals));
289   }
290 
291   // vectorize.enable is set if:
292   // 1) loop hint vectorize.enable is set, or
293   // 2) it is implied when vectorize.predicate is set, or
294   // 3) it is implied when vectorize.width is set.
295   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
296       IsVectorPredicateEnabled ||
297       Attrs.VectorizeWidth > 1 ) {
298     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
299     Args.push_back(
300         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
301                           ConstantAsMetadata::get(ConstantInt::get(
302                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
303   }
304 
305   if (FollowupHasTransforms)
306     Args.push_back(MDNode::get(
307         Ctx,
308         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
309 
310   MDNode *LoopID = MDNode::get(Ctx, Args);
311   LoopID->replaceOperandWith(0, LoopID);
312   HasUserTransforms = true;
313   return LoopID;
314 }
315 
316 MDNode *
317 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
318                                        ArrayRef<Metadata *> LoopProperties,
319                                        bool &HasUserTransforms) {
320   LLVMContext &Ctx = Header->getContext();
321 
322   Optional<bool> Enabled;
323   if (Attrs.DistributeEnable == LoopAttributes::Disable)
324     Enabled = false;
325   if (Attrs.DistributeEnable == LoopAttributes::Enable)
326     Enabled = true;
327 
328   if (Enabled != true) {
329     SmallVector<Metadata *, 4> NewLoopProperties;
330     if (Enabled == false) {
331       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
332       NewLoopProperties.push_back(
333           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
334                             ConstantAsMetadata::get(ConstantInt::get(
335                                 llvm::Type::getInt1Ty(Ctx), 0))}));
336       LoopProperties = NewLoopProperties;
337     }
338     return createLoopVectorizeMetadata(Attrs, LoopProperties,
339                                        HasUserTransforms);
340   }
341 
342   bool FollowupHasTransforms = false;
343   MDNode *Followup =
344       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
345 
346   SmallVector<Metadata *, 4> Args;
347   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
348   Args.push_back(TempNode.get());
349   Args.append(LoopProperties.begin(), LoopProperties.end());
350 
351   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
352                       ConstantAsMetadata::get(ConstantInt::get(
353                           llvm::Type::getInt1Ty(Ctx),
354                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
355   Args.push_back(MDNode::get(Ctx, Vals));
356 
357   if (FollowupHasTransforms)
358     Args.push_back(MDNode::get(
359         Ctx,
360         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
361 
362   MDNode *LoopID = MDNode::get(Ctx, Args);
363   LoopID->replaceOperandWith(0, LoopID);
364   HasUserTransforms = true;
365   return LoopID;
366 }
367 
368 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
369                                            ArrayRef<Metadata *> LoopProperties,
370                                            bool &HasUserTransforms) {
371   LLVMContext &Ctx = Header->getContext();
372 
373   Optional<bool> Enabled;
374   if (Attrs.UnrollEnable == LoopAttributes::Disable)
375     Enabled = false;
376   else if (Attrs.UnrollEnable == LoopAttributes::Full)
377     Enabled = true;
378 
379   if (Enabled != true) {
380     SmallVector<Metadata *, 4> NewLoopProperties;
381     if (Enabled == false) {
382       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
383       NewLoopProperties.push_back(
384           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
385       LoopProperties = NewLoopProperties;
386     }
387     return createLoopDistributeMetadata(Attrs, LoopProperties,
388                                         HasUserTransforms);
389   }
390 
391   SmallVector<Metadata *, 4> Args;
392   TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
393   Args.push_back(TempNode.get());
394   Args.append(LoopProperties.begin(), LoopProperties.end());
395   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
396 
397   // No follow-up: there is no loop after full unrolling.
398   // TODO: Warn if there are transformations after full unrolling.
399 
400   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
401   LoopID->replaceOperandWith(0, LoopID);
402   HasUserTransforms = true;
403   return LoopID;
404 }
405 
406 MDNode *LoopInfo::createMetadata(
407     const LoopAttributes &Attrs,
408     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
409     bool &HasUserTransforms) {
410   SmallVector<Metadata *, 3> LoopProperties;
411 
412   // If we have a valid start debug location for the loop, add it.
413   if (StartLoc) {
414     LoopProperties.push_back(StartLoc.getAsMDNode());
415 
416     // If we also have a valid end debug location for the loop, add it.
417     if (EndLoc)
418       LoopProperties.push_back(EndLoc.getAsMDNode());
419   }
420 
421   assert(!!AccGroup == Attrs.IsParallel &&
422          "There must be an access group iff the loop is parallel");
423   if (Attrs.IsParallel) {
424     LLVMContext &Ctx = Header->getContext();
425     LoopProperties.push_back(MDNode::get(
426         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
427   }
428 
429   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
430                         AdditionalLoopProperties.end());
431   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
432 }
433 
434 LoopAttributes::LoopAttributes(bool IsParallel)
435     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
436       UnrollEnable(LoopAttributes::Unspecified),
437       UnrollAndJamEnable(LoopAttributes::Unspecified),
438       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
439       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
440       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
441       PipelineInitiationInterval(0) {}
442 
443 void LoopAttributes::clear() {
444   IsParallel = false;
445   VectorizeWidth = 0;
446   InterleaveCount = 0;
447   UnrollCount = 0;
448   UnrollAndJamCount = 0;
449   VectorizeEnable = LoopAttributes::Unspecified;
450   UnrollEnable = LoopAttributes::Unspecified;
451   UnrollAndJamEnable = LoopAttributes::Unspecified;
452   VectorizePredicateEnable = LoopAttributes::Unspecified;
453   DistributeEnable = LoopAttributes::Unspecified;
454   PipelineDisabled = false;
455   PipelineInitiationInterval = 0;
456 }
457 
458 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
459                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
460                    LoopInfo *Parent)
461     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
462       Parent(Parent) {
463 
464   if (Attrs.IsParallel) {
465     // Create an access group for this loop.
466     LLVMContext &Ctx = Header->getContext();
467     AccGroup = MDNode::getDistinct(Ctx, {});
468   }
469 
470   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
471       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
472       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
473       Attrs.PipelineInitiationInterval == 0 &&
474       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
475       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
476       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
477       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
478       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
479       !EndLoc)
480     return;
481 
482   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
483 }
484 
485 void LoopInfo::finish() {
486   // We did not annotate the loop body instructions because there are no
487   // attributes for this loop.
488   if (!TempLoopID)
489     return;
490 
491   MDNode *LoopID;
492   LoopAttributes CurLoopAttr = Attrs;
493   LLVMContext &Ctx = Header->getContext();
494 
495   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
496                  Parent->Attrs.UnrollAndJamCount != 0)) {
497     // Parent unroll-and-jams this loop.
498     // Split the transformations in those that happens before the unroll-and-jam
499     // and those after.
500 
501     LoopAttributes BeforeJam, AfterJam;
502 
503     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
504 
505     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
506     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
507     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
508     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
509     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
510 
511     switch (Attrs.UnrollEnable) {
512     case LoopAttributes::Unspecified:
513     case LoopAttributes::Disable:
514       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
515       AfterJam.UnrollEnable = Attrs.UnrollEnable;
516       break;
517     case LoopAttributes::Full:
518       BeforeJam.UnrollEnable = LoopAttributes::Full;
519       break;
520     case LoopAttributes::Enable:
521       AfterJam.UnrollEnable = LoopAttributes::Enable;
522       break;
523     }
524 
525     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
526     AfterJam.UnrollCount = Attrs.UnrollCount;
527     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
528     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
529 
530     // If this loop is subject of an unroll-and-jam by the parent loop, and has
531     // an unroll-and-jam annotation itself, we have to decide whether to first
532     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
533     // UnrollAndJam pass processes loops from inner to outer, so we apply the
534     // inner first.
535     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
536     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
537 
538     // Set the inner followup metadata to process by the outer loop. Only
539     // consider the first inner loop.
540     if (!Parent->UnrollAndJamInnerFollowup) {
541       // Splitting the attributes into a BeforeJam and an AfterJam part will
542       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
543       // to be forwarded to the AfterJam part. We detect the situation here and
544       // add it manually.
545       SmallVector<Metadata *, 1> BeforeLoopProperties;
546       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
547           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
548           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
549         BeforeLoopProperties.push_back(
550             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
551 
552       bool InnerFollowupHasTransform = false;
553       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
554                                              InnerFollowupHasTransform);
555       if (InnerFollowupHasTransform)
556         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
557     }
558 
559     CurLoopAttr = BeforeJam;
560   }
561 
562   bool HasUserTransforms = false;
563   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
564   TempLoopID->replaceAllUsesWith(LoopID);
565 }
566 
567 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
568                          const llvm::DebugLoc &EndLoc) {
569   Active.emplace_back(
570       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
571                    Active.empty() ? nullptr : Active.back().get()));
572   // Clear the attributes so nested loops do not inherit them.
573   StagedAttrs.clear();
574 }
575 
576 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
577                          const clang::CodeGenOptions &CGOpts,
578                          ArrayRef<const clang::Attr *> Attrs,
579                          const llvm::DebugLoc &StartLoc,
580                          const llvm::DebugLoc &EndLoc) {
581 
582   // Identify loop hint attributes from Attrs.
583   for (const auto *Attr : Attrs) {
584     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
585     const OpenCLUnrollHintAttr *OpenCLHint =
586         dyn_cast<OpenCLUnrollHintAttr>(Attr);
587 
588     // Skip non loop hint attributes
589     if (!LH && !OpenCLHint) {
590       continue;
591     }
592 
593     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
594     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
595     unsigned ValueInt = 1;
596     // Translate opencl_unroll_hint attribute argument to
597     // equivalent LoopHintAttr enums.
598     // OpenCL v2.0 s6.11.5:
599     // 0 - enable unroll (no argument).
600     // 1 - disable unroll.
601     // other positive integer n - unroll by n.
602     if (OpenCLHint) {
603       ValueInt = OpenCLHint->getUnrollHint();
604       if (ValueInt == 0) {
605         State = LoopHintAttr::Enable;
606       } else if (ValueInt != 1) {
607         Option = LoopHintAttr::UnrollCount;
608         State = LoopHintAttr::Numeric;
609       }
610     } else if (LH) {
611       auto *ValueExpr = LH->getValue();
612       if (ValueExpr) {
613         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
614         ValueInt = ValueAPS.getSExtValue();
615       }
616 
617       Option = LH->getOption();
618       State = LH->getState();
619     }
620     switch (State) {
621     case LoopHintAttr::Disable:
622       switch (Option) {
623       case LoopHintAttr::Vectorize:
624         // Disable vectorization by specifying a width of 1.
625         setVectorizeWidth(1);
626         break;
627       case LoopHintAttr::Interleave:
628         // Disable interleaving by speciyfing a count of 1.
629         setInterleaveCount(1);
630         break;
631       case LoopHintAttr::Unroll:
632         setUnrollState(LoopAttributes::Disable);
633         break;
634       case LoopHintAttr::UnrollAndJam:
635         setUnrollAndJamState(LoopAttributes::Disable);
636         break;
637       case LoopHintAttr::VectorizePredicate:
638         setVectorizePredicateState(LoopAttributes::Disable);
639         break;
640       case LoopHintAttr::Distribute:
641         setDistributeState(false);
642         break;
643       case LoopHintAttr::PipelineDisabled:
644         setPipelineDisabled(true);
645         break;
646       case LoopHintAttr::UnrollCount:
647       case LoopHintAttr::UnrollAndJamCount:
648       case LoopHintAttr::VectorizeWidth:
649       case LoopHintAttr::InterleaveCount:
650       case LoopHintAttr::PipelineInitiationInterval:
651         llvm_unreachable("Options cannot be disabled.");
652         break;
653       }
654       break;
655     case LoopHintAttr::Enable:
656       switch (Option) {
657       case LoopHintAttr::Vectorize:
658       case LoopHintAttr::Interleave:
659         setVectorizeEnable(true);
660         break;
661       case LoopHintAttr::Unroll:
662         setUnrollState(LoopAttributes::Enable);
663         break;
664       case LoopHintAttr::UnrollAndJam:
665         setUnrollAndJamState(LoopAttributes::Enable);
666         break;
667       case LoopHintAttr::VectorizePredicate:
668         setVectorizePredicateState(LoopAttributes::Enable);
669         break;
670       case LoopHintAttr::Distribute:
671         setDistributeState(true);
672         break;
673       case LoopHintAttr::UnrollCount:
674       case LoopHintAttr::UnrollAndJamCount:
675       case LoopHintAttr::VectorizeWidth:
676       case LoopHintAttr::InterleaveCount:
677       case LoopHintAttr::PipelineDisabled:
678       case LoopHintAttr::PipelineInitiationInterval:
679         llvm_unreachable("Options cannot enabled.");
680         break;
681       }
682       break;
683     case LoopHintAttr::AssumeSafety:
684       switch (Option) {
685       case LoopHintAttr::Vectorize:
686       case LoopHintAttr::Interleave:
687         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
688         setParallel(true);
689         setVectorizeEnable(true);
690         break;
691       case LoopHintAttr::Unroll:
692       case LoopHintAttr::UnrollAndJam:
693       case LoopHintAttr::VectorizePredicate:
694       case LoopHintAttr::UnrollCount:
695       case LoopHintAttr::UnrollAndJamCount:
696       case LoopHintAttr::VectorizeWidth:
697       case LoopHintAttr::InterleaveCount:
698       case LoopHintAttr::Distribute:
699       case LoopHintAttr::PipelineDisabled:
700       case LoopHintAttr::PipelineInitiationInterval:
701         llvm_unreachable("Options cannot be used to assume mem safety.");
702         break;
703       }
704       break;
705     case LoopHintAttr::Full:
706       switch (Option) {
707       case LoopHintAttr::Unroll:
708         setUnrollState(LoopAttributes::Full);
709         break;
710       case LoopHintAttr::UnrollAndJam:
711         setUnrollAndJamState(LoopAttributes::Full);
712         break;
713       case LoopHintAttr::Vectorize:
714       case LoopHintAttr::Interleave:
715       case LoopHintAttr::UnrollCount:
716       case LoopHintAttr::UnrollAndJamCount:
717       case LoopHintAttr::VectorizeWidth:
718       case LoopHintAttr::InterleaveCount:
719       case LoopHintAttr::Distribute:
720       case LoopHintAttr::PipelineDisabled:
721       case LoopHintAttr::PipelineInitiationInterval:
722       case LoopHintAttr::VectorizePredicate:
723         llvm_unreachable("Options cannot be used with 'full' hint.");
724         break;
725       }
726       break;
727     case LoopHintAttr::Numeric:
728       switch (Option) {
729       case LoopHintAttr::VectorizeWidth:
730         setVectorizeWidth(ValueInt);
731         break;
732       case LoopHintAttr::InterleaveCount:
733         setInterleaveCount(ValueInt);
734         break;
735       case LoopHintAttr::UnrollCount:
736         setUnrollCount(ValueInt);
737         break;
738       case LoopHintAttr::UnrollAndJamCount:
739         setUnrollAndJamCount(ValueInt);
740         break;
741       case LoopHintAttr::PipelineInitiationInterval:
742         setPipelineInitiationInterval(ValueInt);
743         break;
744       case LoopHintAttr::Unroll:
745       case LoopHintAttr::UnrollAndJam:
746       case LoopHintAttr::VectorizePredicate:
747       case LoopHintAttr::Vectorize:
748       case LoopHintAttr::Interleave:
749       case LoopHintAttr::Distribute:
750       case LoopHintAttr::PipelineDisabled:
751         llvm_unreachable("Options cannot be assigned a value.");
752         break;
753       }
754       break;
755     }
756   }
757 
758   if (CGOpts.OptimizationLevel > 0)
759     // Disable unrolling for the loop, if unrolling is disabled (via
760     // -fno-unroll-loops) and no pragmas override the decision.
761     if (!CGOpts.UnrollLoops &&
762         (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
763          StagedAttrs.UnrollCount == 0))
764       setUnrollState(LoopAttributes::Disable);
765 
766   /// Stage the attributes.
767   push(Header, StartLoc, EndLoc);
768 }
769 
770 void LoopInfoStack::pop() {
771   assert(!Active.empty() && "No active loops to pop");
772   Active.back()->finish();
773   Active.pop_back();
774 }
775 
776 void LoopInfoStack::InsertHelper(Instruction *I) const {
777   if (I->mayReadOrWriteMemory()) {
778     SmallVector<Metadata *, 4> AccessGroups;
779     for (const auto &AL : Active) {
780       // Here we assume that every loop that has an access group is parallel.
781       if (MDNode *Group = AL->getAccessGroup())
782         AccessGroups.push_back(Group);
783     }
784     MDNode *UnionMD = nullptr;
785     if (AccessGroups.size() == 1)
786       UnionMD = cast<MDNode>(AccessGroups[0]);
787     else if (AccessGroups.size() >= 2)
788       UnionMD = MDNode::get(I->getContext(), AccessGroups);
789     I->setMetadata("llvm.access.group", UnionMD);
790   }
791 
792   if (!hasInfo())
793     return;
794 
795   const LoopInfo &L = getInfo();
796   if (!L.getLoopID())
797     return;
798 
799   if (I->isTerminator()) {
800     for (BasicBlock *Succ : successors(I))
801       if (Succ == L.getHeader()) {
802         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
803         break;
804       }
805     return;
806   }
807 }
808