xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp (revision 924226fba12cc9a228c73b956e1b7fa24c60b055) 
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
13 #include "clang/Basic/CodeGenOptions.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 using namespace clang::CodeGen;
21 using namespace llvm;
22 
23 MDNode *
24 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25   LLVMContext &Ctx = Header->getContext();
26   SmallVector<Metadata *, 4> NewLoopProperties;
27   NewLoopProperties.push_back(nullptr);
28   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29 
30   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
31   LoopID->replaceOperandWith(0, LoopID);
32   return LoopID;
33 }
34 
35 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
36                                            ArrayRef<Metadata *> LoopProperties,
37                                            bool &HasUserTransforms) {
38   LLVMContext &Ctx = Header->getContext();
39 
40   Optional<bool> Enabled;
41   if (Attrs.PipelineDisabled)
42     Enabled = false;
43   else if (Attrs.PipelineInitiationInterval != 0)
44     Enabled = true;
45 
46   if (Enabled != true) {
47     SmallVector<Metadata *, 4> NewLoopProperties;
48     if (Enabled == false) {
49       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
50       NewLoopProperties.push_back(
51           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
52                             ConstantAsMetadata::get(ConstantInt::get(
53                                 llvm::Type::getInt1Ty(Ctx), 1))}));
54       LoopProperties = NewLoopProperties;
55     }
56     return createLoopPropertiesMetadata(LoopProperties);
57   }
58 
59   SmallVector<Metadata *, 4> Args;
60   Args.push_back(nullptr);
61   Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63   if (Attrs.PipelineInitiationInterval > 0) {
64     Metadata *Vals[] = {
65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66         ConstantAsMetadata::get(ConstantInt::get(
67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68     Args.push_back(MDNode::get(Ctx, Vals));
69   }
70 
71   // No follow-up: This is the last transformation.
72 
73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74   LoopID->replaceOperandWith(0, LoopID);
75   HasUserTransforms = true;
76   return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81                                       ArrayRef<Metadata *> LoopProperties,
82                                       bool &HasUserTransforms) {
83   LLVMContext &Ctx = Header->getContext();
84 
85   Optional<bool> Enabled;
86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
87     Enabled = false;
88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
89     Enabled = None;
90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91            Attrs.UnrollCount != 0)
92     Enabled = true;
93 
94   if (Enabled != true) {
95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96     // if unrolling is disabled.
97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98   }
99 
100   SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102   // Apply all loop properties to the unrolled loop.
103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105   // Don't unroll an already unrolled loop.
106   FollowupLoopProperties.push_back(
107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109   bool FollowupHasTransforms = false;
110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111                                               FollowupHasTransforms);
112 
113   SmallVector<Metadata *, 4> Args;
114   Args.push_back(nullptr);
115   Args.append(LoopProperties.begin(), LoopProperties.end());
116 
117   // Setting unroll.count
118   if (Attrs.UnrollCount > 0) {
119     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
120                         ConstantAsMetadata::get(ConstantInt::get(
121                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
122     Args.push_back(MDNode::get(Ctx, Vals));
123   }
124 
125   // Setting unroll.full or unroll.disable
126   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
127     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
128     Args.push_back(MDNode::get(Ctx, Vals));
129   }
130 
131   if (FollowupHasTransforms)
132     Args.push_back(MDNode::get(
133         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
134 
135   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
136   LoopID->replaceOperandWith(0, LoopID);
137   HasUserTransforms = true;
138   return LoopID;
139 }
140 
141 MDNode *
142 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
143                                      ArrayRef<Metadata *> LoopProperties,
144                                      bool &HasUserTransforms) {
145   LLVMContext &Ctx = Header->getContext();
146 
147   Optional<bool> Enabled;
148   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
149     Enabled = false;
150   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
151            Attrs.UnrollAndJamCount != 0)
152     Enabled = true;
153 
154   if (Enabled != true) {
155     SmallVector<Metadata *, 4> NewLoopProperties;
156     if (Enabled == false) {
157       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
158       NewLoopProperties.push_back(MDNode::get(
159           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
160       LoopProperties = NewLoopProperties;
161     }
162     return createPartialUnrollMetadata(Attrs, LoopProperties,
163                                        HasUserTransforms);
164   }
165 
166   SmallVector<Metadata *, 4> FollowupLoopProperties;
167   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
168   FollowupLoopProperties.push_back(
169       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
170 
171   bool FollowupHasTransforms = false;
172   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
173                                                  FollowupHasTransforms);
174 
175   SmallVector<Metadata *, 4> Args;
176   Args.push_back(nullptr);
177   Args.append(LoopProperties.begin(), LoopProperties.end());
178 
179   // Setting unroll_and_jam.count
180   if (Attrs.UnrollAndJamCount > 0) {
181     Metadata *Vals[] = {
182         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
183         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
184                                                  Attrs.UnrollAndJamCount))};
185     Args.push_back(MDNode::get(Ctx, Vals));
186   }
187 
188   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
189     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
190     Args.push_back(MDNode::get(Ctx, Vals));
191   }
192 
193   if (FollowupHasTransforms)
194     Args.push_back(MDNode::get(
195         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
196               Followup}));
197 
198   if (UnrollAndJamInnerFollowup)
199     Args.push_back(MDNode::get(
200         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
201               UnrollAndJamInnerFollowup}));
202 
203   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
204   LoopID->replaceOperandWith(0, LoopID);
205   HasUserTransforms = true;
206   return LoopID;
207 }
208 
209 MDNode *
210 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
211                                       ArrayRef<Metadata *> LoopProperties,
212                                       bool &HasUserTransforms) {
213   LLVMContext &Ctx = Header->getContext();
214 
215   Optional<bool> Enabled;
216   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
217     Enabled = false;
218   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
219            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
220            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
221            Attrs.VectorizeScalable != LoopAttributes::Unspecified)
222     Enabled = true;
223 
224   if (Enabled != true) {
225     SmallVector<Metadata *, 4> NewLoopProperties;
226     if (Enabled == false) {
227       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
228       NewLoopProperties.push_back(
229           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
230                             ConstantAsMetadata::get(ConstantInt::get(
231                                 llvm::Type::getInt1Ty(Ctx), 0))}));
232       LoopProperties = NewLoopProperties;
233     }
234     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
235   }
236 
237   // Apply all loop properties to the vectorized loop.
238   SmallVector<Metadata *, 4> FollowupLoopProperties;
239   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
240 
241   // Don't vectorize an already vectorized loop.
242   FollowupLoopProperties.push_back(
243       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
244 
245   bool FollowupHasTransforms = false;
246   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
247                                                 FollowupHasTransforms);
248 
249   SmallVector<Metadata *, 4> Args;
250   Args.push_back(nullptr);
251   Args.append(LoopProperties.begin(), LoopProperties.end());
252 
253   // Setting vectorize.predicate when it has been specified and vectorization
254   // has not been disabled.
255   bool IsVectorPredicateEnabled = false;
256   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
257     IsVectorPredicateEnabled =
258         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
259 
260     Metadata *Vals[] = {
261         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
262         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
263                                                  IsVectorPredicateEnabled))};
264     Args.push_back(MDNode::get(Ctx, Vals));
265   }
266 
267   // Setting vectorize.width
268   if (Attrs.VectorizeWidth > 0) {
269     Metadata *Vals[] = {
270         MDString::get(Ctx, "llvm.loop.vectorize.width"),
271         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
272                                                  Attrs.VectorizeWidth))};
273 
274     Args.push_back(MDNode::get(Ctx, Vals));
275   }
276 
277   if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
278     bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
279     Metadata *Vals[] = {
280         MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
281         ConstantAsMetadata::get(
282             ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
283     Args.push_back(MDNode::get(Ctx, Vals));
284   }
285 
286   // Setting interleave.count
287   if (Attrs.InterleaveCount > 0) {
288     Metadata *Vals[] = {
289         MDString::get(Ctx, "llvm.loop.interleave.count"),
290         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
291                                                  Attrs.InterleaveCount))};
292     Args.push_back(MDNode::get(Ctx, Vals));
293   }
294 
295   // vectorize.enable is set if:
296   // 1) loop hint vectorize.enable is set, or
297   // 2) it is implied when vectorize.predicate is set, or
298   // 3) it is implied when vectorize.width is set to a value > 1
299   // 4) it is implied when vectorize.scalable.enable is true
300   // 5) it is implied when vectorize.width is unset (0) and the user
301   //    explicitly requested fixed-width vectorization, i.e.
302   //    vectorize.scalable.enable is false.
303   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
304       (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
305       Attrs.VectorizeWidth > 1 ||
306       Attrs.VectorizeScalable == LoopAttributes::Enable ||
307       (Attrs.VectorizeScalable == LoopAttributes::Disable &&
308        Attrs.VectorizeWidth != 1)) {
309     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
310     Args.push_back(
311         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
312                           ConstantAsMetadata::get(ConstantInt::get(
313                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
314   }
315 
316   if (FollowupHasTransforms)
317     Args.push_back(MDNode::get(
318         Ctx,
319         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
320 
321   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
322   LoopID->replaceOperandWith(0, LoopID);
323   HasUserTransforms = true;
324   return LoopID;
325 }
326 
327 MDNode *
328 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
329                                        ArrayRef<Metadata *> LoopProperties,
330                                        bool &HasUserTransforms) {
331   LLVMContext &Ctx = Header->getContext();
332 
333   Optional<bool> Enabled;
334   if (Attrs.DistributeEnable == LoopAttributes::Disable)
335     Enabled = false;
336   if (Attrs.DistributeEnable == LoopAttributes::Enable)
337     Enabled = true;
338 
339   if (Enabled != true) {
340     SmallVector<Metadata *, 4> NewLoopProperties;
341     if (Enabled == false) {
342       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
343       NewLoopProperties.push_back(
344           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
345                             ConstantAsMetadata::get(ConstantInt::get(
346                                 llvm::Type::getInt1Ty(Ctx), 0))}));
347       LoopProperties = NewLoopProperties;
348     }
349     return createLoopVectorizeMetadata(Attrs, LoopProperties,
350                                        HasUserTransforms);
351   }
352 
353   bool FollowupHasTransforms = false;
354   MDNode *Followup =
355       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
356 
357   SmallVector<Metadata *, 4> Args;
358   Args.push_back(nullptr);
359   Args.append(LoopProperties.begin(), LoopProperties.end());
360 
361   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
362                       ConstantAsMetadata::get(ConstantInt::get(
363                           llvm::Type::getInt1Ty(Ctx),
364                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
365   Args.push_back(MDNode::get(Ctx, Vals));
366 
367   if (FollowupHasTransforms)
368     Args.push_back(MDNode::get(
369         Ctx,
370         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
371 
372   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
373   LoopID->replaceOperandWith(0, LoopID);
374   HasUserTransforms = true;
375   return LoopID;
376 }
377 
378 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
379                                            ArrayRef<Metadata *> LoopProperties,
380                                            bool &HasUserTransforms) {
381   LLVMContext &Ctx = Header->getContext();
382 
383   Optional<bool> Enabled;
384   if (Attrs.UnrollEnable == LoopAttributes::Disable)
385     Enabled = false;
386   else if (Attrs.UnrollEnable == LoopAttributes::Full)
387     Enabled = true;
388 
389   if (Enabled != true) {
390     SmallVector<Metadata *, 4> NewLoopProperties;
391     if (Enabled == false) {
392       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
393       NewLoopProperties.push_back(
394           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
395       LoopProperties = NewLoopProperties;
396     }
397     return createLoopDistributeMetadata(Attrs, LoopProperties,
398                                         HasUserTransforms);
399   }
400 
401   SmallVector<Metadata *, 4> Args;
402   Args.push_back(nullptr);
403   Args.append(LoopProperties.begin(), LoopProperties.end());
404   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
405 
406   // No follow-up: there is no loop after full unrolling.
407   // TODO: Warn if there are transformations after full unrolling.
408 
409   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
410   LoopID->replaceOperandWith(0, LoopID);
411   HasUserTransforms = true;
412   return LoopID;
413 }
414 
415 MDNode *LoopInfo::createMetadata(
416     const LoopAttributes &Attrs,
417     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
418     bool &HasUserTransforms) {
419   SmallVector<Metadata *, 3> LoopProperties;
420 
421   // If we have a valid start debug location for the loop, add it.
422   if (StartLoc) {
423     LoopProperties.push_back(StartLoc.getAsMDNode());
424 
425     // If we also have a valid end debug location for the loop, add it.
426     if (EndLoc)
427       LoopProperties.push_back(EndLoc.getAsMDNode());
428   }
429 
430   LLVMContext &Ctx = Header->getContext();
431   if (Attrs.MustProgress)
432     LoopProperties.push_back(
433         MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
434 
435   assert(!!AccGroup == Attrs.IsParallel &&
436          "There must be an access group iff the loop is parallel");
437   if (Attrs.IsParallel) {
438     LoopProperties.push_back(MDNode::get(
439         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
440   }
441 
442   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
443                         AdditionalLoopProperties.end());
444   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
445 }
446 
447 LoopAttributes::LoopAttributes(bool IsParallel)
448     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
449       UnrollEnable(LoopAttributes::Unspecified),
450       UnrollAndJamEnable(LoopAttributes::Unspecified),
451       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
452       VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
453       UnrollCount(0), UnrollAndJamCount(0),
454       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
455       PipelineInitiationInterval(0), MustProgress(false) {}
456 
457 void LoopAttributes::clear() {
458   IsParallel = false;
459   VectorizeWidth = 0;
460   VectorizeScalable = LoopAttributes::Unspecified;
461   InterleaveCount = 0;
462   UnrollCount = 0;
463   UnrollAndJamCount = 0;
464   VectorizeEnable = LoopAttributes::Unspecified;
465   UnrollEnable = LoopAttributes::Unspecified;
466   UnrollAndJamEnable = LoopAttributes::Unspecified;
467   VectorizePredicateEnable = LoopAttributes::Unspecified;
468   DistributeEnable = LoopAttributes::Unspecified;
469   PipelineDisabled = false;
470   PipelineInitiationInterval = 0;
471   MustProgress = false;
472 }
473 
474 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
475                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
476                    LoopInfo *Parent)
477     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
478       Parent(Parent) {
479 
480   if (Attrs.IsParallel) {
481     // Create an access group for this loop.
482     LLVMContext &Ctx = Header->getContext();
483     AccGroup = MDNode::getDistinct(Ctx, {});
484   }
485 
486   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
487       Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
488       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
489       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
490       Attrs.PipelineInitiationInterval == 0 &&
491       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
492       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
493       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
494       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
495       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
496       !EndLoc && !Attrs.MustProgress)
497     return;
498 
499   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
500 }
501 
502 void LoopInfo::finish() {
503   // We did not annotate the loop body instructions because there are no
504   // attributes for this loop.
505   if (!TempLoopID)
506     return;
507 
508   MDNode *LoopID;
509   LoopAttributes CurLoopAttr = Attrs;
510   LLVMContext &Ctx = Header->getContext();
511 
512   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
513                  Parent->Attrs.UnrollAndJamCount != 0)) {
514     // Parent unroll-and-jams this loop.
515     // Split the transformations in those that happens before the unroll-and-jam
516     // and those after.
517 
518     LoopAttributes BeforeJam, AfterJam;
519 
520     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
521 
522     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
523     BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
524     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
525     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
526     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
527     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
528 
529     switch (Attrs.UnrollEnable) {
530     case LoopAttributes::Unspecified:
531     case LoopAttributes::Disable:
532       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
533       AfterJam.UnrollEnable = Attrs.UnrollEnable;
534       break;
535     case LoopAttributes::Full:
536       BeforeJam.UnrollEnable = LoopAttributes::Full;
537       break;
538     case LoopAttributes::Enable:
539       AfterJam.UnrollEnable = LoopAttributes::Enable;
540       break;
541     }
542 
543     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
544     AfterJam.UnrollCount = Attrs.UnrollCount;
545     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
546     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
547 
548     // If this loop is subject of an unroll-and-jam by the parent loop, and has
549     // an unroll-and-jam annotation itself, we have to decide whether to first
550     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
551     // UnrollAndJam pass processes loops from inner to outer, so we apply the
552     // inner first.
553     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
554     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
555 
556     // Set the inner followup metadata to process by the outer loop. Only
557     // consider the first inner loop.
558     if (!Parent->UnrollAndJamInnerFollowup) {
559       // Splitting the attributes into a BeforeJam and an AfterJam part will
560       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
561       // to be forwarded to the AfterJam part. We detect the situation here and
562       // add it manually.
563       SmallVector<Metadata *, 1> BeforeLoopProperties;
564       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
565           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
566           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
567           BeforeJam.VectorizeScalable == LoopAttributes::Enable)
568         BeforeLoopProperties.push_back(
569             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
570 
571       bool InnerFollowupHasTransform = false;
572       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
573                                              InnerFollowupHasTransform);
574       if (InnerFollowupHasTransform)
575         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
576     }
577 
578     CurLoopAttr = BeforeJam;
579   }
580 
581   bool HasUserTransforms = false;
582   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
583   TempLoopID->replaceAllUsesWith(LoopID);
584 }
585 
586 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
587                          const llvm::DebugLoc &EndLoc) {
588   Active.emplace_back(
589       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
590                    Active.empty() ? nullptr : Active.back().get()));
591   // Clear the attributes so nested loops do not inherit them.
592   StagedAttrs.clear();
593 }
594 
595 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
596                          const clang::CodeGenOptions &CGOpts,
597                          ArrayRef<const clang::Attr *> Attrs,
598                          const llvm::DebugLoc &StartLoc,
599                          const llvm::DebugLoc &EndLoc, bool MustProgress) {
600   // Identify loop hint attributes from Attrs.
601   for (const auto *Attr : Attrs) {
602     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
603     const OpenCLUnrollHintAttr *OpenCLHint =
604         dyn_cast<OpenCLUnrollHintAttr>(Attr);
605 
606     // Skip non loop hint attributes
607     if (!LH && !OpenCLHint) {
608       continue;
609     }
610 
611     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
612     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
613     unsigned ValueInt = 1;
614     // Translate opencl_unroll_hint attribute argument to
615     // equivalent LoopHintAttr enums.
616     // OpenCL v2.0 s6.11.5:
617     // 0 - enable unroll (no argument).
618     // 1 - disable unroll.
619     // other positive integer n - unroll by n.
620     if (OpenCLHint) {
621       ValueInt = OpenCLHint->getUnrollHint();
622       if (ValueInt == 0) {
623         State = LoopHintAttr::Enable;
624       } else if (ValueInt != 1) {
625         Option = LoopHintAttr::UnrollCount;
626         State = LoopHintAttr::Numeric;
627       }
628     } else if (LH) {
629       auto *ValueExpr = LH->getValue();
630       if (ValueExpr) {
631         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
632         ValueInt = ValueAPS.getSExtValue();
633       }
634 
635       Option = LH->getOption();
636       State = LH->getState();
637     }
638     switch (State) {
639     case LoopHintAttr::Disable:
640       switch (Option) {
641       case LoopHintAttr::Vectorize:
642         // Disable vectorization by specifying a width of 1.
643         setVectorizeWidth(1);
644         setVectorizeScalable(LoopAttributes::Unspecified);
645         break;
646       case LoopHintAttr::Interleave:
647         // Disable interleaving by speciyfing a count of 1.
648         setInterleaveCount(1);
649         break;
650       case LoopHintAttr::Unroll:
651         setUnrollState(LoopAttributes::Disable);
652         break;
653       case LoopHintAttr::UnrollAndJam:
654         setUnrollAndJamState(LoopAttributes::Disable);
655         break;
656       case LoopHintAttr::VectorizePredicate:
657         setVectorizePredicateState(LoopAttributes::Disable);
658         break;
659       case LoopHintAttr::Distribute:
660         setDistributeState(false);
661         break;
662       case LoopHintAttr::PipelineDisabled:
663         setPipelineDisabled(true);
664         break;
665       case LoopHintAttr::UnrollCount:
666       case LoopHintAttr::UnrollAndJamCount:
667       case LoopHintAttr::VectorizeWidth:
668       case LoopHintAttr::InterleaveCount:
669       case LoopHintAttr::PipelineInitiationInterval:
670         llvm_unreachable("Options cannot be disabled.");
671         break;
672       }
673       break;
674     case LoopHintAttr::Enable:
675       switch (Option) {
676       case LoopHintAttr::Vectorize:
677       case LoopHintAttr::Interleave:
678         setVectorizeEnable(true);
679         break;
680       case LoopHintAttr::Unroll:
681         setUnrollState(LoopAttributes::Enable);
682         break;
683       case LoopHintAttr::UnrollAndJam:
684         setUnrollAndJamState(LoopAttributes::Enable);
685         break;
686       case LoopHintAttr::VectorizePredicate:
687         setVectorizePredicateState(LoopAttributes::Enable);
688         break;
689       case LoopHintAttr::Distribute:
690         setDistributeState(true);
691         break;
692       case LoopHintAttr::UnrollCount:
693       case LoopHintAttr::UnrollAndJamCount:
694       case LoopHintAttr::VectorizeWidth:
695       case LoopHintAttr::InterleaveCount:
696       case LoopHintAttr::PipelineDisabled:
697       case LoopHintAttr::PipelineInitiationInterval:
698         llvm_unreachable("Options cannot enabled.");
699         break;
700       }
701       break;
702     case LoopHintAttr::AssumeSafety:
703       switch (Option) {
704       case LoopHintAttr::Vectorize:
705       case LoopHintAttr::Interleave:
706         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
707         setParallel(true);
708         setVectorizeEnable(true);
709         break;
710       case LoopHintAttr::Unroll:
711       case LoopHintAttr::UnrollAndJam:
712       case LoopHintAttr::VectorizePredicate:
713       case LoopHintAttr::UnrollCount:
714       case LoopHintAttr::UnrollAndJamCount:
715       case LoopHintAttr::VectorizeWidth:
716       case LoopHintAttr::InterleaveCount:
717       case LoopHintAttr::Distribute:
718       case LoopHintAttr::PipelineDisabled:
719       case LoopHintAttr::PipelineInitiationInterval:
720         llvm_unreachable("Options cannot be used to assume mem safety.");
721         break;
722       }
723       break;
724     case LoopHintAttr::Full:
725       switch (Option) {
726       case LoopHintAttr::Unroll:
727         setUnrollState(LoopAttributes::Full);
728         break;
729       case LoopHintAttr::UnrollAndJam:
730         setUnrollAndJamState(LoopAttributes::Full);
731         break;
732       case LoopHintAttr::Vectorize:
733       case LoopHintAttr::Interleave:
734       case LoopHintAttr::UnrollCount:
735       case LoopHintAttr::UnrollAndJamCount:
736       case LoopHintAttr::VectorizeWidth:
737       case LoopHintAttr::InterleaveCount:
738       case LoopHintAttr::Distribute:
739       case LoopHintAttr::PipelineDisabled:
740       case LoopHintAttr::PipelineInitiationInterval:
741       case LoopHintAttr::VectorizePredicate:
742         llvm_unreachable("Options cannot be used with 'full' hint.");
743         break;
744       }
745       break;
746     case LoopHintAttr::FixedWidth:
747     case LoopHintAttr::ScalableWidth:
748       switch (Option) {
749       case LoopHintAttr::VectorizeWidth:
750         setVectorizeScalable(State == LoopHintAttr::ScalableWidth
751                                  ? LoopAttributes::Enable
752                                  : LoopAttributes::Disable);
753         if (LH->getValue())
754           setVectorizeWidth(ValueInt);
755         break;
756       default:
757         llvm_unreachable("Options cannot be used with 'scalable' hint.");
758         break;
759       }
760       break;
761     case LoopHintAttr::Numeric:
762       switch (Option) {
763       case LoopHintAttr::InterleaveCount:
764         setInterleaveCount(ValueInt);
765         break;
766       case LoopHintAttr::UnrollCount:
767         setUnrollCount(ValueInt);
768         break;
769       case LoopHintAttr::UnrollAndJamCount:
770         setUnrollAndJamCount(ValueInt);
771         break;
772       case LoopHintAttr::PipelineInitiationInterval:
773         setPipelineInitiationInterval(ValueInt);
774         break;
775       case LoopHintAttr::Unroll:
776       case LoopHintAttr::UnrollAndJam:
777       case LoopHintAttr::VectorizePredicate:
778       case LoopHintAttr::Vectorize:
779       case LoopHintAttr::VectorizeWidth:
780       case LoopHintAttr::Interleave:
781       case LoopHintAttr::Distribute:
782       case LoopHintAttr::PipelineDisabled:
783         llvm_unreachable("Options cannot be assigned a value.");
784         break;
785       }
786       break;
787     }
788   }
789 
790   setMustProgress(MustProgress);
791 
792   if (CGOpts.OptimizationLevel > 0)
793     // Disable unrolling for the loop, if unrolling is disabled (via
794     // -fno-unroll-loops) and no pragmas override the decision.
795     if (!CGOpts.UnrollLoops &&
796         (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
797          StagedAttrs.UnrollCount == 0))
798       setUnrollState(LoopAttributes::Disable);
799 
800   /// Stage the attributes.
801   push(Header, StartLoc, EndLoc);
802 }
803 
804 void LoopInfoStack::pop() {
805   assert(!Active.empty() && "No active loops to pop");
806   Active.back()->finish();
807   Active.pop_back();
808 }
809 
810 void LoopInfoStack::InsertHelper(Instruction *I) const {
811   if (I->mayReadOrWriteMemory()) {
812     SmallVector<Metadata *, 4> AccessGroups;
813     for (const auto &AL : Active) {
814       // Here we assume that every loop that has an access group is parallel.
815       if (MDNode *Group = AL->getAccessGroup())
816         AccessGroups.push_back(Group);
817     }
818     MDNode *UnionMD = nullptr;
819     if (AccessGroups.size() == 1)
820       UnionMD = cast<MDNode>(AccessGroups[0]);
821     else if (AccessGroups.size() >= 2)
822       UnionMD = MDNode::get(I->getContext(), AccessGroups);
823     I->setMetadata("llvm.access.group", UnionMD);
824   }
825 
826   if (!hasInfo())
827     return;
828 
829   const LoopInfo &L = getInfo();
830   if (!L.getLoopID())
831     return;
832 
833   if (I->isTerminator()) {
834     for (BasicBlock *Succ : successors(I))
835       if (Succ == L.getHeader()) {
836         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
837         break;
838       }
839     return;
840   }
841 }
842