xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp (revision 77013d11e6483b970af25e13c9b892075742f7e5) 
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
13 #include "clang/Basic/CodeGenOptions.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 using namespace clang::CodeGen;
21 using namespace llvm;
22 
23 MDNode *
24 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25   LLVMContext &Ctx = Header->getContext();
26   SmallVector<Metadata *, 4> NewLoopProperties;
27   NewLoopProperties.push_back(nullptr);
28   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29 
30   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
31   LoopID->replaceOperandWith(0, LoopID);
32   return LoopID;
33 }
34 
35 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
36                                            ArrayRef<Metadata *> LoopProperties,
37                                            bool &HasUserTransforms) {
38   LLVMContext &Ctx = Header->getContext();
39 
40   Optional<bool> Enabled;
41   if (Attrs.PipelineDisabled)
42     Enabled = false;
43   else if (Attrs.PipelineInitiationInterval != 0)
44     Enabled = true;
45 
46   if (Enabled != true) {
47     SmallVector<Metadata *, 4> NewLoopProperties;
48     if (Enabled == false) {
49       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
50       NewLoopProperties.push_back(
51           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
52                             ConstantAsMetadata::get(ConstantInt::get(
53                                 llvm::Type::getInt1Ty(Ctx), 1))}));
54       LoopProperties = NewLoopProperties;
55     }
56     return createLoopPropertiesMetadata(LoopProperties);
57   }
58 
59   SmallVector<Metadata *, 4> Args;
60   Args.push_back(nullptr);
61   Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63   if (Attrs.PipelineInitiationInterval > 0) {
64     Metadata *Vals[] = {
65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66         ConstantAsMetadata::get(ConstantInt::get(
67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68     Args.push_back(MDNode::get(Ctx, Vals));
69   }
70 
71   // No follow-up: This is the last transformation.
72 
73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74   LoopID->replaceOperandWith(0, LoopID);
75   HasUserTransforms = true;
76   return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81                                       ArrayRef<Metadata *> LoopProperties,
82                                       bool &HasUserTransforms) {
83   LLVMContext &Ctx = Header->getContext();
84 
85   Optional<bool> Enabled;
86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
87     Enabled = false;
88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
89     Enabled = None;
90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91            Attrs.UnrollCount != 0)
92     Enabled = true;
93 
94   if (Enabled != true) {
95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96     // if unrolling is disabled.
97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98   }
99 
100   SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102   // Apply all loop properties to the unrolled loop.
103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105   // Don't unroll an already unrolled loop.
106   FollowupLoopProperties.push_back(
107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109   bool FollowupHasTransforms = false;
110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111                                               FollowupHasTransforms);
112 
113   SmallVector<Metadata *, 4> Args;
114   Args.push_back(nullptr);
115   Args.append(LoopProperties.begin(), LoopProperties.end());
116 
117   // Setting unroll.count
118   if (Attrs.UnrollCount > 0) {
119     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
120                         ConstantAsMetadata::get(ConstantInt::get(
121                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
122     Args.push_back(MDNode::get(Ctx, Vals));
123   }
124 
125   // Setting unroll.full or unroll.disable
126   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
127     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
128     Args.push_back(MDNode::get(Ctx, Vals));
129   }
130 
131   if (FollowupHasTransforms)
132     Args.push_back(MDNode::get(
133         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
134 
135   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
136   LoopID->replaceOperandWith(0, LoopID);
137   HasUserTransforms = true;
138   return LoopID;
139 }
140 
141 MDNode *
142 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
143                                      ArrayRef<Metadata *> LoopProperties,
144                                      bool &HasUserTransforms) {
145   LLVMContext &Ctx = Header->getContext();
146 
147   Optional<bool> Enabled;
148   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
149     Enabled = false;
150   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
151            Attrs.UnrollAndJamCount != 0)
152     Enabled = true;
153 
154   if (Enabled != true) {
155     SmallVector<Metadata *, 4> NewLoopProperties;
156     if (Enabled == false) {
157       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
158       NewLoopProperties.push_back(MDNode::get(
159           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
160       LoopProperties = NewLoopProperties;
161     }
162     return createPartialUnrollMetadata(Attrs, LoopProperties,
163                                        HasUserTransforms);
164   }
165 
166   SmallVector<Metadata *, 4> FollowupLoopProperties;
167   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
168   FollowupLoopProperties.push_back(
169       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
170 
171   bool FollowupHasTransforms = false;
172   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
173                                                  FollowupHasTransforms);
174 
175   SmallVector<Metadata *, 4> Args;
176   Args.push_back(nullptr);
177   Args.append(LoopProperties.begin(), LoopProperties.end());
178 
179   // Setting unroll_and_jam.count
180   if (Attrs.UnrollAndJamCount > 0) {
181     Metadata *Vals[] = {
182         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
183         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
184                                                  Attrs.UnrollAndJamCount))};
185     Args.push_back(MDNode::get(Ctx, Vals));
186   }
187 
188   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
189     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
190     Args.push_back(MDNode::get(Ctx, Vals));
191   }
192 
193   if (FollowupHasTransforms)
194     Args.push_back(MDNode::get(
195         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
196               Followup}));
197 
198   if (UnrollAndJamInnerFollowup)
199     Args.push_back(MDNode::get(
200         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
201               UnrollAndJamInnerFollowup}));
202 
203   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
204   LoopID->replaceOperandWith(0, LoopID);
205   HasUserTransforms = true;
206   return LoopID;
207 }
208 
209 MDNode *
210 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
211                                       ArrayRef<Metadata *> LoopProperties,
212                                       bool &HasUserTransforms) {
213   LLVMContext &Ctx = Header->getContext();
214 
215   Optional<bool> Enabled;
216   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
217     Enabled = false;
218   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
219            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
220            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
221            Attrs.VectorizeScalable != LoopAttributes::Unspecified)
222     Enabled = true;
223 
224   if (Enabled != true) {
225     SmallVector<Metadata *, 4> NewLoopProperties;
226     if (Enabled == false) {
227       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
228       NewLoopProperties.push_back(
229           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
230                             ConstantAsMetadata::get(ConstantInt::get(
231                                 llvm::Type::getInt1Ty(Ctx), 0))}));
232       LoopProperties = NewLoopProperties;
233     }
234     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
235   }
236 
237   // Apply all loop properties to the vectorized loop.
238   SmallVector<Metadata *, 4> FollowupLoopProperties;
239   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
240 
241   // Don't vectorize an already vectorized loop.
242   FollowupLoopProperties.push_back(
243       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
244 
245   bool FollowupHasTransforms = false;
246   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
247                                                 FollowupHasTransforms);
248 
249   SmallVector<Metadata *, 4> Args;
250   Args.push_back(nullptr);
251   Args.append(LoopProperties.begin(), LoopProperties.end());
252 
253   // Setting vectorize.predicate
254   bool IsVectorPredicateEnabled = false;
255   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
256       Attrs.VectorizeEnable != LoopAttributes::Disable &&
257       Attrs.VectorizeWidth < 1) {
258 
259     IsVectorPredicateEnabled =
260         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
261 
262     Metadata *Vals[] = {
263         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
264         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
265                                                  IsVectorPredicateEnabled))};
266     Args.push_back(MDNode::get(Ctx, Vals));
267   }
268 
269   // Setting vectorize.width
270   if (Attrs.VectorizeWidth > 0) {
271     Metadata *Vals[] = {
272         MDString::get(Ctx, "llvm.loop.vectorize.width"),
273         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
274                                                  Attrs.VectorizeWidth))};
275 
276     Args.push_back(MDNode::get(Ctx, Vals));
277   }
278 
279   if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
280     bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
281     Metadata *Vals[] = {
282         MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
283         ConstantAsMetadata::get(
284             ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
285     Args.push_back(MDNode::get(Ctx, Vals));
286   }
287 
288   // Setting interleave.count
289   if (Attrs.InterleaveCount > 0) {
290     Metadata *Vals[] = {
291         MDString::get(Ctx, "llvm.loop.interleave.count"),
292         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
293                                                  Attrs.InterleaveCount))};
294     Args.push_back(MDNode::get(Ctx, Vals));
295   }
296 
297   // vectorize.enable is set if:
298   // 1) loop hint vectorize.enable is set, or
299   // 2) it is implied when vectorize.predicate is set, or
300   // 3) it is implied when vectorize.width is set to a value > 1
301   // 4) it is implied when vectorize.scalable.enable is true
302   // 5) it is implied when vectorize.width is unset (0) and the user
303   //    explicitly requested fixed-width vectorization, i.e.
304   //    vectorize.scalable.enable is false.
305   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
306       IsVectorPredicateEnabled || Attrs.VectorizeWidth > 1 ||
307       Attrs.VectorizeScalable == LoopAttributes::Enable ||
308       (Attrs.VectorizeScalable == LoopAttributes::Disable &&
309        Attrs.VectorizeWidth != 1)) {
310     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
311     Args.push_back(
312         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
313                           ConstantAsMetadata::get(ConstantInt::get(
314                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
315   }
316 
317   if (FollowupHasTransforms)
318     Args.push_back(MDNode::get(
319         Ctx,
320         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
321 
322   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
323   LoopID->replaceOperandWith(0, LoopID);
324   HasUserTransforms = true;
325   return LoopID;
326 }
327 
328 MDNode *
329 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
330                                        ArrayRef<Metadata *> LoopProperties,
331                                        bool &HasUserTransforms) {
332   LLVMContext &Ctx = Header->getContext();
333 
334   Optional<bool> Enabled;
335   if (Attrs.DistributeEnable == LoopAttributes::Disable)
336     Enabled = false;
337   if (Attrs.DistributeEnable == LoopAttributes::Enable)
338     Enabled = true;
339 
340   if (Enabled != true) {
341     SmallVector<Metadata *, 4> NewLoopProperties;
342     if (Enabled == false) {
343       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
344       NewLoopProperties.push_back(
345           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
346                             ConstantAsMetadata::get(ConstantInt::get(
347                                 llvm::Type::getInt1Ty(Ctx), 0))}));
348       LoopProperties = NewLoopProperties;
349     }
350     return createLoopVectorizeMetadata(Attrs, LoopProperties,
351                                        HasUserTransforms);
352   }
353 
354   bool FollowupHasTransforms = false;
355   MDNode *Followup =
356       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
357 
358   SmallVector<Metadata *, 4> Args;
359   Args.push_back(nullptr);
360   Args.append(LoopProperties.begin(), LoopProperties.end());
361 
362   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
363                       ConstantAsMetadata::get(ConstantInt::get(
364                           llvm::Type::getInt1Ty(Ctx),
365                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
366   Args.push_back(MDNode::get(Ctx, Vals));
367 
368   if (FollowupHasTransforms)
369     Args.push_back(MDNode::get(
370         Ctx,
371         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
372 
373   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
374   LoopID->replaceOperandWith(0, LoopID);
375   HasUserTransforms = true;
376   return LoopID;
377 }
378 
379 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
380                                            ArrayRef<Metadata *> LoopProperties,
381                                            bool &HasUserTransforms) {
382   LLVMContext &Ctx = Header->getContext();
383 
384   Optional<bool> Enabled;
385   if (Attrs.UnrollEnable == LoopAttributes::Disable)
386     Enabled = false;
387   else if (Attrs.UnrollEnable == LoopAttributes::Full)
388     Enabled = true;
389 
390   if (Enabled != true) {
391     SmallVector<Metadata *, 4> NewLoopProperties;
392     if (Enabled == false) {
393       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
394       NewLoopProperties.push_back(
395           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
396       LoopProperties = NewLoopProperties;
397     }
398     return createLoopDistributeMetadata(Attrs, LoopProperties,
399                                         HasUserTransforms);
400   }
401 
402   SmallVector<Metadata *, 4> Args;
403   Args.push_back(nullptr);
404   Args.append(LoopProperties.begin(), LoopProperties.end());
405   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
406 
407   // No follow-up: there is no loop after full unrolling.
408   // TODO: Warn if there are transformations after full unrolling.
409 
410   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
411   LoopID->replaceOperandWith(0, LoopID);
412   HasUserTransforms = true;
413   return LoopID;
414 }
415 
416 MDNode *LoopInfo::createMetadata(
417     const LoopAttributes &Attrs,
418     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
419     bool &HasUserTransforms) {
420   SmallVector<Metadata *, 3> LoopProperties;
421 
422   // If we have a valid start debug location for the loop, add it.
423   if (StartLoc) {
424     LoopProperties.push_back(StartLoc.getAsMDNode());
425 
426     // If we also have a valid end debug location for the loop, add it.
427     if (EndLoc)
428       LoopProperties.push_back(EndLoc.getAsMDNode());
429   }
430 
431   LLVMContext &Ctx = Header->getContext();
432   if (Attrs.MustProgress)
433     LoopProperties.push_back(
434         MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
435 
436   assert(!!AccGroup == Attrs.IsParallel &&
437          "There must be an access group iff the loop is parallel");
438   if (Attrs.IsParallel) {
439     LoopProperties.push_back(MDNode::get(
440         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
441   }
442 
443   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
444                         AdditionalLoopProperties.end());
445   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
446 }
447 
448 LoopAttributes::LoopAttributes(bool IsParallel)
449     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
450       UnrollEnable(LoopAttributes::Unspecified),
451       UnrollAndJamEnable(LoopAttributes::Unspecified),
452       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
453       VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
454       UnrollCount(0), UnrollAndJamCount(0),
455       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
456       PipelineInitiationInterval(0), MustProgress(false) {}
457 
458 void LoopAttributes::clear() {
459   IsParallel = false;
460   VectorizeWidth = 0;
461   VectorizeScalable = LoopAttributes::Unspecified;
462   InterleaveCount = 0;
463   UnrollCount = 0;
464   UnrollAndJamCount = 0;
465   VectorizeEnable = LoopAttributes::Unspecified;
466   UnrollEnable = LoopAttributes::Unspecified;
467   UnrollAndJamEnable = LoopAttributes::Unspecified;
468   VectorizePredicateEnable = LoopAttributes::Unspecified;
469   DistributeEnable = LoopAttributes::Unspecified;
470   PipelineDisabled = false;
471   PipelineInitiationInterval = 0;
472   MustProgress = false;
473 }
474 
475 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
476                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
477                    LoopInfo *Parent)
478     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
479       Parent(Parent) {
480 
481   if (Attrs.IsParallel) {
482     // Create an access group for this loop.
483     LLVMContext &Ctx = Header->getContext();
484     AccGroup = MDNode::getDistinct(Ctx, {});
485   }
486 
487   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
488       Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
489       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
490       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
491       Attrs.PipelineInitiationInterval == 0 &&
492       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
493       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
494       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
495       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
496       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
497       !EndLoc && !Attrs.MustProgress)
498     return;
499 
500   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
501 }
502 
503 void LoopInfo::finish() {
504   // We did not annotate the loop body instructions because there are no
505   // attributes for this loop.
506   if (!TempLoopID)
507     return;
508 
509   MDNode *LoopID;
510   LoopAttributes CurLoopAttr = Attrs;
511   LLVMContext &Ctx = Header->getContext();
512 
513   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
514                  Parent->Attrs.UnrollAndJamCount != 0)) {
515     // Parent unroll-and-jams this loop.
516     // Split the transformations in those that happens before the unroll-and-jam
517     // and those after.
518 
519     LoopAttributes BeforeJam, AfterJam;
520 
521     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
522 
523     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
524     BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
525     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
526     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
527     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
528     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
529 
530     switch (Attrs.UnrollEnable) {
531     case LoopAttributes::Unspecified:
532     case LoopAttributes::Disable:
533       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
534       AfterJam.UnrollEnable = Attrs.UnrollEnable;
535       break;
536     case LoopAttributes::Full:
537       BeforeJam.UnrollEnable = LoopAttributes::Full;
538       break;
539     case LoopAttributes::Enable:
540       AfterJam.UnrollEnable = LoopAttributes::Enable;
541       break;
542     }
543 
544     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
545     AfterJam.UnrollCount = Attrs.UnrollCount;
546     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
547     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
548 
549     // If this loop is subject of an unroll-and-jam by the parent loop, and has
550     // an unroll-and-jam annotation itself, we have to decide whether to first
551     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
552     // UnrollAndJam pass processes loops from inner to outer, so we apply the
553     // inner first.
554     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
555     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
556 
557     // Set the inner followup metadata to process by the outer loop. Only
558     // consider the first inner loop.
559     if (!Parent->UnrollAndJamInnerFollowup) {
560       // Splitting the attributes into a BeforeJam and an AfterJam part will
561       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
562       // to be forwarded to the AfterJam part. We detect the situation here and
563       // add it manually.
564       SmallVector<Metadata *, 1> BeforeLoopProperties;
565       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
566           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
567           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
568           BeforeJam.VectorizeScalable == LoopAttributes::Enable)
569         BeforeLoopProperties.push_back(
570             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
571 
572       bool InnerFollowupHasTransform = false;
573       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
574                                              InnerFollowupHasTransform);
575       if (InnerFollowupHasTransform)
576         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
577     }
578 
579     CurLoopAttr = BeforeJam;
580   }
581 
582   bool HasUserTransforms = false;
583   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
584   TempLoopID->replaceAllUsesWith(LoopID);
585 }
586 
587 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
588                          const llvm::DebugLoc &EndLoc) {
589   Active.emplace_back(
590       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
591                    Active.empty() ? nullptr : Active.back().get()));
592   // Clear the attributes so nested loops do not inherit them.
593   StagedAttrs.clear();
594 }
595 
596 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
597                          const clang::CodeGenOptions &CGOpts,
598                          ArrayRef<const clang::Attr *> Attrs,
599                          const llvm::DebugLoc &StartLoc,
600                          const llvm::DebugLoc &EndLoc, bool MustProgress) {
601   // Identify loop hint attributes from Attrs.
602   for (const auto *Attr : Attrs) {
603     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
604     const OpenCLUnrollHintAttr *OpenCLHint =
605         dyn_cast<OpenCLUnrollHintAttr>(Attr);
606 
607     // Skip non loop hint attributes
608     if (!LH && !OpenCLHint) {
609       continue;
610     }
611 
612     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
613     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
614     unsigned ValueInt = 1;
615     // Translate opencl_unroll_hint attribute argument to
616     // equivalent LoopHintAttr enums.
617     // OpenCL v2.0 s6.11.5:
618     // 0 - enable unroll (no argument).
619     // 1 - disable unroll.
620     // other positive integer n - unroll by n.
621     if (OpenCLHint) {
622       ValueInt = OpenCLHint->getUnrollHint();
623       if (ValueInt == 0) {
624         State = LoopHintAttr::Enable;
625       } else if (ValueInt != 1) {
626         Option = LoopHintAttr::UnrollCount;
627         State = LoopHintAttr::Numeric;
628       }
629     } else if (LH) {
630       auto *ValueExpr = LH->getValue();
631       if (ValueExpr) {
632         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
633         ValueInt = ValueAPS.getSExtValue();
634       }
635 
636       Option = LH->getOption();
637       State = LH->getState();
638     }
639     switch (State) {
640     case LoopHintAttr::Disable:
641       switch (Option) {
642       case LoopHintAttr::Vectorize:
643         // Disable vectorization by specifying a width of 1.
644         setVectorizeWidth(1);
645         setVectorizeScalable(LoopAttributes::Unspecified);
646         break;
647       case LoopHintAttr::Interleave:
648         // Disable interleaving by speciyfing a count of 1.
649         setInterleaveCount(1);
650         break;
651       case LoopHintAttr::Unroll:
652         setUnrollState(LoopAttributes::Disable);
653         break;
654       case LoopHintAttr::UnrollAndJam:
655         setUnrollAndJamState(LoopAttributes::Disable);
656         break;
657       case LoopHintAttr::VectorizePredicate:
658         setVectorizePredicateState(LoopAttributes::Disable);
659         break;
660       case LoopHintAttr::Distribute:
661         setDistributeState(false);
662         break;
663       case LoopHintAttr::PipelineDisabled:
664         setPipelineDisabled(true);
665         break;
666       case LoopHintAttr::UnrollCount:
667       case LoopHintAttr::UnrollAndJamCount:
668       case LoopHintAttr::VectorizeWidth:
669       case LoopHintAttr::InterleaveCount:
670       case LoopHintAttr::PipelineInitiationInterval:
671         llvm_unreachable("Options cannot be disabled.");
672         break;
673       }
674       break;
675     case LoopHintAttr::Enable:
676       switch (Option) {
677       case LoopHintAttr::Vectorize:
678       case LoopHintAttr::Interleave:
679         setVectorizeEnable(true);
680         break;
681       case LoopHintAttr::Unroll:
682         setUnrollState(LoopAttributes::Enable);
683         break;
684       case LoopHintAttr::UnrollAndJam:
685         setUnrollAndJamState(LoopAttributes::Enable);
686         break;
687       case LoopHintAttr::VectorizePredicate:
688         setVectorizePredicateState(LoopAttributes::Enable);
689         break;
690       case LoopHintAttr::Distribute:
691         setDistributeState(true);
692         break;
693       case LoopHintAttr::UnrollCount:
694       case LoopHintAttr::UnrollAndJamCount:
695       case LoopHintAttr::VectorizeWidth:
696       case LoopHintAttr::InterleaveCount:
697       case LoopHintAttr::PipelineDisabled:
698       case LoopHintAttr::PipelineInitiationInterval:
699         llvm_unreachable("Options cannot enabled.");
700         break;
701       }
702       break;
703     case LoopHintAttr::AssumeSafety:
704       switch (Option) {
705       case LoopHintAttr::Vectorize:
706       case LoopHintAttr::Interleave:
707         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
708         setParallel(true);
709         setVectorizeEnable(true);
710         break;
711       case LoopHintAttr::Unroll:
712       case LoopHintAttr::UnrollAndJam:
713       case LoopHintAttr::VectorizePredicate:
714       case LoopHintAttr::UnrollCount:
715       case LoopHintAttr::UnrollAndJamCount:
716       case LoopHintAttr::VectorizeWidth:
717       case LoopHintAttr::InterleaveCount:
718       case LoopHintAttr::Distribute:
719       case LoopHintAttr::PipelineDisabled:
720       case LoopHintAttr::PipelineInitiationInterval:
721         llvm_unreachable("Options cannot be used to assume mem safety.");
722         break;
723       }
724       break;
725     case LoopHintAttr::Full:
726       switch (Option) {
727       case LoopHintAttr::Unroll:
728         setUnrollState(LoopAttributes::Full);
729         break;
730       case LoopHintAttr::UnrollAndJam:
731         setUnrollAndJamState(LoopAttributes::Full);
732         break;
733       case LoopHintAttr::Vectorize:
734       case LoopHintAttr::Interleave:
735       case LoopHintAttr::UnrollCount:
736       case LoopHintAttr::UnrollAndJamCount:
737       case LoopHintAttr::VectorizeWidth:
738       case LoopHintAttr::InterleaveCount:
739       case LoopHintAttr::Distribute:
740       case LoopHintAttr::PipelineDisabled:
741       case LoopHintAttr::PipelineInitiationInterval:
742       case LoopHintAttr::VectorizePredicate:
743         llvm_unreachable("Options cannot be used with 'full' hint.");
744         break;
745       }
746       break;
747     case LoopHintAttr::FixedWidth:
748     case LoopHintAttr::ScalableWidth:
749       switch (Option) {
750       case LoopHintAttr::VectorizeWidth:
751         setVectorizeScalable(State == LoopHintAttr::ScalableWidth
752                                  ? LoopAttributes::Enable
753                                  : LoopAttributes::Disable);
754         if (LH->getValue())
755           setVectorizeWidth(ValueInt);
756         break;
757       default:
758         llvm_unreachable("Options cannot be used with 'scalable' hint.");
759         break;
760       }
761       break;
762     case LoopHintAttr::Numeric:
763       switch (Option) {
764       case LoopHintAttr::InterleaveCount:
765         setInterleaveCount(ValueInt);
766         break;
767       case LoopHintAttr::UnrollCount:
768         setUnrollCount(ValueInt);
769         break;
770       case LoopHintAttr::UnrollAndJamCount:
771         setUnrollAndJamCount(ValueInt);
772         break;
773       case LoopHintAttr::PipelineInitiationInterval:
774         setPipelineInitiationInterval(ValueInt);
775         break;
776       case LoopHintAttr::Unroll:
777       case LoopHintAttr::UnrollAndJam:
778       case LoopHintAttr::VectorizePredicate:
779       case LoopHintAttr::Vectorize:
780       case LoopHintAttr::VectorizeWidth:
781       case LoopHintAttr::Interleave:
782       case LoopHintAttr::Distribute:
783       case LoopHintAttr::PipelineDisabled:
784         llvm_unreachable("Options cannot be assigned a value.");
785         break;
786       }
787       break;
788     }
789   }
790 
791   setMustProgress(MustProgress);
792 
793   if (CGOpts.OptimizationLevel > 0)
794     // Disable unrolling for the loop, if unrolling is disabled (via
795     // -fno-unroll-loops) and no pragmas override the decision.
796     if (!CGOpts.UnrollLoops &&
797         (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
798          StagedAttrs.UnrollCount == 0))
799       setUnrollState(LoopAttributes::Disable);
800 
801   /// Stage the attributes.
802   push(Header, StartLoc, EndLoc);
803 }
804 
805 void LoopInfoStack::pop() {
806   assert(!Active.empty() && "No active loops to pop");
807   Active.back()->finish();
808   Active.pop_back();
809 }
810 
811 void LoopInfoStack::InsertHelper(Instruction *I) const {
812   if (I->mayReadOrWriteMemory()) {
813     SmallVector<Metadata *, 4> AccessGroups;
814     for (const auto &AL : Active) {
815       // Here we assume that every loop that has an access group is parallel.
816       if (MDNode *Group = AL->getAccessGroup())
817         AccessGroups.push_back(Group);
818     }
819     MDNode *UnionMD = nullptr;
820     if (AccessGroups.size() == 1)
821       UnionMD = cast<MDNode>(AccessGroups[0]);
822     else if (AccessGroups.size() >= 2)
823       UnionMD = MDNode::get(I->getContext(), AccessGroups);
824     I->setMetadata("llvm.access.group", UnionMD);
825   }
826 
827   if (!hasInfo())
828     return;
829 
830   const LoopInfo &L = getInfo();
831   if (!L.getLoopID())
832     return;
833 
834   if (I->isTerminator()) {
835     for (BasicBlock *Succ : successors(I))
836       if (Succ == L.getHeader()) {
837         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
838         break;
839       }
840     return;
841   }
842 }
843