1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "CGLoopInfo.h" 10 #include "clang/AST/ASTContext.h" 11 #include "clang/AST/Attr.h" 12 #include "clang/AST/Expr.h" 13 #include "clang/Basic/CodeGenOptions.h" 14 #include "llvm/IR/BasicBlock.h" 15 #include "llvm/IR/CFG.h" 16 #include "llvm/IR/Constants.h" 17 #include "llvm/IR/InstrTypes.h" 18 #include "llvm/IR/Instructions.h" 19 #include "llvm/IR/Metadata.h" 20 #include <optional> 21 using namespace clang::CodeGen; 22 using namespace llvm; 23 24 MDNode * 25 LoopInfo::createFollowupMetadata(const char *FollowupName, 26 ArrayRef<llvm::Metadata *> LoopProperties) { 27 LLVMContext &Ctx = Header->getContext(); 28 29 SmallVector<Metadata *, 4> Args; 30 Args.push_back(MDString::get(Ctx, FollowupName)); 31 Args.append(LoopProperties.begin(), LoopProperties.end()); 32 return MDNode::get(Ctx, Args); 33 } 34 35 SmallVector<Metadata *, 4> 36 LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs, 37 ArrayRef<Metadata *> LoopProperties, 38 bool &HasUserTransforms) { 39 LLVMContext &Ctx = Header->getContext(); 40 41 std::optional<bool> Enabled; 42 if (Attrs.PipelineDisabled) 43 Enabled = false; 44 else if (Attrs.PipelineInitiationInterval != 0) 45 Enabled = true; 46 47 SmallVector<Metadata *, 4> Args; 48 Args.append(LoopProperties.begin(), LoopProperties.end()); 49 50 if (Enabled != true) { 51 if (Enabled == false) { 52 Args.push_back( 53 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"), 54 ConstantAsMetadata::get(ConstantInt::get( 55 llvm::Type::getInt1Ty(Ctx), 1))})); 56 } 57 return Args; 58 } 59 60 if (Attrs.PipelineInitiationInterval > 0) { 61 Metadata *Vals[] = { 62 MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"), 63 ConstantAsMetadata::get(ConstantInt::get( 64 llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))}; 65 Args.push_back(MDNode::get(Ctx, Vals)); 66 } 67 68 // No follow-up: This is the last transformation. 69 70 HasUserTransforms = true; 71 return Args; 72 } 73 74 SmallVector<Metadata *, 4> 75 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs, 76 ArrayRef<Metadata *> LoopProperties, 77 bool &HasUserTransforms) { 78 LLVMContext &Ctx = Header->getContext(); 79 80 std::optional<bool> Enabled; 81 if (Attrs.UnrollEnable == LoopAttributes::Disable) 82 Enabled = false; 83 else if (Attrs.UnrollEnable == LoopAttributes::Full) 84 Enabled = std::nullopt; 85 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified || 86 Attrs.UnrollCount != 0) 87 Enabled = true; 88 89 if (Enabled != true) { 90 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable 91 // if unrolling is disabled. 92 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms); 93 } 94 95 SmallVector<Metadata *, 4> FollowupLoopProperties; 96 97 // Apply all loop properties to the unrolled loop. 98 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); 99 100 // Don't unroll an already unrolled loop. 101 FollowupLoopProperties.push_back( 102 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable"))); 103 104 bool FollowupHasTransforms = false; 105 SmallVector<Metadata *, 4> Followup = createPipeliningMetadata( 106 Attrs, FollowupLoopProperties, FollowupHasTransforms); 107 108 SmallVector<Metadata *, 4> Args; 109 Args.append(LoopProperties.begin(), LoopProperties.end()); 110 111 // Setting unroll.count 112 if (Attrs.UnrollCount > 0) { 113 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), 114 ConstantAsMetadata::get(ConstantInt::get( 115 llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; 116 Args.push_back(MDNode::get(Ctx, Vals)); 117 } 118 119 // Setting unroll.full or unroll.disable 120 if (Attrs.UnrollEnable == LoopAttributes::Enable) { 121 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")}; 122 Args.push_back(MDNode::get(Ctx, Vals)); 123 } 124 125 if (FollowupHasTransforms) 126 Args.push_back( 127 createFollowupMetadata("llvm.loop.unroll.followup_all", Followup)); 128 129 HasUserTransforms = true; 130 return Args; 131 } 132 133 SmallVector<Metadata *, 4> 134 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs, 135 ArrayRef<Metadata *> LoopProperties, 136 bool &HasUserTransforms) { 137 LLVMContext &Ctx = Header->getContext(); 138 139 std::optional<bool> Enabled; 140 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable) 141 Enabled = false; 142 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable || 143 Attrs.UnrollAndJamCount != 0) 144 Enabled = true; 145 146 if (Enabled != true) { 147 SmallVector<Metadata *, 4> NewLoopProperties; 148 if (Enabled == false) { 149 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); 150 NewLoopProperties.push_back(MDNode::get( 151 Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable"))); 152 LoopProperties = NewLoopProperties; 153 } 154 return createPartialUnrollMetadata(Attrs, LoopProperties, 155 HasUserTransforms); 156 } 157 158 SmallVector<Metadata *, 4> FollowupLoopProperties; 159 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); 160 FollowupLoopProperties.push_back( 161 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable"))); 162 163 bool FollowupHasTransforms = false; 164 SmallVector<Metadata *, 4> Followup = createPartialUnrollMetadata( 165 Attrs, FollowupLoopProperties, FollowupHasTransforms); 166 167 SmallVector<Metadata *, 4> Args; 168 Args.append(LoopProperties.begin(), LoopProperties.end()); 169 170 // Setting unroll_and_jam.count 171 if (Attrs.UnrollAndJamCount > 0) { 172 Metadata *Vals[] = { 173 MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"), 174 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 175 Attrs.UnrollAndJamCount))}; 176 Args.push_back(MDNode::get(Ctx, Vals)); 177 } 178 179 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) { 180 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")}; 181 Args.push_back(MDNode::get(Ctx, Vals)); 182 } 183 184 if (FollowupHasTransforms) 185 Args.push_back(createFollowupMetadata( 186 "llvm.loop.unroll_and_jam.followup_outer", Followup)); 187 188 if (UnrollAndJamInnerFollowup.has_value()) 189 Args.push_back(createFollowupMetadata( 190 "llvm.loop.unroll_and_jam.followup_inner", *UnrollAndJamInnerFollowup)); 191 192 HasUserTransforms = true; 193 return Args; 194 } 195 196 SmallVector<Metadata *, 4> 197 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, 198 ArrayRef<Metadata *> LoopProperties, 199 bool &HasUserTransforms) { 200 LLVMContext &Ctx = Header->getContext(); 201 202 std::optional<bool> Enabled; 203 if (Attrs.VectorizeEnable == LoopAttributes::Disable) 204 Enabled = false; 205 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || 206 Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified || 207 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 || 208 Attrs.VectorizeScalable != LoopAttributes::Unspecified) 209 Enabled = true; 210 211 if (Enabled != true) { 212 SmallVector<Metadata *, 4> NewLoopProperties; 213 if (Enabled == false) { 214 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); 215 NewLoopProperties.push_back( 216 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), 217 ConstantAsMetadata::get(ConstantInt::get( 218 llvm::Type::getInt1Ty(Ctx), 0))})); 219 LoopProperties = NewLoopProperties; 220 } 221 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms); 222 } 223 224 // Apply all loop properties to the vectorized loop. 225 SmallVector<Metadata *, 4> FollowupLoopProperties; 226 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); 227 228 // Don't vectorize an already vectorized loop. 229 FollowupLoopProperties.push_back( 230 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); 231 232 bool FollowupHasTransforms = false; 233 SmallVector<Metadata *, 4> Followup = createUnrollAndJamMetadata( 234 Attrs, FollowupLoopProperties, FollowupHasTransforms); 235 236 SmallVector<Metadata *, 4> Args; 237 Args.append(LoopProperties.begin(), LoopProperties.end()); 238 239 // Setting vectorize.predicate when it has been specified and vectorization 240 // has not been disabled. 241 bool IsVectorPredicateEnabled = false; 242 if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) { 243 IsVectorPredicateEnabled = 244 (Attrs.VectorizePredicateEnable == LoopAttributes::Enable); 245 246 Metadata *Vals[] = { 247 MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"), 248 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx), 249 IsVectorPredicateEnabled))}; 250 Args.push_back(MDNode::get(Ctx, Vals)); 251 } 252 253 // Setting vectorize.width 254 if (Attrs.VectorizeWidth > 0) { 255 Metadata *Vals[] = { 256 MDString::get(Ctx, "llvm.loop.vectorize.width"), 257 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 258 Attrs.VectorizeWidth))}; 259 260 Args.push_back(MDNode::get(Ctx, Vals)); 261 } 262 263 if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) { 264 bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable; 265 Metadata *Vals[] = { 266 MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"), 267 ConstantAsMetadata::get( 268 ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))}; 269 Args.push_back(MDNode::get(Ctx, Vals)); 270 } 271 272 // Setting interleave.count 273 if (Attrs.InterleaveCount > 0) { 274 Metadata *Vals[] = { 275 MDString::get(Ctx, "llvm.loop.interleave.count"), 276 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 277 Attrs.InterleaveCount))}; 278 Args.push_back(MDNode::get(Ctx, Vals)); 279 } 280 281 // vectorize.enable is set if: 282 // 1) loop hint vectorize.enable is set, or 283 // 2) it is implied when vectorize.predicate is set, or 284 // 3) it is implied when vectorize.width is set to a value > 1 285 // 4) it is implied when vectorize.scalable.enable is true 286 // 5) it is implied when vectorize.width is unset (0) and the user 287 // explicitly requested fixed-width vectorization, i.e. 288 // vectorize.scalable.enable is false. 289 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || 290 (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) || 291 Attrs.VectorizeWidth > 1 || 292 Attrs.VectorizeScalable == LoopAttributes::Enable || 293 (Attrs.VectorizeScalable == LoopAttributes::Disable && 294 Attrs.VectorizeWidth != 1)) { 295 bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable; 296 Args.push_back( 297 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), 298 ConstantAsMetadata::get(ConstantInt::get( 299 llvm::Type::getInt1Ty(Ctx), AttrVal))})); 300 } 301 302 if (FollowupHasTransforms) 303 Args.push_back( 304 createFollowupMetadata("llvm.loop.vectorize.followup_all", Followup)); 305 306 HasUserTransforms = true; 307 return Args; 308 } 309 310 SmallVector<Metadata *, 4> 311 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs, 312 ArrayRef<Metadata *> LoopProperties, 313 bool &HasUserTransforms) { 314 LLVMContext &Ctx = Header->getContext(); 315 316 std::optional<bool> Enabled; 317 if (Attrs.DistributeEnable == LoopAttributes::Disable) 318 Enabled = false; 319 if (Attrs.DistributeEnable == LoopAttributes::Enable) 320 Enabled = true; 321 322 if (Enabled != true) { 323 SmallVector<Metadata *, 4> NewLoopProperties; 324 if (Enabled == false) { 325 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); 326 NewLoopProperties.push_back( 327 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"), 328 ConstantAsMetadata::get(ConstantInt::get( 329 llvm::Type::getInt1Ty(Ctx), 0))})); 330 LoopProperties = NewLoopProperties; 331 } 332 return createLoopVectorizeMetadata(Attrs, LoopProperties, 333 HasUserTransforms); 334 } 335 336 bool FollowupHasTransforms = false; 337 SmallVector<Metadata *, 4> Followup = 338 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms); 339 340 SmallVector<Metadata *, 4> Args; 341 Args.append(LoopProperties.begin(), LoopProperties.end()); 342 343 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), 344 ConstantAsMetadata::get(ConstantInt::get( 345 llvm::Type::getInt1Ty(Ctx), 346 (Attrs.DistributeEnable == LoopAttributes::Enable)))}; 347 Args.push_back(MDNode::get(Ctx, Vals)); 348 349 if (FollowupHasTransforms) 350 Args.push_back( 351 createFollowupMetadata("llvm.loop.distribute.followup_all", Followup)); 352 353 HasUserTransforms = true; 354 return Args; 355 } 356 357 SmallVector<Metadata *, 4> 358 LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs, 359 ArrayRef<Metadata *> LoopProperties, 360 bool &HasUserTransforms) { 361 LLVMContext &Ctx = Header->getContext(); 362 363 std::optional<bool> Enabled; 364 if (Attrs.UnrollEnable == LoopAttributes::Disable) 365 Enabled = false; 366 else if (Attrs.UnrollEnable == LoopAttributes::Full) 367 Enabled = true; 368 369 if (Enabled != true) { 370 SmallVector<Metadata *, 4> NewLoopProperties; 371 if (Enabled == false) { 372 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); 373 NewLoopProperties.push_back( 374 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable"))); 375 LoopProperties = NewLoopProperties; 376 } 377 return createLoopDistributeMetadata(Attrs, LoopProperties, 378 HasUserTransforms); 379 } 380 381 SmallVector<Metadata *, 4> Args; 382 Args.append(LoopProperties.begin(), LoopProperties.end()); 383 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))); 384 385 // No follow-up: there is no loop after full unrolling. 386 // TODO: Warn if there are transformations after full unrolling. 387 388 HasUserTransforms = true; 389 return Args; 390 } 391 392 SmallVector<Metadata *, 4> LoopInfo::createMetadata( 393 const LoopAttributes &Attrs, 394 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties, 395 bool &HasUserTransforms) { 396 SmallVector<Metadata *, 3> LoopProperties; 397 398 // If we have a valid start debug location for the loop, add it. 399 if (StartLoc) { 400 LoopProperties.push_back(StartLoc.getAsMDNode()); 401 402 // If we also have a valid end debug location for the loop, add it. 403 if (EndLoc) 404 LoopProperties.push_back(EndLoc.getAsMDNode()); 405 } 406 407 LLVMContext &Ctx = Header->getContext(); 408 if (Attrs.MustProgress) 409 LoopProperties.push_back( 410 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress"))); 411 412 assert(!!AccGroup == Attrs.IsParallel && 413 "There must be an access group iff the loop is parallel"); 414 if (Attrs.IsParallel) { 415 LoopProperties.push_back(MDNode::get( 416 Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup})); 417 } 418 419 // Setting clang::code_align attribute. 420 if (Attrs.CodeAlign > 0) { 421 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.align"), 422 ConstantAsMetadata::get(ConstantInt::get( 423 llvm::Type::getInt32Ty(Ctx), Attrs.CodeAlign))}; 424 LoopProperties.push_back(MDNode::get(Ctx, Vals)); 425 } 426 427 llvm::append_range(LoopProperties, AdditionalLoopProperties); 428 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms); 429 } 430 431 LoopAttributes::LoopAttributes(bool IsParallel) 432 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), 433 UnrollEnable(LoopAttributes::Unspecified), 434 UnrollAndJamEnable(LoopAttributes::Unspecified), 435 VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0), 436 VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0), 437 UnrollCount(0), UnrollAndJamCount(0), 438 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false), 439 PipelineInitiationInterval(0), CodeAlign(0), MustProgress(false) {} 440 441 void LoopAttributes::clear() { 442 IsParallel = false; 443 VectorizeWidth = 0; 444 VectorizeScalable = LoopAttributes::Unspecified; 445 InterleaveCount = 0; 446 UnrollCount = 0; 447 UnrollAndJamCount = 0; 448 VectorizeEnable = LoopAttributes::Unspecified; 449 UnrollEnable = LoopAttributes::Unspecified; 450 UnrollAndJamEnable = LoopAttributes::Unspecified; 451 VectorizePredicateEnable = LoopAttributes::Unspecified; 452 DistributeEnable = LoopAttributes::Unspecified; 453 PipelineDisabled = false; 454 PipelineInitiationInterval = 0; 455 CodeAlign = 0; 456 MustProgress = false; 457 } 458 459 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, 460 const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, 461 LoopInfo *Parent) 462 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc), 463 Parent(Parent) { 464 465 if (Attrs.IsParallel) { 466 // Create an access group for this loop. 467 LLVMContext &Ctx = Header->getContext(); 468 AccGroup = MDNode::getDistinct(Ctx, {}); 469 } 470 471 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && 472 Attrs.VectorizeScalable == LoopAttributes::Unspecified && 473 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && 474 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled && 475 Attrs.PipelineInitiationInterval == 0 && 476 Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified && 477 Attrs.VectorizeEnable == LoopAttributes::Unspecified && 478 Attrs.UnrollEnable == LoopAttributes::Unspecified && 479 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && 480 Attrs.DistributeEnable == LoopAttributes::Unspecified && 481 Attrs.CodeAlign == 0 && !StartLoc && !EndLoc && !Attrs.MustProgress) 482 return; 483 484 TempLoopID = MDNode::getTemporary(Header->getContext(), {}); 485 } 486 487 void LoopInfo::finish() { 488 // We did not annotate the loop body instructions because there are no 489 // attributes for this loop. 490 if (!TempLoopID) 491 return; 492 493 MDNode *LoopID; 494 LoopAttributes CurLoopAttr = Attrs; 495 LLVMContext &Ctx = Header->getContext(); 496 497 if (Parent && (Parent->Attrs.UnrollAndJamEnable || 498 Parent->Attrs.UnrollAndJamCount != 0)) { 499 // Parent unroll-and-jams this loop. 500 // Split the transformations in those that happens before the unroll-and-jam 501 // and those after. 502 503 LoopAttributes BeforeJam, AfterJam; 504 505 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel; 506 507 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth; 508 BeforeJam.VectorizeScalable = Attrs.VectorizeScalable; 509 BeforeJam.InterleaveCount = Attrs.InterleaveCount; 510 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable; 511 BeforeJam.DistributeEnable = Attrs.DistributeEnable; 512 BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable; 513 514 switch (Attrs.UnrollEnable) { 515 case LoopAttributes::Unspecified: 516 case LoopAttributes::Disable: 517 BeforeJam.UnrollEnable = Attrs.UnrollEnable; 518 AfterJam.UnrollEnable = Attrs.UnrollEnable; 519 break; 520 case LoopAttributes::Full: 521 BeforeJam.UnrollEnable = LoopAttributes::Full; 522 break; 523 case LoopAttributes::Enable: 524 AfterJam.UnrollEnable = LoopAttributes::Enable; 525 break; 526 } 527 528 AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable; 529 AfterJam.UnrollCount = Attrs.UnrollCount; 530 AfterJam.PipelineDisabled = Attrs.PipelineDisabled; 531 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval; 532 533 // If this loop is subject of an unroll-and-jam by the parent loop, and has 534 // an unroll-and-jam annotation itself, we have to decide whether to first 535 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The 536 // UnrollAndJam pass processes loops from inner to outer, so we apply the 537 // inner first. 538 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount; 539 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable; 540 541 // Set the inner followup metadata to process by the outer loop. Only 542 // consider the first inner loop. 543 if (!Parent->UnrollAndJamInnerFollowup) { 544 // Splitting the attributes into a BeforeJam and an AfterJam part will 545 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam) 546 // to be forwarded to the AfterJam part. We detect the situation here and 547 // add it manually. 548 SmallVector<Metadata *, 1> BeforeLoopProperties; 549 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified || 550 BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified || 551 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 || 552 BeforeJam.VectorizeScalable == LoopAttributes::Enable) 553 BeforeLoopProperties.push_back( 554 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); 555 556 bool InnerFollowupHasTransform = false; 557 SmallVector<Metadata *, 4> InnerFollowup = createMetadata( 558 AfterJam, BeforeLoopProperties, InnerFollowupHasTransform); 559 if (InnerFollowupHasTransform) 560 Parent->UnrollAndJamInnerFollowup = InnerFollowup; 561 } 562 563 CurLoopAttr = BeforeJam; 564 } 565 566 bool HasUserTransforms = false; 567 SmallVector<Metadata *, 4> Properties = 568 createMetadata(CurLoopAttr, {}, HasUserTransforms); 569 SmallVector<Metadata *, 4> Args; 570 Args.push_back(nullptr); 571 Args.append(Properties.begin(), Properties.end()); 572 LoopID = MDNode::getDistinct(Ctx, Args); 573 LoopID->replaceOperandWith(0, LoopID); 574 575 TempLoopID->replaceAllUsesWith(LoopID); 576 } 577 578 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc, 579 const llvm::DebugLoc &EndLoc) { 580 Active.emplace_back( 581 new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc, 582 Active.empty() ? nullptr : Active.back().get())); 583 // Clear the attributes so nested loops do not inherit them. 584 StagedAttrs.clear(); 585 } 586 587 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, 588 const clang::CodeGenOptions &CGOpts, 589 ArrayRef<const clang::Attr *> Attrs, 590 const llvm::DebugLoc &StartLoc, 591 const llvm::DebugLoc &EndLoc, bool MustProgress) { 592 // Identify loop hint attributes from Attrs. 593 for (const auto *Attr : Attrs) { 594 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); 595 const OpenCLUnrollHintAttr *OpenCLHint = 596 dyn_cast<OpenCLUnrollHintAttr>(Attr); 597 const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Attr); 598 // Skip non loop hint attributes 599 if (!LH && !OpenCLHint && !HLSLLoopHint) { 600 continue; 601 } 602 603 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll; 604 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable; 605 unsigned ValueInt = 1; 606 // Translate opencl_unroll_hint attribute argument to 607 // equivalent LoopHintAttr enums. 608 // OpenCL v2.0 s6.11.5: 609 // 0 - enable unroll (no argument). 610 // 1 - disable unroll. 611 // other positive integer n - unroll by n. 612 if (OpenCLHint) { 613 ValueInt = OpenCLHint->getUnrollHint(); 614 if (ValueInt == 0) { 615 State = LoopHintAttr::Enable; 616 } else if (ValueInt != 1) { 617 Option = LoopHintAttr::UnrollCount; 618 State = LoopHintAttr::Numeric; 619 } 620 } else if (HLSLLoopHint) { 621 ValueInt = HLSLLoopHint->getDirective(); 622 if (HLSLLoopHint->getSemanticSpelling() == 623 HLSLLoopHintAttr::Spelling::Microsoft_unroll) { 624 if (ValueInt == 0) 625 State = LoopHintAttr::Enable; 626 if (ValueInt > 0) { 627 Option = LoopHintAttr::UnrollCount; 628 State = LoopHintAttr::Numeric; 629 } 630 } 631 } else if (LH) { 632 auto *ValueExpr = LH->getValue(); 633 if (ValueExpr) { 634 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); 635 ValueInt = ValueAPS.getSExtValue(); 636 } 637 638 Option = LH->getOption(); 639 State = LH->getState(); 640 } 641 switch (State) { 642 case LoopHintAttr::Disable: 643 switch (Option) { 644 case LoopHintAttr::Vectorize: 645 // Disable vectorization by specifying a width of 1. 646 setVectorizeWidth(1); 647 setVectorizeScalable(LoopAttributes::Unspecified); 648 break; 649 case LoopHintAttr::Interleave: 650 // Disable interleaving by speciyfing a count of 1. 651 setInterleaveCount(1); 652 break; 653 case LoopHintAttr::Unroll: 654 setUnrollState(LoopAttributes::Disable); 655 break; 656 case LoopHintAttr::UnrollAndJam: 657 setUnrollAndJamState(LoopAttributes::Disable); 658 break; 659 case LoopHintAttr::VectorizePredicate: 660 setVectorizePredicateState(LoopAttributes::Disable); 661 break; 662 case LoopHintAttr::Distribute: 663 setDistributeState(false); 664 break; 665 case LoopHintAttr::PipelineDisabled: 666 setPipelineDisabled(true); 667 break; 668 case LoopHintAttr::UnrollCount: 669 case LoopHintAttr::UnrollAndJamCount: 670 case LoopHintAttr::VectorizeWidth: 671 case LoopHintAttr::InterleaveCount: 672 case LoopHintAttr::PipelineInitiationInterval: 673 llvm_unreachable("Options cannot be disabled."); 674 break; 675 } 676 break; 677 case LoopHintAttr::Enable: 678 switch (Option) { 679 case LoopHintAttr::Vectorize: 680 case LoopHintAttr::Interleave: 681 setVectorizeEnable(true); 682 break; 683 case LoopHintAttr::Unroll: 684 setUnrollState(LoopAttributes::Enable); 685 break; 686 case LoopHintAttr::UnrollAndJam: 687 setUnrollAndJamState(LoopAttributes::Enable); 688 break; 689 case LoopHintAttr::VectorizePredicate: 690 setVectorizePredicateState(LoopAttributes::Enable); 691 break; 692 case LoopHintAttr::Distribute: 693 setDistributeState(true); 694 break; 695 case LoopHintAttr::UnrollCount: 696 case LoopHintAttr::UnrollAndJamCount: 697 case LoopHintAttr::VectorizeWidth: 698 case LoopHintAttr::InterleaveCount: 699 case LoopHintAttr::PipelineDisabled: 700 case LoopHintAttr::PipelineInitiationInterval: 701 llvm_unreachable("Options cannot enabled."); 702 break; 703 } 704 break; 705 case LoopHintAttr::AssumeSafety: 706 switch (Option) { 707 case LoopHintAttr::Vectorize: 708 case LoopHintAttr::Interleave: 709 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. 710 setParallel(true); 711 setVectorizeEnable(true); 712 break; 713 case LoopHintAttr::Unroll: 714 case LoopHintAttr::UnrollAndJam: 715 case LoopHintAttr::VectorizePredicate: 716 case LoopHintAttr::UnrollCount: 717 case LoopHintAttr::UnrollAndJamCount: 718 case LoopHintAttr::VectorizeWidth: 719 case LoopHintAttr::InterleaveCount: 720 case LoopHintAttr::Distribute: 721 case LoopHintAttr::PipelineDisabled: 722 case LoopHintAttr::PipelineInitiationInterval: 723 llvm_unreachable("Options cannot be used to assume mem safety."); 724 break; 725 } 726 break; 727 case LoopHintAttr::Full: 728 switch (Option) { 729 case LoopHintAttr::Unroll: 730 setUnrollState(LoopAttributes::Full); 731 break; 732 case LoopHintAttr::UnrollAndJam: 733 setUnrollAndJamState(LoopAttributes::Full); 734 break; 735 case LoopHintAttr::Vectorize: 736 case LoopHintAttr::Interleave: 737 case LoopHintAttr::UnrollCount: 738 case LoopHintAttr::UnrollAndJamCount: 739 case LoopHintAttr::VectorizeWidth: 740 case LoopHintAttr::InterleaveCount: 741 case LoopHintAttr::Distribute: 742 case LoopHintAttr::PipelineDisabled: 743 case LoopHintAttr::PipelineInitiationInterval: 744 case LoopHintAttr::VectorizePredicate: 745 llvm_unreachable("Options cannot be used with 'full' hint."); 746 break; 747 } 748 break; 749 case LoopHintAttr::FixedWidth: 750 case LoopHintAttr::ScalableWidth: 751 switch (Option) { 752 case LoopHintAttr::VectorizeWidth: 753 setVectorizeScalable(State == LoopHintAttr::ScalableWidth 754 ? LoopAttributes::Enable 755 : LoopAttributes::Disable); 756 if (LH->getValue()) 757 setVectorizeWidth(ValueInt); 758 break; 759 default: 760 llvm_unreachable("Options cannot be used with 'scalable' hint."); 761 break; 762 } 763 break; 764 case LoopHintAttr::Numeric: 765 switch (Option) { 766 case LoopHintAttr::InterleaveCount: 767 setInterleaveCount(ValueInt); 768 break; 769 case LoopHintAttr::UnrollCount: 770 setUnrollCount(ValueInt); 771 break; 772 case LoopHintAttr::UnrollAndJamCount: 773 setUnrollAndJamCount(ValueInt); 774 break; 775 case LoopHintAttr::PipelineInitiationInterval: 776 setPipelineInitiationInterval(ValueInt); 777 break; 778 case LoopHintAttr::Unroll: 779 case LoopHintAttr::UnrollAndJam: 780 case LoopHintAttr::VectorizePredicate: 781 case LoopHintAttr::Vectorize: 782 case LoopHintAttr::VectorizeWidth: 783 case LoopHintAttr::Interleave: 784 case LoopHintAttr::Distribute: 785 case LoopHintAttr::PipelineDisabled: 786 llvm_unreachable("Options cannot be assigned a value."); 787 break; 788 } 789 break; 790 } 791 } 792 793 // Identify loop attribute 'code_align' from Attrs. 794 // For attribute code_align: 795 // n - 'llvm.loop.align i32 n' metadata will be emitted. 796 if (const auto *CodeAlign = getSpecificAttr<CodeAlignAttr>(Attrs)) { 797 const auto *CE = cast<ConstantExpr>(CodeAlign->getAlignment()); 798 llvm::APSInt ArgVal = CE->getResultAsAPSInt(); 799 setCodeAlign(ArgVal.getSExtValue()); 800 } 801 802 setMustProgress(MustProgress); 803 804 if (CGOpts.OptimizationLevel > 0) 805 // Disable unrolling for the loop, if unrolling is disabled (via 806 // -fno-unroll-loops) and no pragmas override the decision. 807 if (!CGOpts.UnrollLoops && 808 (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified && 809 StagedAttrs.UnrollCount == 0)) 810 setUnrollState(LoopAttributes::Disable); 811 812 /// Stage the attributes. 813 push(Header, StartLoc, EndLoc); 814 } 815 816 void LoopInfoStack::pop() { 817 assert(!Active.empty() && "No active loops to pop"); 818 Active.back()->finish(); 819 Active.pop_back(); 820 } 821 822 void LoopInfoStack::InsertHelper(Instruction *I) const { 823 if (I->mayReadOrWriteMemory()) { 824 SmallVector<Metadata *, 4> AccessGroups; 825 for (const auto &AL : Active) { 826 // Here we assume that every loop that has an access group is parallel. 827 if (MDNode *Group = AL->getAccessGroup()) 828 AccessGroups.push_back(Group); 829 } 830 MDNode *UnionMD = nullptr; 831 if (AccessGroups.size() == 1) 832 UnionMD = cast<MDNode>(AccessGroups[0]); 833 else if (AccessGroups.size() >= 2) 834 UnionMD = MDNode::get(I->getContext(), AccessGroups); 835 I->setMetadata("llvm.access.group", UnionMD); 836 } 837 838 if (!hasInfo()) 839 return; 840 841 const LoopInfo &L = getInfo(); 842 if (!L.getLoopID()) 843 return; 844 845 if (I->isTerminator()) { 846 for (BasicBlock *Succ : successors(I)) 847 if (Succ == L.getHeader()) { 848 I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID()); 849 break; 850 } 851 return; 852 } 853 } 854