1 //===- Metadata.cpp - Implement Metadata classes --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the Metadata classes. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/IR/Metadata.h" 14 #include "LLVMContextImpl.h" 15 #include "MetadataImpl.h" 16 #include "llvm/ADT/APFloat.h" 17 #include "llvm/ADT/APInt.h" 18 #include "llvm/ADT/ArrayRef.h" 19 #include "llvm/ADT/DenseSet.h" 20 #include "llvm/ADT/None.h" 21 #include "llvm/ADT/STLExtras.h" 22 #include "llvm/ADT/SetVector.h" 23 #include "llvm/ADT/SmallPtrSet.h" 24 #include "llvm/ADT/SmallSet.h" 25 #include "llvm/ADT/SmallVector.h" 26 #include "llvm/ADT/StringMap.h" 27 #include "llvm/ADT/StringRef.h" 28 #include "llvm/ADT/Twine.h" 29 #include "llvm/IR/Argument.h" 30 #include "llvm/IR/BasicBlock.h" 31 #include "llvm/IR/Constant.h" 32 #include "llvm/IR/ConstantRange.h" 33 #include "llvm/IR/Constants.h" 34 #include "llvm/IR/DebugInfoMetadata.h" 35 #include "llvm/IR/DebugLoc.h" 36 #include "llvm/IR/Function.h" 37 #include "llvm/IR/GlobalObject.h" 38 #include "llvm/IR/GlobalVariable.h" 39 #include "llvm/IR/Instruction.h" 40 #include "llvm/IR/LLVMContext.h" 41 #include "llvm/IR/MDBuilder.h" 42 #include "llvm/IR/Module.h" 43 #include "llvm/IR/TrackingMDRef.h" 44 #include "llvm/IR/Type.h" 45 #include "llvm/IR/Value.h" 46 #include "llvm/Support/Casting.h" 47 #include "llvm/Support/ErrorHandling.h" 48 #include "llvm/Support/MathExtras.h" 49 #include <algorithm> 50 #include <cassert> 51 #include <cstddef> 52 #include <cstdint> 53 #include <type_traits> 54 #include <utility> 55 #include <vector> 56 57 using namespace llvm; 58 59 MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD) 60 : Value(Ty, MetadataAsValueVal), MD(MD) { 61 track(); 62 } 63 64 MetadataAsValue::~MetadataAsValue() { 65 getType()->getContext().pImpl->MetadataAsValues.erase(MD); 66 untrack(); 67 } 68 69 /// Canonicalize metadata arguments to intrinsics. 70 /// 71 /// To support bitcode upgrades (and assembly semantic sugar) for \a 72 /// MetadataAsValue, we need to canonicalize certain metadata. 73 /// 74 /// - nullptr is replaced by an empty MDNode. 75 /// - An MDNode with a single null operand is replaced by an empty MDNode. 76 /// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped. 77 /// 78 /// This maintains readability of bitcode from when metadata was a type of 79 /// value, and these bridges were unnecessary. 80 static Metadata *canonicalizeMetadataForValue(LLVMContext &Context, 81 Metadata *MD) { 82 if (!MD) 83 // !{} 84 return MDNode::get(Context, None); 85 86 // Return early if this isn't a single-operand MDNode. 87 auto *N = dyn_cast<MDNode>(MD); 88 if (!N || N->getNumOperands() != 1) 89 return MD; 90 91 if (!N->getOperand(0)) 92 // !{} 93 return MDNode::get(Context, None); 94 95 if (auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(0))) 96 // Look through the MDNode. 97 return C; 98 99 return MD; 100 } 101 102 MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) { 103 MD = canonicalizeMetadataForValue(Context, MD); 104 auto *&Entry = Context.pImpl->MetadataAsValues[MD]; 105 if (!Entry) 106 Entry = new MetadataAsValue(Type::getMetadataTy(Context), MD); 107 return Entry; 108 } 109 110 MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context, 111 Metadata *MD) { 112 MD = canonicalizeMetadataForValue(Context, MD); 113 auto &Store = Context.pImpl->MetadataAsValues; 114 return Store.lookup(MD); 115 } 116 117 void MetadataAsValue::handleChangedMetadata(Metadata *MD) { 118 LLVMContext &Context = getContext(); 119 MD = canonicalizeMetadataForValue(Context, MD); 120 auto &Store = Context.pImpl->MetadataAsValues; 121 122 // Stop tracking the old metadata. 123 Store.erase(this->MD); 124 untrack(); 125 this->MD = nullptr; 126 127 // Start tracking MD, or RAUW if necessary. 128 auto *&Entry = Store[MD]; 129 if (Entry) { 130 replaceAllUsesWith(Entry); 131 delete this; 132 return; 133 } 134 135 this->MD = MD; 136 track(); 137 Entry = this; 138 } 139 140 void MetadataAsValue::track() { 141 if (MD) 142 MetadataTracking::track(&MD, *MD, *this); 143 } 144 145 void MetadataAsValue::untrack() { 146 if (MD) 147 MetadataTracking::untrack(MD); 148 } 149 150 bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) { 151 assert(Ref && "Expected live reference"); 152 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) && 153 "Reference without owner must be direct"); 154 if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) { 155 R->addRef(Ref, Owner); 156 return true; 157 } 158 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) { 159 assert(!PH->Use && "Placeholders can only be used once"); 160 assert(!Owner && "Unexpected callback to owner"); 161 PH->Use = static_cast<Metadata **>(Ref); 162 return true; 163 } 164 return false; 165 } 166 167 void MetadataTracking::untrack(void *Ref, Metadata &MD) { 168 assert(Ref && "Expected live reference"); 169 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) 170 R->dropRef(Ref); 171 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) 172 PH->Use = nullptr; 173 } 174 175 bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) { 176 assert(Ref && "Expected live reference"); 177 assert(New && "Expected live reference"); 178 assert(Ref != New && "Expected change"); 179 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) { 180 R->moveRef(Ref, New, MD); 181 return true; 182 } 183 assert(!isa<DistinctMDOperandPlaceholder>(MD) && 184 "Unexpected move of an MDOperand"); 185 assert(!isReplaceable(MD) && 186 "Expected un-replaceable metadata, since we didn't move a reference"); 187 return false; 188 } 189 190 bool MetadataTracking::isReplaceable(const Metadata &MD) { 191 return ReplaceableMetadataImpl::isReplaceable(MD); 192 } 193 194 SmallVector<Metadata *> ReplaceableMetadataImpl::getAllArgListUsers() { 195 SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID; 196 for (auto Pair : UseMap) { 197 OwnerTy Owner = Pair.second.first; 198 if (!Owner.is<Metadata *>()) 199 continue; 200 Metadata *OwnerMD = Owner.get<Metadata *>(); 201 if (OwnerMD->getMetadataID() == Metadata::DIArgListKind) 202 MDUsersWithID.push_back(&UseMap[Pair.first]); 203 } 204 llvm::sort(MDUsersWithID, [](auto UserA, auto UserB) { 205 return UserA->second < UserB->second; 206 }); 207 SmallVector<Metadata *> MDUsers; 208 for (auto UserWithID : MDUsersWithID) 209 MDUsers.push_back(UserWithID->first.get<Metadata *>()); 210 return MDUsers; 211 } 212 213 void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) { 214 bool WasInserted = 215 UseMap.insert(std::make_pair(Ref, std::make_pair(Owner, NextIndex))) 216 .second; 217 (void)WasInserted; 218 assert(WasInserted && "Expected to add a reference"); 219 220 ++NextIndex; 221 assert(NextIndex != 0 && "Unexpected overflow"); 222 } 223 224 void ReplaceableMetadataImpl::dropRef(void *Ref) { 225 bool WasErased = UseMap.erase(Ref); 226 (void)WasErased; 227 assert(WasErased && "Expected to drop a reference"); 228 } 229 230 void ReplaceableMetadataImpl::moveRef(void *Ref, void *New, 231 const Metadata &MD) { 232 auto I = UseMap.find(Ref); 233 assert(I != UseMap.end() && "Expected to move a reference"); 234 auto OwnerAndIndex = I->second; 235 UseMap.erase(I); 236 bool WasInserted = UseMap.insert(std::make_pair(New, OwnerAndIndex)).second; 237 (void)WasInserted; 238 assert(WasInserted && "Expected to add a reference"); 239 240 // Check that the references are direct if there's no owner. 241 (void)MD; 242 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) && 243 "Reference without owner must be direct"); 244 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) && 245 "Reference without owner must be direct"); 246 } 247 248 void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) { 249 if (UseMap.empty()) 250 return; 251 252 // Copy out uses since UseMap will get touched below. 253 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; 254 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); 255 llvm::sort(Uses, [](const UseTy &L, const UseTy &R) { 256 return L.second.second < R.second.second; 257 }); 258 for (const auto &Pair : Uses) { 259 // Check that this Ref hasn't disappeared after RAUW (when updating a 260 // previous Ref). 261 if (!UseMap.count(Pair.first)) 262 continue; 263 264 OwnerTy Owner = Pair.second.first; 265 if (!Owner) { 266 // Update unowned tracking references directly. 267 Metadata *&Ref = *static_cast<Metadata **>(Pair.first); 268 Ref = MD; 269 if (MD) 270 MetadataTracking::track(Ref); 271 UseMap.erase(Pair.first); 272 continue; 273 } 274 275 // Check for MetadataAsValue. 276 if (Owner.is<MetadataAsValue *>()) { 277 Owner.get<MetadataAsValue *>()->handleChangedMetadata(MD); 278 continue; 279 } 280 281 // There's a Metadata owner -- dispatch. 282 Metadata *OwnerMD = Owner.get<Metadata *>(); 283 switch (OwnerMD->getMetadataID()) { 284 #define HANDLE_METADATA_LEAF(CLASS) \ 285 case Metadata::CLASS##Kind: \ 286 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \ 287 continue; 288 #include "llvm/IR/Metadata.def" 289 default: 290 llvm_unreachable("Invalid metadata subclass"); 291 } 292 } 293 assert(UseMap.empty() && "Expected all uses to be replaced"); 294 } 295 296 void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) { 297 if (UseMap.empty()) 298 return; 299 300 if (!ResolveUsers) { 301 UseMap.clear(); 302 return; 303 } 304 305 // Copy out uses since UseMap could get touched below. 306 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; 307 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); 308 llvm::sort(Uses, [](const UseTy &L, const UseTy &R) { 309 return L.second.second < R.second.second; 310 }); 311 UseMap.clear(); 312 for (const auto &Pair : Uses) { 313 auto Owner = Pair.second.first; 314 if (!Owner) 315 continue; 316 if (Owner.is<MetadataAsValue *>()) 317 continue; 318 319 // Resolve MDNodes that point at this. 320 auto *OwnerMD = dyn_cast<MDNode>(Owner.get<Metadata *>()); 321 if (!OwnerMD) 322 continue; 323 if (OwnerMD->isResolved()) 324 continue; 325 OwnerMD->decrementUnresolvedOperandCount(); 326 } 327 } 328 329 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) { 330 if (auto *N = dyn_cast<MDNode>(&MD)) 331 return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses(); 332 return dyn_cast<ValueAsMetadata>(&MD); 333 } 334 335 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) { 336 if (auto *N = dyn_cast<MDNode>(&MD)) 337 return N->isResolved() ? nullptr : N->Context.getReplaceableUses(); 338 return dyn_cast<ValueAsMetadata>(&MD); 339 } 340 341 bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) { 342 if (auto *N = dyn_cast<MDNode>(&MD)) 343 return !N->isResolved(); 344 return isa<ValueAsMetadata>(&MD); 345 } 346 347 static DISubprogram *getLocalFunctionMetadata(Value *V) { 348 assert(V && "Expected value"); 349 if (auto *A = dyn_cast<Argument>(V)) { 350 if (auto *Fn = A->getParent()) 351 return Fn->getSubprogram(); 352 return nullptr; 353 } 354 355 if (BasicBlock *BB = cast<Instruction>(V)->getParent()) { 356 if (auto *Fn = BB->getParent()) 357 return Fn->getSubprogram(); 358 return nullptr; 359 } 360 361 return nullptr; 362 } 363 364 ValueAsMetadata *ValueAsMetadata::get(Value *V) { 365 assert(V && "Unexpected null Value"); 366 367 auto &Context = V->getContext(); 368 auto *&Entry = Context.pImpl->ValuesAsMetadata[V]; 369 if (!Entry) { 370 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) && 371 "Expected constant or function-local value"); 372 assert(!V->IsUsedByMD && "Expected this to be the only metadata use"); 373 V->IsUsedByMD = true; 374 if (auto *C = dyn_cast<Constant>(V)) 375 Entry = new ConstantAsMetadata(C); 376 else 377 Entry = new LocalAsMetadata(V); 378 } 379 380 return Entry; 381 } 382 383 ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) { 384 assert(V && "Unexpected null Value"); 385 return V->getContext().pImpl->ValuesAsMetadata.lookup(V); 386 } 387 388 void ValueAsMetadata::handleDeletion(Value *V) { 389 assert(V && "Expected valid value"); 390 391 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata; 392 auto I = Store.find(V); 393 if (I == Store.end()) 394 return; 395 396 // Remove old entry from the map. 397 ValueAsMetadata *MD = I->second; 398 assert(MD && "Expected valid metadata"); 399 assert(MD->getValue() == V && "Expected valid mapping"); 400 Store.erase(I); 401 402 // Delete the metadata. 403 MD->replaceAllUsesWith(nullptr); 404 delete MD; 405 } 406 407 void ValueAsMetadata::handleRAUW(Value *From, Value *To) { 408 assert(From && "Expected valid value"); 409 assert(To && "Expected valid value"); 410 assert(From != To && "Expected changed value"); 411 assert(From->getType() == To->getType() && "Unexpected type change"); 412 413 LLVMContext &Context = From->getType()->getContext(); 414 auto &Store = Context.pImpl->ValuesAsMetadata; 415 auto I = Store.find(From); 416 if (I == Store.end()) { 417 assert(!From->IsUsedByMD && "Expected From not to be used by metadata"); 418 return; 419 } 420 421 // Remove old entry from the map. 422 assert(From->IsUsedByMD && "Expected From to be used by metadata"); 423 From->IsUsedByMD = false; 424 ValueAsMetadata *MD = I->second; 425 assert(MD && "Expected valid metadata"); 426 assert(MD->getValue() == From && "Expected valid mapping"); 427 Store.erase(I); 428 429 if (isa<LocalAsMetadata>(MD)) { 430 if (auto *C = dyn_cast<Constant>(To)) { 431 // Local became a constant. 432 MD->replaceAllUsesWith(ConstantAsMetadata::get(C)); 433 delete MD; 434 return; 435 } 436 if (getLocalFunctionMetadata(From) && getLocalFunctionMetadata(To) && 437 getLocalFunctionMetadata(From) != getLocalFunctionMetadata(To)) { 438 // DISubprogram changed. 439 MD->replaceAllUsesWith(nullptr); 440 delete MD; 441 return; 442 } 443 } else if (!isa<Constant>(To)) { 444 // Changed to function-local value. 445 MD->replaceAllUsesWith(nullptr); 446 delete MD; 447 return; 448 } 449 450 auto *&Entry = Store[To]; 451 if (Entry) { 452 // The target already exists. 453 MD->replaceAllUsesWith(Entry); 454 delete MD; 455 return; 456 } 457 458 // Update MD in place (and update the map entry). 459 assert(!To->IsUsedByMD && "Expected this to be the only metadata use"); 460 To->IsUsedByMD = true; 461 MD->V = To; 462 Entry = MD; 463 } 464 465 //===----------------------------------------------------------------------===// 466 // MDString implementation. 467 // 468 469 MDString *MDString::get(LLVMContext &Context, StringRef Str) { 470 auto &Store = Context.pImpl->MDStringCache; 471 auto I = Store.try_emplace(Str); 472 auto &MapEntry = I.first->getValue(); 473 if (!I.second) 474 return &MapEntry; 475 MapEntry.Entry = &*I.first; 476 return &MapEntry; 477 } 478 479 StringRef MDString::getString() const { 480 assert(Entry && "Expected to find string map entry"); 481 return Entry->first(); 482 } 483 484 //===----------------------------------------------------------------------===// 485 // MDNode implementation. 486 // 487 488 // Assert that the MDNode types will not be unaligned by the objects 489 // prepended to them. 490 #define HANDLE_MDNODE_LEAF(CLASS) \ 491 static_assert( \ 492 alignof(uint64_t) >= alignof(CLASS), \ 493 "Alignment is insufficient after objects prepended to " #CLASS); 494 #include "llvm/IR/Metadata.def" 495 496 void *MDNode::operator new(size_t Size, unsigned NumOps) { 497 size_t OpSize = NumOps * sizeof(MDOperand); 498 // uint64_t is the most aligned type we need support (ensured by static_assert 499 // above) 500 OpSize = alignTo(OpSize, alignof(uint64_t)); 501 void *Ptr = reinterpret_cast<char *>(::operator new(OpSize + Size)) + OpSize; 502 MDOperand *O = static_cast<MDOperand *>(Ptr); 503 for (MDOperand *E = O - NumOps; O != E; --O) 504 (void)new (O - 1) MDOperand; 505 return Ptr; 506 } 507 508 // Repress memory sanitization, due to use-after-destroy by operator 509 // delete. Bug report 24578 identifies this issue. 510 LLVM_NO_SANITIZE_MEMORY_ATTRIBUTE void MDNode::operator delete(void *Mem) { 511 MDNode *N = static_cast<MDNode *>(Mem); 512 size_t OpSize = N->NumOperands * sizeof(MDOperand); 513 OpSize = alignTo(OpSize, alignof(uint64_t)); 514 515 MDOperand *O = static_cast<MDOperand *>(Mem); 516 for (MDOperand *E = O - N->NumOperands; O != E; --O) 517 (O - 1)->~MDOperand(); 518 ::operator delete(reinterpret_cast<char *>(Mem) - OpSize); 519 } 520 521 MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage, 522 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2) 523 : Metadata(ID, Storage), NumOperands(Ops1.size() + Ops2.size()), 524 NumUnresolved(0), Context(Context) { 525 unsigned Op = 0; 526 for (Metadata *MD : Ops1) 527 setOperand(Op++, MD); 528 for (Metadata *MD : Ops2) 529 setOperand(Op++, MD); 530 531 if (!isUniqued()) 532 return; 533 534 // Count the unresolved operands. If there are any, RAUW support will be 535 // added lazily on first reference. 536 countUnresolvedOperands(); 537 } 538 539 TempMDNode MDNode::clone() const { 540 switch (getMetadataID()) { 541 default: 542 llvm_unreachable("Invalid MDNode subclass"); 543 #define HANDLE_MDNODE_LEAF(CLASS) \ 544 case CLASS##Kind: \ 545 return cast<CLASS>(this)->cloneImpl(); 546 #include "llvm/IR/Metadata.def" 547 } 548 } 549 550 static bool isOperandUnresolved(Metadata *Op) { 551 if (auto *N = dyn_cast_or_null<MDNode>(Op)) 552 return !N->isResolved(); 553 return false; 554 } 555 556 void MDNode::countUnresolvedOperands() { 557 assert(NumUnresolved == 0 && "Expected unresolved ops to be uncounted"); 558 assert(isUniqued() && "Expected this to be uniqued"); 559 NumUnresolved = count_if(operands(), isOperandUnresolved); 560 } 561 562 void MDNode::makeUniqued() { 563 assert(isTemporary() && "Expected this to be temporary"); 564 assert(!isResolved() && "Expected this to be unresolved"); 565 566 // Enable uniquing callbacks. 567 for (auto &Op : mutable_operands()) 568 Op.reset(Op.get(), this); 569 570 // Make this 'uniqued'. 571 Storage = Uniqued; 572 countUnresolvedOperands(); 573 if (!NumUnresolved) { 574 dropReplaceableUses(); 575 assert(isResolved() && "Expected this to be resolved"); 576 } 577 578 assert(isUniqued() && "Expected this to be uniqued"); 579 } 580 581 void MDNode::makeDistinct() { 582 assert(isTemporary() && "Expected this to be temporary"); 583 assert(!isResolved() && "Expected this to be unresolved"); 584 585 // Drop RAUW support and store as a distinct node. 586 dropReplaceableUses(); 587 storeDistinctInContext(); 588 589 assert(isDistinct() && "Expected this to be distinct"); 590 assert(isResolved() && "Expected this to be resolved"); 591 } 592 593 void MDNode::resolve() { 594 assert(isUniqued() && "Expected this to be uniqued"); 595 assert(!isResolved() && "Expected this to be unresolved"); 596 597 NumUnresolved = 0; 598 dropReplaceableUses(); 599 600 assert(isResolved() && "Expected this to be resolved"); 601 } 602 603 void MDNode::dropReplaceableUses() { 604 assert(!NumUnresolved && "Unexpected unresolved operand"); 605 606 // Drop any RAUW support. 607 if (Context.hasReplaceableUses()) 608 Context.takeReplaceableUses()->resolveAllUses(); 609 } 610 611 void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) { 612 assert(isUniqued() && "Expected this to be uniqued"); 613 assert(NumUnresolved != 0 && "Expected unresolved operands"); 614 615 // Check if an operand was resolved. 616 if (!isOperandUnresolved(Old)) { 617 if (isOperandUnresolved(New)) 618 // An operand was un-resolved! 619 ++NumUnresolved; 620 } else if (!isOperandUnresolved(New)) 621 decrementUnresolvedOperandCount(); 622 } 623 624 void MDNode::decrementUnresolvedOperandCount() { 625 assert(!isResolved() && "Expected this to be unresolved"); 626 if (isTemporary()) 627 return; 628 629 assert(isUniqued() && "Expected this to be uniqued"); 630 if (--NumUnresolved) 631 return; 632 633 // Last unresolved operand has just been resolved. 634 dropReplaceableUses(); 635 assert(isResolved() && "Expected this to become resolved"); 636 } 637 638 void MDNode::resolveCycles() { 639 if (isResolved()) 640 return; 641 642 // Resolve this node immediately. 643 resolve(); 644 645 // Resolve all operands. 646 for (const auto &Op : operands()) { 647 auto *N = dyn_cast_or_null<MDNode>(Op); 648 if (!N) 649 continue; 650 651 assert(!N->isTemporary() && 652 "Expected all forward declarations to be resolved"); 653 if (!N->isResolved()) 654 N->resolveCycles(); 655 } 656 } 657 658 static bool hasSelfReference(MDNode *N) { 659 return llvm::is_contained(N->operands(), N); 660 } 661 662 MDNode *MDNode::replaceWithPermanentImpl() { 663 switch (getMetadataID()) { 664 default: 665 // If this type isn't uniquable, replace with a distinct node. 666 return replaceWithDistinctImpl(); 667 668 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 669 case CLASS##Kind: \ 670 break; 671 #include "llvm/IR/Metadata.def" 672 } 673 674 // Even if this type is uniquable, self-references have to be distinct. 675 if (hasSelfReference(this)) 676 return replaceWithDistinctImpl(); 677 return replaceWithUniquedImpl(); 678 } 679 680 MDNode *MDNode::replaceWithUniquedImpl() { 681 // Try to uniquify in place. 682 MDNode *UniquedNode = uniquify(); 683 684 if (UniquedNode == this) { 685 makeUniqued(); 686 return this; 687 } 688 689 // Collision, so RAUW instead. 690 replaceAllUsesWith(UniquedNode); 691 deleteAsSubclass(); 692 return UniquedNode; 693 } 694 695 MDNode *MDNode::replaceWithDistinctImpl() { 696 makeDistinct(); 697 return this; 698 } 699 700 void MDTuple::recalculateHash() { 701 setHash(MDTupleInfo::KeyTy::calculateHash(this)); 702 } 703 704 void MDNode::dropAllReferences() { 705 for (unsigned I = 0, E = NumOperands; I != E; ++I) 706 setOperand(I, nullptr); 707 if (Context.hasReplaceableUses()) { 708 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false); 709 (void)Context.takeReplaceableUses(); 710 } 711 } 712 713 void MDNode::handleChangedOperand(void *Ref, Metadata *New) { 714 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin(); 715 assert(Op < getNumOperands() && "Expected valid operand"); 716 717 if (!isUniqued()) { 718 // This node is not uniqued. Just set the operand and be done with it. 719 setOperand(Op, New); 720 return; 721 } 722 723 // This node is uniqued. 724 eraseFromStore(); 725 726 Metadata *Old = getOperand(Op); 727 setOperand(Op, New); 728 729 // Drop uniquing for self-reference cycles and deleted constants. 730 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Old))) { 731 if (!isResolved()) 732 resolve(); 733 storeDistinctInContext(); 734 return; 735 } 736 737 // Re-unique the node. 738 auto *Uniqued = uniquify(); 739 if (Uniqued == this) { 740 if (!isResolved()) 741 resolveAfterOperandChange(Old, New); 742 return; 743 } 744 745 // Collision. 746 if (!isResolved()) { 747 // Still unresolved, so RAUW. 748 // 749 // First, clear out all operands to prevent any recursion (similar to 750 // dropAllReferences(), but we still need the use-list). 751 for (unsigned O = 0, E = getNumOperands(); O != E; ++O) 752 setOperand(O, nullptr); 753 if (Context.hasReplaceableUses()) 754 Context.getReplaceableUses()->replaceAllUsesWith(Uniqued); 755 deleteAsSubclass(); 756 return; 757 } 758 759 // Store in non-uniqued form if RAUW isn't possible. 760 storeDistinctInContext(); 761 } 762 763 void MDNode::deleteAsSubclass() { 764 switch (getMetadataID()) { 765 default: 766 llvm_unreachable("Invalid subclass of MDNode"); 767 #define HANDLE_MDNODE_LEAF(CLASS) \ 768 case CLASS##Kind: \ 769 delete cast<CLASS>(this); \ 770 break; 771 #include "llvm/IR/Metadata.def" 772 } 773 } 774 775 template <class T, class InfoT> 776 static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) { 777 if (T *U = getUniqued(Store, N)) 778 return U; 779 780 Store.insert(N); 781 return N; 782 } 783 784 template <class NodeTy> struct MDNode::HasCachedHash { 785 using Yes = char[1]; 786 using No = char[2]; 787 template <class U, U Val> struct SFINAE {}; 788 789 template <class U> 790 static Yes &check(SFINAE<void (U::*)(unsigned), &U::setHash> *); 791 template <class U> static No &check(...); 792 793 static const bool value = sizeof(check<NodeTy>(nullptr)) == sizeof(Yes); 794 }; 795 796 MDNode *MDNode::uniquify() { 797 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node"); 798 799 // Try to insert into uniquing store. 800 switch (getMetadataID()) { 801 default: 802 llvm_unreachable("Invalid or non-uniquable subclass of MDNode"); 803 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 804 case CLASS##Kind: { \ 805 CLASS *SubclassThis = cast<CLASS>(this); \ 806 std::integral_constant<bool, HasCachedHash<CLASS>::value> \ 807 ShouldRecalculateHash; \ 808 dispatchRecalculateHash(SubclassThis, ShouldRecalculateHash); \ 809 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \ 810 } 811 #include "llvm/IR/Metadata.def" 812 } 813 } 814 815 void MDNode::eraseFromStore() { 816 switch (getMetadataID()) { 817 default: 818 llvm_unreachable("Invalid or non-uniquable subclass of MDNode"); 819 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 820 case CLASS##Kind: \ 821 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \ 822 break; 823 #include "llvm/IR/Metadata.def" 824 } 825 } 826 827 MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs, 828 StorageType Storage, bool ShouldCreate) { 829 unsigned Hash = 0; 830 if (Storage == Uniqued) { 831 MDTupleInfo::KeyTy Key(MDs); 832 if (auto *N = getUniqued(Context.pImpl->MDTuples, Key)) 833 return N; 834 if (!ShouldCreate) 835 return nullptr; 836 Hash = Key.getHash(); 837 } else { 838 assert(ShouldCreate && "Expected non-uniqued nodes to always be created"); 839 } 840 841 return storeImpl(new (MDs.size()) MDTuple(Context, Storage, Hash, MDs), 842 Storage, Context.pImpl->MDTuples); 843 } 844 845 void MDNode::deleteTemporary(MDNode *N) { 846 assert(N->isTemporary() && "Expected temporary node"); 847 N->replaceAllUsesWith(nullptr); 848 N->deleteAsSubclass(); 849 } 850 851 void MDNode::storeDistinctInContext() { 852 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses"); 853 assert(!NumUnresolved && "Unexpected unresolved nodes"); 854 Storage = Distinct; 855 assert(isResolved() && "Expected this to be resolved"); 856 857 // Reset the hash. 858 switch (getMetadataID()) { 859 default: 860 llvm_unreachable("Invalid subclass of MDNode"); 861 #define HANDLE_MDNODE_LEAF(CLASS) \ 862 case CLASS##Kind: { \ 863 std::integral_constant<bool, HasCachedHash<CLASS>::value> ShouldResetHash; \ 864 dispatchResetHash(cast<CLASS>(this), ShouldResetHash); \ 865 break; \ 866 } 867 #include "llvm/IR/Metadata.def" 868 } 869 870 getContext().pImpl->DistinctMDNodes.push_back(this); 871 } 872 873 void MDNode::replaceOperandWith(unsigned I, Metadata *New) { 874 if (getOperand(I) == New) 875 return; 876 877 if (!isUniqued()) { 878 setOperand(I, New); 879 return; 880 } 881 882 handleChangedOperand(mutable_begin() + I, New); 883 } 884 885 void MDNode::setOperand(unsigned I, Metadata *New) { 886 assert(I < NumOperands); 887 mutable_begin()[I].reset(New, isUniqued() ? this : nullptr); 888 } 889 890 /// Get a node or a self-reference that looks like it. 891 /// 892 /// Special handling for finding self-references, for use by \a 893 /// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from 894 /// when self-referencing nodes were still uniqued. If the first operand has 895 /// the same operands as \c Ops, return the first operand instead. 896 static MDNode *getOrSelfReference(LLVMContext &Context, 897 ArrayRef<Metadata *> Ops) { 898 if (!Ops.empty()) 899 if (MDNode *N = dyn_cast_or_null<MDNode>(Ops[0])) 900 if (N->getNumOperands() == Ops.size() && N == N->getOperand(0)) { 901 for (unsigned I = 1, E = Ops.size(); I != E; ++I) 902 if (Ops[I] != N->getOperand(I)) 903 return MDNode::get(Context, Ops); 904 return N; 905 } 906 907 return MDNode::get(Context, Ops); 908 } 909 910 MDNode *MDNode::concatenate(MDNode *A, MDNode *B) { 911 if (!A) 912 return B; 913 if (!B) 914 return A; 915 916 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); 917 MDs.insert(B->op_begin(), B->op_end()); 918 919 // FIXME: This preserves long-standing behaviour, but is it really the right 920 // behaviour? Or was that an unintended side-effect of node uniquing? 921 return getOrSelfReference(A->getContext(), MDs.getArrayRef()); 922 } 923 924 MDNode *MDNode::intersect(MDNode *A, MDNode *B) { 925 if (!A || !B) 926 return nullptr; 927 928 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); 929 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end()); 930 MDs.remove_if([&](Metadata *MD) { return !BSet.count(MD); }); 931 932 // FIXME: This preserves long-standing behaviour, but is it really the right 933 // behaviour? Or was that an unintended side-effect of node uniquing? 934 return getOrSelfReference(A->getContext(), MDs.getArrayRef()); 935 } 936 937 MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) { 938 if (!A || !B) 939 return nullptr; 940 941 // Take the intersection of domains then union the scopes 942 // within those domains 943 SmallPtrSet<const MDNode *, 16> ADomains; 944 SmallPtrSet<const MDNode *, 16> IntersectDomains; 945 SmallSetVector<Metadata *, 4> MDs; 946 for (const MDOperand &MDOp : A->operands()) 947 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 948 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 949 ADomains.insert(Domain); 950 951 for (const MDOperand &MDOp : B->operands()) 952 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 953 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 954 if (ADomains.contains(Domain)) { 955 IntersectDomains.insert(Domain); 956 MDs.insert(MDOp); 957 } 958 959 for (const MDOperand &MDOp : A->operands()) 960 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 961 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 962 if (IntersectDomains.contains(Domain)) 963 MDs.insert(MDOp); 964 965 return MDs.empty() ? nullptr 966 : getOrSelfReference(A->getContext(), MDs.getArrayRef()); 967 } 968 969 MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) { 970 if (!A || !B) 971 return nullptr; 972 973 APFloat AVal = mdconst::extract<ConstantFP>(A->getOperand(0))->getValueAPF(); 974 APFloat BVal = mdconst::extract<ConstantFP>(B->getOperand(0))->getValueAPF(); 975 if (AVal < BVal) 976 return A; 977 return B; 978 } 979 980 static bool isContiguous(const ConstantRange &A, const ConstantRange &B) { 981 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper(); 982 } 983 984 static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) { 985 return !A.intersectWith(B).isEmptySet() || isContiguous(A, B); 986 } 987 988 static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints, 989 ConstantInt *Low, ConstantInt *High) { 990 ConstantRange NewRange(Low->getValue(), High->getValue()); 991 unsigned Size = EndPoints.size(); 992 APInt LB = EndPoints[Size - 2]->getValue(); 993 APInt LE = EndPoints[Size - 1]->getValue(); 994 ConstantRange LastRange(LB, LE); 995 if (canBeMerged(NewRange, LastRange)) { 996 ConstantRange Union = LastRange.unionWith(NewRange); 997 Type *Ty = High->getType(); 998 EndPoints[Size - 2] = 999 cast<ConstantInt>(ConstantInt::get(Ty, Union.getLower())); 1000 EndPoints[Size - 1] = 1001 cast<ConstantInt>(ConstantInt::get(Ty, Union.getUpper())); 1002 return true; 1003 } 1004 return false; 1005 } 1006 1007 static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints, 1008 ConstantInt *Low, ConstantInt *High) { 1009 if (!EndPoints.empty()) 1010 if (tryMergeRange(EndPoints, Low, High)) 1011 return; 1012 1013 EndPoints.push_back(Low); 1014 EndPoints.push_back(High); 1015 } 1016 1017 MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) { 1018 // Given two ranges, we want to compute the union of the ranges. This 1019 // is slightly complicated by having to combine the intervals and merge 1020 // the ones that overlap. 1021 1022 if (!A || !B) 1023 return nullptr; 1024 1025 if (A == B) 1026 return A; 1027 1028 // First, walk both lists in order of the lower boundary of each interval. 1029 // At each step, try to merge the new interval to the last one we adedd. 1030 SmallVector<ConstantInt *, 4> EndPoints; 1031 int AI = 0; 1032 int BI = 0; 1033 int AN = A->getNumOperands() / 2; 1034 int BN = B->getNumOperands() / 2; 1035 while (AI < AN && BI < BN) { 1036 ConstantInt *ALow = mdconst::extract<ConstantInt>(A->getOperand(2 * AI)); 1037 ConstantInt *BLow = mdconst::extract<ConstantInt>(B->getOperand(2 * BI)); 1038 1039 if (ALow->getValue().slt(BLow->getValue())) { 1040 addRange(EndPoints, ALow, 1041 mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1))); 1042 ++AI; 1043 } else { 1044 addRange(EndPoints, BLow, 1045 mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1))); 1046 ++BI; 1047 } 1048 } 1049 while (AI < AN) { 1050 addRange(EndPoints, mdconst::extract<ConstantInt>(A->getOperand(2 * AI)), 1051 mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1))); 1052 ++AI; 1053 } 1054 while (BI < BN) { 1055 addRange(EndPoints, mdconst::extract<ConstantInt>(B->getOperand(2 * BI)), 1056 mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1))); 1057 ++BI; 1058 } 1059 1060 // If we have more than 2 ranges (4 endpoints) we have to try to merge 1061 // the last and first ones. 1062 unsigned Size = EndPoints.size(); 1063 if (Size > 4) { 1064 ConstantInt *FB = EndPoints[0]; 1065 ConstantInt *FE = EndPoints[1]; 1066 if (tryMergeRange(EndPoints, FB, FE)) { 1067 for (unsigned i = 0; i < Size - 2; ++i) { 1068 EndPoints[i] = EndPoints[i + 2]; 1069 } 1070 EndPoints.resize(Size - 2); 1071 } 1072 } 1073 1074 // If in the end we have a single range, it is possible that it is now the 1075 // full range. Just drop the metadata in that case. 1076 if (EndPoints.size() == 2) { 1077 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue()); 1078 if (Range.isFullSet()) 1079 return nullptr; 1080 } 1081 1082 SmallVector<Metadata *, 4> MDs; 1083 MDs.reserve(EndPoints.size()); 1084 for (auto *I : EndPoints) 1085 MDs.push_back(ConstantAsMetadata::get(I)); 1086 return MDNode::get(A->getContext(), MDs); 1087 } 1088 1089 MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) { 1090 if (!A || !B) 1091 return nullptr; 1092 1093 ConstantInt *AVal = mdconst::extract<ConstantInt>(A->getOperand(0)); 1094 ConstantInt *BVal = mdconst::extract<ConstantInt>(B->getOperand(0)); 1095 if (AVal->getZExtValue() < BVal->getZExtValue()) 1096 return A; 1097 return B; 1098 } 1099 1100 //===----------------------------------------------------------------------===// 1101 // NamedMDNode implementation. 1102 // 1103 1104 static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) { 1105 return *(SmallVector<TrackingMDRef, 4> *)Operands; 1106 } 1107 1108 NamedMDNode::NamedMDNode(const Twine &N) 1109 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {} 1110 1111 NamedMDNode::~NamedMDNode() { 1112 dropAllReferences(); 1113 delete &getNMDOps(Operands); 1114 } 1115 1116 unsigned NamedMDNode::getNumOperands() const { 1117 return (unsigned)getNMDOps(Operands).size(); 1118 } 1119 1120 MDNode *NamedMDNode::getOperand(unsigned i) const { 1121 assert(i < getNumOperands() && "Invalid Operand number!"); 1122 auto *N = getNMDOps(Operands)[i].get(); 1123 return cast_or_null<MDNode>(N); 1124 } 1125 1126 void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(M); } 1127 1128 void NamedMDNode::setOperand(unsigned I, MDNode *New) { 1129 assert(I < getNumOperands() && "Invalid operand number"); 1130 getNMDOps(Operands)[I].reset(New); 1131 } 1132 1133 void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(this); } 1134 1135 void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); } 1136 1137 StringRef NamedMDNode::getName() const { return StringRef(Name); } 1138 1139 //===----------------------------------------------------------------------===// 1140 // Instruction Metadata method implementations. 1141 // 1142 1143 MDNode *MDAttachments::lookup(unsigned ID) const { 1144 for (const auto &A : Attachments) 1145 if (A.MDKind == ID) 1146 return A.Node; 1147 return nullptr; 1148 } 1149 1150 void MDAttachments::get(unsigned ID, SmallVectorImpl<MDNode *> &Result) const { 1151 for (const auto &A : Attachments) 1152 if (A.MDKind == ID) 1153 Result.push_back(A.Node); 1154 } 1155 1156 void MDAttachments::getAll( 1157 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1158 for (const auto &A : Attachments) 1159 Result.emplace_back(A.MDKind, A.Node); 1160 1161 // Sort the resulting array so it is stable with respect to metadata IDs. We 1162 // need to preserve the original insertion order though. 1163 if (Result.size() > 1) 1164 llvm::stable_sort(Result, less_first()); 1165 } 1166 1167 void MDAttachments::set(unsigned ID, MDNode *MD) { 1168 erase(ID); 1169 if (MD) 1170 insert(ID, *MD); 1171 } 1172 1173 void MDAttachments::insert(unsigned ID, MDNode &MD) { 1174 Attachments.push_back({ID, TrackingMDNodeRef(&MD)}); 1175 } 1176 1177 bool MDAttachments::erase(unsigned ID) { 1178 if (empty()) 1179 return false; 1180 1181 // Common case is one value. 1182 if (Attachments.size() == 1 && Attachments.back().MDKind == ID) { 1183 Attachments.pop_back(); 1184 return true; 1185 } 1186 1187 auto OldSize = Attachments.size(); 1188 llvm::erase_if(Attachments, 1189 [ID](const Attachment &A) { return A.MDKind == ID; }); 1190 return OldSize != Attachments.size(); 1191 } 1192 1193 MDNode *Value::getMetadata(unsigned KindID) const { 1194 if (!hasMetadata()) 1195 return nullptr; 1196 const auto &Info = getContext().pImpl->ValueMetadata[this]; 1197 assert(!Info.empty() && "bit out of sync with hash table"); 1198 return Info.lookup(KindID); 1199 } 1200 1201 MDNode *Value::getMetadata(StringRef Kind) const { 1202 if (!hasMetadata()) 1203 return nullptr; 1204 const auto &Info = getContext().pImpl->ValueMetadata[this]; 1205 assert(!Info.empty() && "bit out of sync with hash table"); 1206 return Info.lookup(getContext().getMDKindID(Kind)); 1207 } 1208 1209 void Value::getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const { 1210 if (hasMetadata()) 1211 getContext().pImpl->ValueMetadata[this].get(KindID, MDs); 1212 } 1213 1214 void Value::getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const { 1215 if (hasMetadata()) 1216 getMetadata(getContext().getMDKindID(Kind), MDs); 1217 } 1218 1219 void Value::getAllMetadata( 1220 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const { 1221 if (hasMetadata()) { 1222 assert(getContext().pImpl->ValueMetadata.count(this) && 1223 "bit out of sync with hash table"); 1224 const auto &Info = getContext().pImpl->ValueMetadata.find(this)->second; 1225 assert(!Info.empty() && "Shouldn't have called this"); 1226 Info.getAll(MDs); 1227 } 1228 } 1229 1230 void Value::setMetadata(unsigned KindID, MDNode *Node) { 1231 assert(isa<Instruction>(this) || isa<GlobalObject>(this)); 1232 1233 // Handle the case when we're adding/updating metadata on a value. 1234 if (Node) { 1235 auto &Info = getContext().pImpl->ValueMetadata[this]; 1236 assert(!Info.empty() == HasMetadata && "bit out of sync with hash table"); 1237 if (Info.empty()) 1238 HasMetadata = true; 1239 Info.set(KindID, Node); 1240 return; 1241 } 1242 1243 // Otherwise, we're removing metadata from an instruction. 1244 assert((HasMetadata == (getContext().pImpl->ValueMetadata.count(this) > 0)) && 1245 "bit out of sync with hash table"); 1246 if (!HasMetadata) 1247 return; // Nothing to remove! 1248 auto &Info = getContext().pImpl->ValueMetadata[this]; 1249 1250 // Handle removal of an existing value. 1251 Info.erase(KindID); 1252 if (!Info.empty()) 1253 return; 1254 getContext().pImpl->ValueMetadata.erase(this); 1255 HasMetadata = false; 1256 } 1257 1258 void Value::setMetadata(StringRef Kind, MDNode *Node) { 1259 if (!Node && !HasMetadata) 1260 return; 1261 setMetadata(getContext().getMDKindID(Kind), Node); 1262 } 1263 1264 void Value::addMetadata(unsigned KindID, MDNode &MD) { 1265 assert(isa<Instruction>(this) || isa<GlobalObject>(this)); 1266 if (!HasMetadata) 1267 HasMetadata = true; 1268 getContext().pImpl->ValueMetadata[this].insert(KindID, MD); 1269 } 1270 1271 void Value::addMetadata(StringRef Kind, MDNode &MD) { 1272 addMetadata(getContext().getMDKindID(Kind), MD); 1273 } 1274 1275 bool Value::eraseMetadata(unsigned KindID) { 1276 // Nothing to unset. 1277 if (!HasMetadata) 1278 return false; 1279 1280 auto &Store = getContext().pImpl->ValueMetadata[this]; 1281 bool Changed = Store.erase(KindID); 1282 if (Store.empty()) 1283 clearMetadata(); 1284 return Changed; 1285 } 1286 1287 void Value::clearMetadata() { 1288 if (!HasMetadata) 1289 return; 1290 assert(getContext().pImpl->ValueMetadata.count(this) && 1291 "bit out of sync with hash table"); 1292 getContext().pImpl->ValueMetadata.erase(this); 1293 HasMetadata = false; 1294 } 1295 1296 void Instruction::setMetadata(StringRef Kind, MDNode *Node) { 1297 if (!Node && !hasMetadata()) 1298 return; 1299 setMetadata(getContext().getMDKindID(Kind), Node); 1300 } 1301 1302 MDNode *Instruction::getMetadataImpl(StringRef Kind) const { 1303 return getMetadataImpl(getContext().getMDKindID(Kind)); 1304 } 1305 1306 void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) { 1307 if (!Value::hasMetadata()) 1308 return; // Nothing to remove! 1309 1310 if (KnownIDs.empty()) { 1311 // Just drop our entry at the store. 1312 clearMetadata(); 1313 return; 1314 } 1315 1316 SmallSet<unsigned, 4> KnownSet; 1317 KnownSet.insert(KnownIDs.begin(), KnownIDs.end()); 1318 1319 auto &MetadataStore = getContext().pImpl->ValueMetadata; 1320 auto &Info = MetadataStore[this]; 1321 assert(!Info.empty() && "bit out of sync with hash table"); 1322 Info.remove_if([&KnownSet](const MDAttachments::Attachment &I) { 1323 return !KnownSet.count(I.MDKind); 1324 }); 1325 1326 if (Info.empty()) { 1327 // Drop our entry at the store. 1328 clearMetadata(); 1329 } 1330 } 1331 1332 void Instruction::setMetadata(unsigned KindID, MDNode *Node) { 1333 if (!Node && !hasMetadata()) 1334 return; 1335 1336 // Handle 'dbg' as a special case since it is not stored in the hash table. 1337 if (KindID == LLVMContext::MD_dbg) { 1338 DbgLoc = DebugLoc(Node); 1339 return; 1340 } 1341 1342 Value::setMetadata(KindID, Node); 1343 } 1344 1345 void Instruction::addAnnotationMetadata(StringRef Name) { 1346 MDBuilder MDB(getContext()); 1347 1348 auto *Existing = getMetadata(LLVMContext::MD_annotation); 1349 SmallVector<Metadata *, 4> Names; 1350 bool AppendName = true; 1351 if (Existing) { 1352 auto *Tuple = cast<MDTuple>(Existing); 1353 for (auto &N : Tuple->operands()) { 1354 if (cast<MDString>(N.get())->getString() == Name) 1355 AppendName = false; 1356 Names.push_back(N.get()); 1357 } 1358 } 1359 if (AppendName) 1360 Names.push_back(MDB.createString(Name)); 1361 1362 MDNode *MD = MDTuple::get(getContext(), Names); 1363 setMetadata(LLVMContext::MD_annotation, MD); 1364 } 1365 1366 AAMDNodes Instruction::getAAMetadata() const { 1367 AAMDNodes Result; 1368 Result.TBAA = getMetadata(LLVMContext::MD_tbaa); 1369 Result.TBAAStruct = getMetadata(LLVMContext::MD_tbaa_struct); 1370 Result.Scope = getMetadata(LLVMContext::MD_alias_scope); 1371 Result.NoAlias = getMetadata(LLVMContext::MD_noalias); 1372 return Result; 1373 } 1374 1375 void Instruction::setAAMetadata(const AAMDNodes &N) { 1376 setMetadata(LLVMContext::MD_tbaa, N.TBAA); 1377 setMetadata(LLVMContext::MD_tbaa_struct, N.TBAAStruct); 1378 setMetadata(LLVMContext::MD_alias_scope, N.Scope); 1379 setMetadata(LLVMContext::MD_noalias, N.NoAlias); 1380 } 1381 1382 MDNode *Instruction::getMetadataImpl(unsigned KindID) const { 1383 // Handle 'dbg' as a special case since it is not stored in the hash table. 1384 if (KindID == LLVMContext::MD_dbg) 1385 return DbgLoc.getAsMDNode(); 1386 return Value::getMetadata(KindID); 1387 } 1388 1389 void Instruction::getAllMetadataImpl( 1390 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1391 Result.clear(); 1392 1393 // Handle 'dbg' as a special case since it is not stored in the hash table. 1394 if (DbgLoc) { 1395 Result.push_back( 1396 std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode())); 1397 } 1398 Value::getAllMetadata(Result); 1399 } 1400 1401 bool Instruction::extractProfMetadata(uint64_t &TrueVal, 1402 uint64_t &FalseVal) const { 1403 assert( 1404 (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select) && 1405 "Looking for branch weights on something besides branch or select"); 1406 1407 auto *ProfileData = getMetadata(LLVMContext::MD_prof); 1408 if (!ProfileData || ProfileData->getNumOperands() != 3) 1409 return false; 1410 1411 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); 1412 if (!ProfDataName || !ProfDataName->getString().equals("branch_weights")) 1413 return false; 1414 1415 auto *CITrue = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1)); 1416 auto *CIFalse = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2)); 1417 if (!CITrue || !CIFalse) 1418 return false; 1419 1420 TrueVal = CITrue->getValue().getZExtValue(); 1421 FalseVal = CIFalse->getValue().getZExtValue(); 1422 1423 return true; 1424 } 1425 1426 bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const { 1427 assert( 1428 (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select || 1429 getOpcode() == Instruction::Call || getOpcode() == Instruction::Invoke || 1430 getOpcode() == Instruction::IndirectBr || 1431 getOpcode() == Instruction::Switch) && 1432 "Looking for branch weights on something besides branch"); 1433 1434 TotalVal = 0; 1435 auto *ProfileData = getMetadata(LLVMContext::MD_prof); 1436 if (!ProfileData) 1437 return false; 1438 1439 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); 1440 if (!ProfDataName) 1441 return false; 1442 1443 if (ProfDataName->getString().equals("branch_weights")) { 1444 TotalVal = 0; 1445 for (unsigned i = 1; i < ProfileData->getNumOperands(); i++) { 1446 auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i)); 1447 if (!V) 1448 return false; 1449 TotalVal += V->getValue().getZExtValue(); 1450 } 1451 return true; 1452 } else if (ProfDataName->getString().equals("VP") && 1453 ProfileData->getNumOperands() > 3) { 1454 TotalVal = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2)) 1455 ->getValue() 1456 .getZExtValue(); 1457 return true; 1458 } 1459 return false; 1460 } 1461 1462 void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) { 1463 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs; 1464 Other->getAllMetadata(MDs); 1465 for (auto &MD : MDs) { 1466 // We need to adjust the type metadata offset. 1467 if (Offset != 0 && MD.first == LLVMContext::MD_type) { 1468 auto *OffsetConst = cast<ConstantInt>( 1469 cast<ConstantAsMetadata>(MD.second->getOperand(0))->getValue()); 1470 Metadata *TypeId = MD.second->getOperand(1); 1471 auto *NewOffsetMD = ConstantAsMetadata::get(ConstantInt::get( 1472 OffsetConst->getType(), OffsetConst->getValue() + Offset)); 1473 addMetadata(LLVMContext::MD_type, 1474 *MDNode::get(getContext(), {NewOffsetMD, TypeId})); 1475 continue; 1476 } 1477 // If an offset adjustment was specified we need to modify the DIExpression 1478 // to prepend the adjustment: 1479 // !DIExpression(DW_OP_plus, Offset, [original expr]) 1480 auto *Attachment = MD.second; 1481 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) { 1482 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Attachment); 1483 DIExpression *E = nullptr; 1484 if (!GV) { 1485 auto *GVE = cast<DIGlobalVariableExpression>(Attachment); 1486 GV = GVE->getVariable(); 1487 E = GVE->getExpression(); 1488 } 1489 ArrayRef<uint64_t> OrigElements; 1490 if (E) 1491 OrigElements = E->getElements(); 1492 std::vector<uint64_t> Elements(OrigElements.size() + 2); 1493 Elements[0] = dwarf::DW_OP_plus_uconst; 1494 Elements[1] = Offset; 1495 llvm::copy(OrigElements, Elements.begin() + 2); 1496 E = DIExpression::get(getContext(), Elements); 1497 Attachment = DIGlobalVariableExpression::get(getContext(), GV, E); 1498 } 1499 addMetadata(MD.first, *Attachment); 1500 } 1501 } 1502 1503 void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) { 1504 addMetadata( 1505 LLVMContext::MD_type, 1506 *MDTuple::get(getContext(), 1507 {ConstantAsMetadata::get(ConstantInt::get( 1508 Type::getInt64Ty(getContext()), Offset)), 1509 TypeID})); 1510 } 1511 1512 void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) { 1513 // Remove any existing vcall visibility metadata first in case we are 1514 // updating. 1515 eraseMetadata(LLVMContext::MD_vcall_visibility); 1516 addMetadata(LLVMContext::MD_vcall_visibility, 1517 *MDNode::get(getContext(), 1518 {ConstantAsMetadata::get(ConstantInt::get( 1519 Type::getInt64Ty(getContext()), Visibility))})); 1520 } 1521 1522 GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const { 1523 if (MDNode *MD = getMetadata(LLVMContext::MD_vcall_visibility)) { 1524 uint64_t Val = cast<ConstantInt>( 1525 cast<ConstantAsMetadata>(MD->getOperand(0))->getValue()) 1526 ->getZExtValue(); 1527 assert(Val <= 2 && "unknown vcall visibility!"); 1528 return (VCallVisibility)Val; 1529 } 1530 return VCallVisibility::VCallVisibilityPublic; 1531 } 1532 1533 void Function::setSubprogram(DISubprogram *SP) { 1534 setMetadata(LLVMContext::MD_dbg, SP); 1535 } 1536 1537 DISubprogram *Function::getSubprogram() const { 1538 return cast_or_null<DISubprogram>(getMetadata(LLVMContext::MD_dbg)); 1539 } 1540 1541 bool Function::isDebugInfoForProfiling() const { 1542 if (DISubprogram *SP = getSubprogram()) { 1543 if (DICompileUnit *CU = SP->getUnit()) { 1544 return CU->getDebugInfoForProfiling(); 1545 } 1546 } 1547 return false; 1548 } 1549 1550 void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) { 1551 addMetadata(LLVMContext::MD_dbg, *GV); 1552 } 1553 1554 void GlobalVariable::getDebugInfo( 1555 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const { 1556 SmallVector<MDNode *, 1> MDs; 1557 getMetadata(LLVMContext::MD_dbg, MDs); 1558 for (MDNode *MD : MDs) 1559 GVs.push_back(cast<DIGlobalVariableExpression>(MD)); 1560 } 1561