1 //===- Metadata.cpp - Implement Metadata classes --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the Metadata classes. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/IR/Metadata.h" 14 #include "LLVMContextImpl.h" 15 #include "MetadataImpl.h" 16 #include "llvm/ADT/APFloat.h" 17 #include "llvm/ADT/APInt.h" 18 #include "llvm/ADT/ArrayRef.h" 19 #include "llvm/ADT/DenseSet.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/SetVector.h" 22 #include "llvm/ADT/SmallPtrSet.h" 23 #include "llvm/ADT/SmallSet.h" 24 #include "llvm/ADT/SmallVector.h" 25 #include "llvm/ADT/StringMap.h" 26 #include "llvm/ADT/StringRef.h" 27 #include "llvm/ADT/Twine.h" 28 #include "llvm/IR/Argument.h" 29 #include "llvm/IR/BasicBlock.h" 30 #include "llvm/IR/Constant.h" 31 #include "llvm/IR/ConstantRange.h" 32 #include "llvm/IR/Constants.h" 33 #include "llvm/IR/DebugInfoMetadata.h" 34 #include "llvm/IR/DebugLoc.h" 35 #include "llvm/IR/Function.h" 36 #include "llvm/IR/GlobalObject.h" 37 #include "llvm/IR/GlobalVariable.h" 38 #include "llvm/IR/Instruction.h" 39 #include "llvm/IR/LLVMContext.h" 40 #include "llvm/IR/MDBuilder.h" 41 #include "llvm/IR/Module.h" 42 #include "llvm/IR/ProfDataUtils.h" 43 #include "llvm/IR/TrackingMDRef.h" 44 #include "llvm/IR/Type.h" 45 #include "llvm/IR/Value.h" 46 #include "llvm/Support/Casting.h" 47 #include "llvm/Support/ErrorHandling.h" 48 #include "llvm/Support/MathExtras.h" 49 #include <algorithm> 50 #include <cassert> 51 #include <cstddef> 52 #include <cstdint> 53 #include <type_traits> 54 #include <utility> 55 #include <vector> 56 57 using namespace llvm; 58 59 MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD) 60 : Value(Ty, MetadataAsValueVal), MD(MD) { 61 track(); 62 } 63 64 MetadataAsValue::~MetadataAsValue() { 65 getType()->getContext().pImpl->MetadataAsValues.erase(MD); 66 untrack(); 67 } 68 69 /// Canonicalize metadata arguments to intrinsics. 70 /// 71 /// To support bitcode upgrades (and assembly semantic sugar) for \a 72 /// MetadataAsValue, we need to canonicalize certain metadata. 73 /// 74 /// - nullptr is replaced by an empty MDNode. 75 /// - An MDNode with a single null operand is replaced by an empty MDNode. 76 /// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped. 77 /// 78 /// This maintains readability of bitcode from when metadata was a type of 79 /// value, and these bridges were unnecessary. 80 static Metadata *canonicalizeMetadataForValue(LLVMContext &Context, 81 Metadata *MD) { 82 if (!MD) 83 // !{} 84 return MDNode::get(Context, std::nullopt); 85 86 // Return early if this isn't a single-operand MDNode. 87 auto *N = dyn_cast<MDNode>(MD); 88 if (!N || N->getNumOperands() != 1) 89 return MD; 90 91 if (!N->getOperand(0)) 92 // !{} 93 return MDNode::get(Context, std::nullopt); 94 95 if (auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(0))) 96 // Look through the MDNode. 97 return C; 98 99 return MD; 100 } 101 102 MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) { 103 MD = canonicalizeMetadataForValue(Context, MD); 104 auto *&Entry = Context.pImpl->MetadataAsValues[MD]; 105 if (!Entry) 106 Entry = new MetadataAsValue(Type::getMetadataTy(Context), MD); 107 return Entry; 108 } 109 110 MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context, 111 Metadata *MD) { 112 MD = canonicalizeMetadataForValue(Context, MD); 113 auto &Store = Context.pImpl->MetadataAsValues; 114 return Store.lookup(MD); 115 } 116 117 void MetadataAsValue::handleChangedMetadata(Metadata *MD) { 118 LLVMContext &Context = getContext(); 119 MD = canonicalizeMetadataForValue(Context, MD); 120 auto &Store = Context.pImpl->MetadataAsValues; 121 122 // Stop tracking the old metadata. 123 Store.erase(this->MD); 124 untrack(); 125 this->MD = nullptr; 126 127 // Start tracking MD, or RAUW if necessary. 128 auto *&Entry = Store[MD]; 129 if (Entry) { 130 replaceAllUsesWith(Entry); 131 delete this; 132 return; 133 } 134 135 this->MD = MD; 136 track(); 137 Entry = this; 138 } 139 140 void MetadataAsValue::track() { 141 if (MD) 142 MetadataTracking::track(&MD, *MD, *this); 143 } 144 145 void MetadataAsValue::untrack() { 146 if (MD) 147 MetadataTracking::untrack(MD); 148 } 149 150 bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) { 151 assert(Ref && "Expected live reference"); 152 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) && 153 "Reference without owner must be direct"); 154 if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) { 155 R->addRef(Ref, Owner); 156 return true; 157 } 158 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) { 159 assert(!PH->Use && "Placeholders can only be used once"); 160 assert(!Owner && "Unexpected callback to owner"); 161 PH->Use = static_cast<Metadata **>(Ref); 162 return true; 163 } 164 return false; 165 } 166 167 void MetadataTracking::untrack(void *Ref, Metadata &MD) { 168 assert(Ref && "Expected live reference"); 169 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) 170 R->dropRef(Ref); 171 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) 172 PH->Use = nullptr; 173 } 174 175 bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) { 176 assert(Ref && "Expected live reference"); 177 assert(New && "Expected live reference"); 178 assert(Ref != New && "Expected change"); 179 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) { 180 R->moveRef(Ref, New, MD); 181 return true; 182 } 183 assert(!isa<DistinctMDOperandPlaceholder>(MD) && 184 "Unexpected move of an MDOperand"); 185 assert(!isReplaceable(MD) && 186 "Expected un-replaceable metadata, since we didn't move a reference"); 187 return false; 188 } 189 190 bool MetadataTracking::isReplaceable(const Metadata &MD) { 191 return ReplaceableMetadataImpl::isReplaceable(MD); 192 } 193 194 SmallVector<Metadata *> ReplaceableMetadataImpl::getAllArgListUsers() { 195 SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID; 196 for (auto Pair : UseMap) { 197 OwnerTy Owner = Pair.second.first; 198 if (!Owner.is<Metadata *>()) 199 continue; 200 Metadata *OwnerMD = Owner.get<Metadata *>(); 201 if (OwnerMD->getMetadataID() == Metadata::DIArgListKind) 202 MDUsersWithID.push_back(&UseMap[Pair.first]); 203 } 204 llvm::sort(MDUsersWithID, [](auto UserA, auto UserB) { 205 return UserA->second < UserB->second; 206 }); 207 SmallVector<Metadata *> MDUsers; 208 for (auto *UserWithID : MDUsersWithID) 209 MDUsers.push_back(UserWithID->first.get<Metadata *>()); 210 return MDUsers; 211 } 212 213 void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) { 214 bool WasInserted = 215 UseMap.insert(std::make_pair(Ref, std::make_pair(Owner, NextIndex))) 216 .second; 217 (void)WasInserted; 218 assert(WasInserted && "Expected to add a reference"); 219 220 ++NextIndex; 221 assert(NextIndex != 0 && "Unexpected overflow"); 222 } 223 224 void ReplaceableMetadataImpl::dropRef(void *Ref) { 225 bool WasErased = UseMap.erase(Ref); 226 (void)WasErased; 227 assert(WasErased && "Expected to drop a reference"); 228 } 229 230 void ReplaceableMetadataImpl::moveRef(void *Ref, void *New, 231 const Metadata &MD) { 232 auto I = UseMap.find(Ref); 233 assert(I != UseMap.end() && "Expected to move a reference"); 234 auto OwnerAndIndex = I->second; 235 UseMap.erase(I); 236 bool WasInserted = UseMap.insert(std::make_pair(New, OwnerAndIndex)).second; 237 (void)WasInserted; 238 assert(WasInserted && "Expected to add a reference"); 239 240 // Check that the references are direct if there's no owner. 241 (void)MD; 242 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) && 243 "Reference without owner must be direct"); 244 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) && 245 "Reference without owner must be direct"); 246 } 247 248 void ReplaceableMetadataImpl::SalvageDebugInfo(const Constant &C) { 249 if (!C.isUsedByMetadata()) { 250 return; 251 } 252 253 LLVMContext &Context = C.getType()->getContext(); 254 auto &Store = Context.pImpl->ValuesAsMetadata; 255 auto I = Store.find(&C); 256 ValueAsMetadata *MD = I->second; 257 using UseTy = 258 std::pair<void *, std::pair<MetadataTracking::OwnerTy, uint64_t>>; 259 // Copy out uses and update value of Constant used by debug info metadata with undef below 260 SmallVector<UseTy, 8> Uses(MD->UseMap.begin(), MD->UseMap.end()); 261 262 for (const auto &Pair : Uses) { 263 MetadataTracking::OwnerTy Owner = Pair.second.first; 264 if (!Owner) 265 continue; 266 if (!Owner.is<Metadata *>()) 267 continue; 268 auto *OwnerMD = dyn_cast<MDNode>(Owner.get<Metadata *>()); 269 if (!OwnerMD) 270 continue; 271 if (isa<DINode>(OwnerMD)) { 272 OwnerMD->handleChangedOperand( 273 Pair.first, ValueAsMetadata::get(UndefValue::get(C.getType()))); 274 } 275 } 276 } 277 278 void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) { 279 if (UseMap.empty()) 280 return; 281 282 // Copy out uses since UseMap will get touched below. 283 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; 284 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); 285 llvm::sort(Uses, llvm::less_second()); 286 for (const auto &Pair : Uses) { 287 // Check that this Ref hasn't disappeared after RAUW (when updating a 288 // previous Ref). 289 if (!UseMap.count(Pair.first)) 290 continue; 291 292 OwnerTy Owner = Pair.second.first; 293 if (!Owner) { 294 // Update unowned tracking references directly. 295 Metadata *&Ref = *static_cast<Metadata **>(Pair.first); 296 Ref = MD; 297 if (MD) 298 MetadataTracking::track(Ref); 299 UseMap.erase(Pair.first); 300 continue; 301 } 302 303 // Check for MetadataAsValue. 304 if (Owner.is<MetadataAsValue *>()) { 305 Owner.get<MetadataAsValue *>()->handleChangedMetadata(MD); 306 continue; 307 } 308 309 // There's a Metadata owner -- dispatch. 310 Metadata *OwnerMD = Owner.get<Metadata *>(); 311 switch (OwnerMD->getMetadataID()) { 312 #define HANDLE_METADATA_LEAF(CLASS) \ 313 case Metadata::CLASS##Kind: \ 314 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \ 315 continue; 316 #include "llvm/IR/Metadata.def" 317 default: 318 llvm_unreachable("Invalid metadata subclass"); 319 } 320 } 321 assert(UseMap.empty() && "Expected all uses to be replaced"); 322 } 323 324 void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) { 325 if (UseMap.empty()) 326 return; 327 328 if (!ResolveUsers) { 329 UseMap.clear(); 330 return; 331 } 332 333 // Copy out uses since UseMap could get touched below. 334 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; 335 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); 336 llvm::sort(Uses, [](const UseTy &L, const UseTy &R) { 337 return L.second.second < R.second.second; 338 }); 339 UseMap.clear(); 340 for (const auto &Pair : Uses) { 341 auto Owner = Pair.second.first; 342 if (!Owner) 343 continue; 344 if (Owner.is<MetadataAsValue *>()) 345 continue; 346 347 // Resolve MDNodes that point at this. 348 auto *OwnerMD = dyn_cast<MDNode>(Owner.get<Metadata *>()); 349 if (!OwnerMD) 350 continue; 351 if (OwnerMD->isResolved()) 352 continue; 353 OwnerMD->decrementUnresolvedOperandCount(); 354 } 355 } 356 357 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) { 358 if (auto *N = dyn_cast<MDNode>(&MD)) 359 return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses(); 360 return dyn_cast<ValueAsMetadata>(&MD); 361 } 362 363 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) { 364 if (auto *N = dyn_cast<MDNode>(&MD)) 365 return N->isResolved() ? nullptr : N->Context.getReplaceableUses(); 366 return dyn_cast<ValueAsMetadata>(&MD); 367 } 368 369 bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) { 370 if (auto *N = dyn_cast<MDNode>(&MD)) 371 return !N->isResolved(); 372 return isa<ValueAsMetadata>(&MD); 373 } 374 375 static DISubprogram *getLocalFunctionMetadata(Value *V) { 376 assert(V && "Expected value"); 377 if (auto *A = dyn_cast<Argument>(V)) { 378 if (auto *Fn = A->getParent()) 379 return Fn->getSubprogram(); 380 return nullptr; 381 } 382 383 if (BasicBlock *BB = cast<Instruction>(V)->getParent()) { 384 if (auto *Fn = BB->getParent()) 385 return Fn->getSubprogram(); 386 return nullptr; 387 } 388 389 return nullptr; 390 } 391 392 ValueAsMetadata *ValueAsMetadata::get(Value *V) { 393 assert(V && "Unexpected null Value"); 394 395 auto &Context = V->getContext(); 396 auto *&Entry = Context.pImpl->ValuesAsMetadata[V]; 397 if (!Entry) { 398 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) && 399 "Expected constant or function-local value"); 400 assert(!V->IsUsedByMD && "Expected this to be the only metadata use"); 401 V->IsUsedByMD = true; 402 if (auto *C = dyn_cast<Constant>(V)) 403 Entry = new ConstantAsMetadata(C); 404 else 405 Entry = new LocalAsMetadata(V); 406 } 407 408 return Entry; 409 } 410 411 ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) { 412 assert(V && "Unexpected null Value"); 413 return V->getContext().pImpl->ValuesAsMetadata.lookup(V); 414 } 415 416 void ValueAsMetadata::handleDeletion(Value *V) { 417 assert(V && "Expected valid value"); 418 419 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata; 420 auto I = Store.find(V); 421 if (I == Store.end()) 422 return; 423 424 // Remove old entry from the map. 425 ValueAsMetadata *MD = I->second; 426 assert(MD && "Expected valid metadata"); 427 assert(MD->getValue() == V && "Expected valid mapping"); 428 Store.erase(I); 429 430 // Delete the metadata. 431 MD->replaceAllUsesWith(nullptr); 432 delete MD; 433 } 434 435 void ValueAsMetadata::handleRAUW(Value *From, Value *To) { 436 assert(From && "Expected valid value"); 437 assert(To && "Expected valid value"); 438 assert(From != To && "Expected changed value"); 439 assert(From->getType() == To->getType() && "Unexpected type change"); 440 441 LLVMContext &Context = From->getType()->getContext(); 442 auto &Store = Context.pImpl->ValuesAsMetadata; 443 auto I = Store.find(From); 444 if (I == Store.end()) { 445 assert(!From->IsUsedByMD && "Expected From not to be used by metadata"); 446 return; 447 } 448 449 // Remove old entry from the map. 450 assert(From->IsUsedByMD && "Expected From to be used by metadata"); 451 From->IsUsedByMD = false; 452 ValueAsMetadata *MD = I->second; 453 assert(MD && "Expected valid metadata"); 454 assert(MD->getValue() == From && "Expected valid mapping"); 455 Store.erase(I); 456 457 if (isa<LocalAsMetadata>(MD)) { 458 if (auto *C = dyn_cast<Constant>(To)) { 459 // Local became a constant. 460 MD->replaceAllUsesWith(ConstantAsMetadata::get(C)); 461 delete MD; 462 return; 463 } 464 if (getLocalFunctionMetadata(From) && getLocalFunctionMetadata(To) && 465 getLocalFunctionMetadata(From) != getLocalFunctionMetadata(To)) { 466 // DISubprogram changed. 467 MD->replaceAllUsesWith(nullptr); 468 delete MD; 469 return; 470 } 471 } else if (!isa<Constant>(To)) { 472 // Changed to function-local value. 473 MD->replaceAllUsesWith(nullptr); 474 delete MD; 475 return; 476 } 477 478 auto *&Entry = Store[To]; 479 if (Entry) { 480 // The target already exists. 481 MD->replaceAllUsesWith(Entry); 482 delete MD; 483 return; 484 } 485 486 // Update MD in place (and update the map entry). 487 assert(!To->IsUsedByMD && "Expected this to be the only metadata use"); 488 To->IsUsedByMD = true; 489 MD->V = To; 490 Entry = MD; 491 } 492 493 //===----------------------------------------------------------------------===// 494 // MDString implementation. 495 // 496 497 MDString *MDString::get(LLVMContext &Context, StringRef Str) { 498 auto &Store = Context.pImpl->MDStringCache; 499 auto I = Store.try_emplace(Str); 500 auto &MapEntry = I.first->getValue(); 501 if (!I.second) 502 return &MapEntry; 503 MapEntry.Entry = &*I.first; 504 return &MapEntry; 505 } 506 507 StringRef MDString::getString() const { 508 assert(Entry && "Expected to find string map entry"); 509 return Entry->first(); 510 } 511 512 //===----------------------------------------------------------------------===// 513 // MDNode implementation. 514 // 515 516 // Assert that the MDNode types will not be unaligned by the objects 517 // prepended to them. 518 #define HANDLE_MDNODE_LEAF(CLASS) \ 519 static_assert( \ 520 alignof(uint64_t) >= alignof(CLASS), \ 521 "Alignment is insufficient after objects prepended to " #CLASS); 522 #include "llvm/IR/Metadata.def" 523 524 void *MDNode::operator new(size_t Size, size_t NumOps, StorageType Storage) { 525 // uint64_t is the most aligned type we need support (ensured by static_assert 526 // above) 527 size_t AllocSize = 528 alignTo(Header::getAllocSize(Storage, NumOps), alignof(uint64_t)); 529 char *Mem = reinterpret_cast<char *>(::operator new(AllocSize + Size)); 530 Header *H = new (Mem + AllocSize - sizeof(Header)) Header(NumOps, Storage); 531 return reinterpret_cast<void *>(H + 1); 532 } 533 534 void MDNode::operator delete(void *N) { 535 Header *H = reinterpret_cast<Header *>(N) - 1; 536 void *Mem = H->getAllocation(); 537 H->~Header(); 538 ::operator delete(Mem); 539 } 540 541 MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage, 542 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2) 543 : Metadata(ID, Storage), Context(Context) { 544 unsigned Op = 0; 545 for (Metadata *MD : Ops1) 546 setOperand(Op++, MD); 547 for (Metadata *MD : Ops2) 548 setOperand(Op++, MD); 549 550 if (!isUniqued()) 551 return; 552 553 // Count the unresolved operands. If there are any, RAUW support will be 554 // added lazily on first reference. 555 countUnresolvedOperands(); 556 } 557 558 TempMDNode MDNode::clone() const { 559 switch (getMetadataID()) { 560 default: 561 llvm_unreachable("Invalid MDNode subclass"); 562 #define HANDLE_MDNODE_LEAF(CLASS) \ 563 case CLASS##Kind: \ 564 return cast<CLASS>(this)->cloneImpl(); 565 #include "llvm/IR/Metadata.def" 566 } 567 } 568 569 MDNode::Header::Header(size_t NumOps, StorageType Storage) { 570 IsLarge = isLarge(NumOps); 571 IsResizable = isResizable(Storage); 572 SmallSize = getSmallSize(NumOps, IsResizable, IsLarge); 573 if (IsLarge) { 574 SmallNumOps = 0; 575 new (getLargePtr()) LargeStorageVector(); 576 getLarge().resize(NumOps); 577 return; 578 } 579 SmallNumOps = NumOps; 580 MDOperand *O = reinterpret_cast<MDOperand *>(this) - SmallSize; 581 for (MDOperand *E = O + SmallSize; O != E;) 582 (void)new (O++) MDOperand(); 583 } 584 585 MDNode::Header::~Header() { 586 if (IsLarge) { 587 getLarge().~LargeStorageVector(); 588 return; 589 } 590 MDOperand *O = reinterpret_cast<MDOperand *>(this); 591 for (MDOperand *E = O - SmallSize; O != E; --O) 592 (void)(O - 1)->~MDOperand(); 593 } 594 595 void *MDNode::Header::getSmallPtr() { 596 static_assert(alignof(MDOperand) <= alignof(Header), 597 "MDOperand too strongly aligned"); 598 return reinterpret_cast<char *>(const_cast<Header *>(this)) - 599 sizeof(MDOperand) * SmallSize; 600 } 601 602 void MDNode::Header::resize(size_t NumOps) { 603 assert(IsResizable && "Node is not resizable"); 604 if (operands().size() == NumOps) 605 return; 606 607 if (IsLarge) 608 getLarge().resize(NumOps); 609 else if (NumOps <= SmallSize) 610 resizeSmall(NumOps); 611 else 612 resizeSmallToLarge(NumOps); 613 } 614 615 void MDNode::Header::resizeSmall(size_t NumOps) { 616 assert(!IsLarge && "Expected a small MDNode"); 617 assert(NumOps <= SmallSize && "NumOps too large for small resize"); 618 619 MutableArrayRef<MDOperand> ExistingOps = operands(); 620 assert(NumOps != ExistingOps.size() && "Expected a different size"); 621 622 int NumNew = (int)NumOps - (int)ExistingOps.size(); 623 MDOperand *O = ExistingOps.end(); 624 for (int I = 0, E = NumNew; I < E; ++I) 625 (O++)->reset(); 626 for (int I = 0, E = NumNew; I > E; --I) 627 (--O)->reset(); 628 SmallNumOps = NumOps; 629 assert(O == operands().end() && "Operands not (un)initialized until the end"); 630 } 631 632 void MDNode::Header::resizeSmallToLarge(size_t NumOps) { 633 assert(!IsLarge && "Expected a small MDNode"); 634 assert(NumOps > SmallSize && "Expected NumOps to be larger than allocation"); 635 LargeStorageVector NewOps; 636 NewOps.resize(NumOps); 637 llvm::move(operands(), NewOps.begin()); 638 resizeSmall(0); 639 new (getLargePtr()) LargeStorageVector(std::move(NewOps)); 640 IsLarge = true; 641 } 642 643 static bool isOperandUnresolved(Metadata *Op) { 644 if (auto *N = dyn_cast_or_null<MDNode>(Op)) 645 return !N->isResolved(); 646 return false; 647 } 648 649 void MDNode::countUnresolvedOperands() { 650 assert(getNumUnresolved() == 0 && "Expected unresolved ops to be uncounted"); 651 assert(isUniqued() && "Expected this to be uniqued"); 652 setNumUnresolved(count_if(operands(), isOperandUnresolved)); 653 } 654 655 void MDNode::makeUniqued() { 656 assert(isTemporary() && "Expected this to be temporary"); 657 assert(!isResolved() && "Expected this to be unresolved"); 658 659 // Enable uniquing callbacks. 660 for (auto &Op : mutable_operands()) 661 Op.reset(Op.get(), this); 662 663 // Make this 'uniqued'. 664 Storage = Uniqued; 665 countUnresolvedOperands(); 666 if (!getNumUnresolved()) { 667 dropReplaceableUses(); 668 assert(isResolved() && "Expected this to be resolved"); 669 } 670 671 assert(isUniqued() && "Expected this to be uniqued"); 672 } 673 674 void MDNode::makeDistinct() { 675 assert(isTemporary() && "Expected this to be temporary"); 676 assert(!isResolved() && "Expected this to be unresolved"); 677 678 // Drop RAUW support and store as a distinct node. 679 dropReplaceableUses(); 680 storeDistinctInContext(); 681 682 assert(isDistinct() && "Expected this to be distinct"); 683 assert(isResolved() && "Expected this to be resolved"); 684 } 685 686 void MDNode::resolve() { 687 assert(isUniqued() && "Expected this to be uniqued"); 688 assert(!isResolved() && "Expected this to be unresolved"); 689 690 setNumUnresolved(0); 691 dropReplaceableUses(); 692 693 assert(isResolved() && "Expected this to be resolved"); 694 } 695 696 void MDNode::dropReplaceableUses() { 697 assert(!getNumUnresolved() && "Unexpected unresolved operand"); 698 699 // Drop any RAUW support. 700 if (Context.hasReplaceableUses()) 701 Context.takeReplaceableUses()->resolveAllUses(); 702 } 703 704 void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) { 705 assert(isUniqued() && "Expected this to be uniqued"); 706 assert(getNumUnresolved() != 0 && "Expected unresolved operands"); 707 708 // Check if an operand was resolved. 709 if (!isOperandUnresolved(Old)) { 710 if (isOperandUnresolved(New)) 711 // An operand was un-resolved! 712 setNumUnresolved(getNumUnresolved() + 1); 713 } else if (!isOperandUnresolved(New)) 714 decrementUnresolvedOperandCount(); 715 } 716 717 void MDNode::decrementUnresolvedOperandCount() { 718 assert(!isResolved() && "Expected this to be unresolved"); 719 if (isTemporary()) 720 return; 721 722 assert(isUniqued() && "Expected this to be uniqued"); 723 setNumUnresolved(getNumUnresolved() - 1); 724 if (getNumUnresolved()) 725 return; 726 727 // Last unresolved operand has just been resolved. 728 dropReplaceableUses(); 729 assert(isResolved() && "Expected this to become resolved"); 730 } 731 732 void MDNode::resolveCycles() { 733 if (isResolved()) 734 return; 735 736 // Resolve this node immediately. 737 resolve(); 738 739 // Resolve all operands. 740 for (const auto &Op : operands()) { 741 auto *N = dyn_cast_or_null<MDNode>(Op); 742 if (!N) 743 continue; 744 745 assert(!N->isTemporary() && 746 "Expected all forward declarations to be resolved"); 747 if (!N->isResolved()) 748 N->resolveCycles(); 749 } 750 } 751 752 static bool hasSelfReference(MDNode *N) { 753 return llvm::is_contained(N->operands(), N); 754 } 755 756 MDNode *MDNode::replaceWithPermanentImpl() { 757 switch (getMetadataID()) { 758 default: 759 // If this type isn't uniquable, replace with a distinct node. 760 return replaceWithDistinctImpl(); 761 762 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 763 case CLASS##Kind: \ 764 break; 765 #include "llvm/IR/Metadata.def" 766 } 767 768 // Even if this type is uniquable, self-references have to be distinct. 769 if (hasSelfReference(this)) 770 return replaceWithDistinctImpl(); 771 return replaceWithUniquedImpl(); 772 } 773 774 MDNode *MDNode::replaceWithUniquedImpl() { 775 // Try to uniquify in place. 776 MDNode *UniquedNode = uniquify(); 777 778 if (UniquedNode == this) { 779 makeUniqued(); 780 return this; 781 } 782 783 // Collision, so RAUW instead. 784 replaceAllUsesWith(UniquedNode); 785 deleteAsSubclass(); 786 return UniquedNode; 787 } 788 789 MDNode *MDNode::replaceWithDistinctImpl() { 790 makeDistinct(); 791 return this; 792 } 793 794 void MDTuple::recalculateHash() { 795 setHash(MDTupleInfo::KeyTy::calculateHash(this)); 796 } 797 798 void MDNode::dropAllReferences() { 799 for (unsigned I = 0, E = getNumOperands(); I != E; ++I) 800 setOperand(I, nullptr); 801 if (Context.hasReplaceableUses()) { 802 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false); 803 (void)Context.takeReplaceableUses(); 804 } 805 } 806 807 void MDNode::handleChangedOperand(void *Ref, Metadata *New) { 808 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin(); 809 assert(Op < getNumOperands() && "Expected valid operand"); 810 811 if (!isUniqued()) { 812 // This node is not uniqued. Just set the operand and be done with it. 813 setOperand(Op, New); 814 return; 815 } 816 817 // This node is uniqued. 818 eraseFromStore(); 819 820 Metadata *Old = getOperand(Op); 821 setOperand(Op, New); 822 823 // Drop uniquing for self-reference cycles and deleted constants. 824 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Old))) { 825 if (!isResolved()) 826 resolve(); 827 storeDistinctInContext(); 828 return; 829 } 830 831 // Re-unique the node. 832 auto *Uniqued = uniquify(); 833 if (Uniqued == this) { 834 if (!isResolved()) 835 resolveAfterOperandChange(Old, New); 836 return; 837 } 838 839 // Collision. 840 if (!isResolved()) { 841 // Still unresolved, so RAUW. 842 // 843 // First, clear out all operands to prevent any recursion (similar to 844 // dropAllReferences(), but we still need the use-list). 845 for (unsigned O = 0, E = getNumOperands(); O != E; ++O) 846 setOperand(O, nullptr); 847 if (Context.hasReplaceableUses()) 848 Context.getReplaceableUses()->replaceAllUsesWith(Uniqued); 849 deleteAsSubclass(); 850 return; 851 } 852 853 // Store in non-uniqued form if RAUW isn't possible. 854 storeDistinctInContext(); 855 } 856 857 void MDNode::deleteAsSubclass() { 858 switch (getMetadataID()) { 859 default: 860 llvm_unreachable("Invalid subclass of MDNode"); 861 #define HANDLE_MDNODE_LEAF(CLASS) \ 862 case CLASS##Kind: \ 863 delete cast<CLASS>(this); \ 864 break; 865 #include "llvm/IR/Metadata.def" 866 } 867 } 868 869 template <class T, class InfoT> 870 static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) { 871 if (T *U = getUniqued(Store, N)) 872 return U; 873 874 Store.insert(N); 875 return N; 876 } 877 878 template <class NodeTy> struct MDNode::HasCachedHash { 879 using Yes = char[1]; 880 using No = char[2]; 881 template <class U, U Val> struct SFINAE {}; 882 883 template <class U> 884 static Yes &check(SFINAE<void (U::*)(unsigned), &U::setHash> *); 885 template <class U> static No &check(...); 886 887 static const bool value = sizeof(check<NodeTy>(nullptr)) == sizeof(Yes); 888 }; 889 890 MDNode *MDNode::uniquify() { 891 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node"); 892 893 // Try to insert into uniquing store. 894 switch (getMetadataID()) { 895 default: 896 llvm_unreachable("Invalid or non-uniquable subclass of MDNode"); 897 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 898 case CLASS##Kind: { \ 899 CLASS *SubclassThis = cast<CLASS>(this); \ 900 std::integral_constant<bool, HasCachedHash<CLASS>::value> \ 901 ShouldRecalculateHash; \ 902 dispatchRecalculateHash(SubclassThis, ShouldRecalculateHash); \ 903 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \ 904 } 905 #include "llvm/IR/Metadata.def" 906 } 907 } 908 909 void MDNode::eraseFromStore() { 910 switch (getMetadataID()) { 911 default: 912 llvm_unreachable("Invalid or non-uniquable subclass of MDNode"); 913 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 914 case CLASS##Kind: \ 915 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \ 916 break; 917 #include "llvm/IR/Metadata.def" 918 } 919 } 920 921 MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs, 922 StorageType Storage, bool ShouldCreate) { 923 unsigned Hash = 0; 924 if (Storage == Uniqued) { 925 MDTupleInfo::KeyTy Key(MDs); 926 if (auto *N = getUniqued(Context.pImpl->MDTuples, Key)) 927 return N; 928 if (!ShouldCreate) 929 return nullptr; 930 Hash = Key.getHash(); 931 } else { 932 assert(ShouldCreate && "Expected non-uniqued nodes to always be created"); 933 } 934 935 return storeImpl(new (MDs.size(), Storage) 936 MDTuple(Context, Storage, Hash, MDs), 937 Storage, Context.pImpl->MDTuples); 938 } 939 940 void MDNode::deleteTemporary(MDNode *N) { 941 assert(N->isTemporary() && "Expected temporary node"); 942 N->replaceAllUsesWith(nullptr); 943 N->deleteAsSubclass(); 944 } 945 946 void MDNode::storeDistinctInContext() { 947 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses"); 948 assert(!getNumUnresolved() && "Unexpected unresolved nodes"); 949 Storage = Distinct; 950 assert(isResolved() && "Expected this to be resolved"); 951 952 // Reset the hash. 953 switch (getMetadataID()) { 954 default: 955 llvm_unreachable("Invalid subclass of MDNode"); 956 #define HANDLE_MDNODE_LEAF(CLASS) \ 957 case CLASS##Kind: { \ 958 std::integral_constant<bool, HasCachedHash<CLASS>::value> ShouldResetHash; \ 959 dispatchResetHash(cast<CLASS>(this), ShouldResetHash); \ 960 break; \ 961 } 962 #include "llvm/IR/Metadata.def" 963 } 964 965 getContext().pImpl->DistinctMDNodes.push_back(this); 966 } 967 968 void MDNode::replaceOperandWith(unsigned I, Metadata *New) { 969 if (getOperand(I) == New) 970 return; 971 972 if (!isUniqued()) { 973 setOperand(I, New); 974 return; 975 } 976 977 handleChangedOperand(mutable_begin() + I, New); 978 } 979 980 void MDNode::setOperand(unsigned I, Metadata *New) { 981 assert(I < getNumOperands()); 982 mutable_begin()[I].reset(New, isUniqued() ? this : nullptr); 983 } 984 985 /// Get a node or a self-reference that looks like it. 986 /// 987 /// Special handling for finding self-references, for use by \a 988 /// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from 989 /// when self-referencing nodes were still uniqued. If the first operand has 990 /// the same operands as \c Ops, return the first operand instead. 991 static MDNode *getOrSelfReference(LLVMContext &Context, 992 ArrayRef<Metadata *> Ops) { 993 if (!Ops.empty()) 994 if (MDNode *N = dyn_cast_or_null<MDNode>(Ops[0])) 995 if (N->getNumOperands() == Ops.size() && N == N->getOperand(0)) { 996 for (unsigned I = 1, E = Ops.size(); I != E; ++I) 997 if (Ops[I] != N->getOperand(I)) 998 return MDNode::get(Context, Ops); 999 return N; 1000 } 1001 1002 return MDNode::get(Context, Ops); 1003 } 1004 1005 MDNode *MDNode::concatenate(MDNode *A, MDNode *B) { 1006 if (!A) 1007 return B; 1008 if (!B) 1009 return A; 1010 1011 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); 1012 MDs.insert(B->op_begin(), B->op_end()); 1013 1014 // FIXME: This preserves long-standing behaviour, but is it really the right 1015 // behaviour? Or was that an unintended side-effect of node uniquing? 1016 return getOrSelfReference(A->getContext(), MDs.getArrayRef()); 1017 } 1018 1019 MDNode *MDNode::intersect(MDNode *A, MDNode *B) { 1020 if (!A || !B) 1021 return nullptr; 1022 1023 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); 1024 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end()); 1025 MDs.remove_if([&](Metadata *MD) { return !BSet.count(MD); }); 1026 1027 // FIXME: This preserves long-standing behaviour, but is it really the right 1028 // behaviour? Or was that an unintended side-effect of node uniquing? 1029 return getOrSelfReference(A->getContext(), MDs.getArrayRef()); 1030 } 1031 1032 MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) { 1033 if (!A || !B) 1034 return nullptr; 1035 1036 // Take the intersection of domains then union the scopes 1037 // within those domains 1038 SmallPtrSet<const MDNode *, 16> ADomains; 1039 SmallPtrSet<const MDNode *, 16> IntersectDomains; 1040 SmallSetVector<Metadata *, 4> MDs; 1041 for (const MDOperand &MDOp : A->operands()) 1042 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 1043 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 1044 ADomains.insert(Domain); 1045 1046 for (const MDOperand &MDOp : B->operands()) 1047 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 1048 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 1049 if (ADomains.contains(Domain)) { 1050 IntersectDomains.insert(Domain); 1051 MDs.insert(MDOp); 1052 } 1053 1054 for (const MDOperand &MDOp : A->operands()) 1055 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 1056 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 1057 if (IntersectDomains.contains(Domain)) 1058 MDs.insert(MDOp); 1059 1060 return MDs.empty() ? nullptr 1061 : getOrSelfReference(A->getContext(), MDs.getArrayRef()); 1062 } 1063 1064 MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) { 1065 if (!A || !B) 1066 return nullptr; 1067 1068 APFloat AVal = mdconst::extract<ConstantFP>(A->getOperand(0))->getValueAPF(); 1069 APFloat BVal = mdconst::extract<ConstantFP>(B->getOperand(0))->getValueAPF(); 1070 if (AVal < BVal) 1071 return A; 1072 return B; 1073 } 1074 1075 static bool isContiguous(const ConstantRange &A, const ConstantRange &B) { 1076 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper(); 1077 } 1078 1079 static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) { 1080 return !A.intersectWith(B).isEmptySet() || isContiguous(A, B); 1081 } 1082 1083 static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints, 1084 ConstantInt *Low, ConstantInt *High) { 1085 ConstantRange NewRange(Low->getValue(), High->getValue()); 1086 unsigned Size = EndPoints.size(); 1087 APInt LB = EndPoints[Size - 2]->getValue(); 1088 APInt LE = EndPoints[Size - 1]->getValue(); 1089 ConstantRange LastRange(LB, LE); 1090 if (canBeMerged(NewRange, LastRange)) { 1091 ConstantRange Union = LastRange.unionWith(NewRange); 1092 Type *Ty = High->getType(); 1093 EndPoints[Size - 2] = 1094 cast<ConstantInt>(ConstantInt::get(Ty, Union.getLower())); 1095 EndPoints[Size - 1] = 1096 cast<ConstantInt>(ConstantInt::get(Ty, Union.getUpper())); 1097 return true; 1098 } 1099 return false; 1100 } 1101 1102 static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints, 1103 ConstantInt *Low, ConstantInt *High) { 1104 if (!EndPoints.empty()) 1105 if (tryMergeRange(EndPoints, Low, High)) 1106 return; 1107 1108 EndPoints.push_back(Low); 1109 EndPoints.push_back(High); 1110 } 1111 1112 MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) { 1113 // Given two ranges, we want to compute the union of the ranges. This 1114 // is slightly complicated by having to combine the intervals and merge 1115 // the ones that overlap. 1116 1117 if (!A || !B) 1118 return nullptr; 1119 1120 if (A == B) 1121 return A; 1122 1123 // First, walk both lists in order of the lower boundary of each interval. 1124 // At each step, try to merge the new interval to the last one we adedd. 1125 SmallVector<ConstantInt *, 4> EndPoints; 1126 int AI = 0; 1127 int BI = 0; 1128 int AN = A->getNumOperands() / 2; 1129 int BN = B->getNumOperands() / 2; 1130 while (AI < AN && BI < BN) { 1131 ConstantInt *ALow = mdconst::extract<ConstantInt>(A->getOperand(2 * AI)); 1132 ConstantInt *BLow = mdconst::extract<ConstantInt>(B->getOperand(2 * BI)); 1133 1134 if (ALow->getValue().slt(BLow->getValue())) { 1135 addRange(EndPoints, ALow, 1136 mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1))); 1137 ++AI; 1138 } else { 1139 addRange(EndPoints, BLow, 1140 mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1))); 1141 ++BI; 1142 } 1143 } 1144 while (AI < AN) { 1145 addRange(EndPoints, mdconst::extract<ConstantInt>(A->getOperand(2 * AI)), 1146 mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1))); 1147 ++AI; 1148 } 1149 while (BI < BN) { 1150 addRange(EndPoints, mdconst::extract<ConstantInt>(B->getOperand(2 * BI)), 1151 mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1))); 1152 ++BI; 1153 } 1154 1155 // If we have more than 2 ranges (4 endpoints) we have to try to merge 1156 // the last and first ones. 1157 unsigned Size = EndPoints.size(); 1158 if (Size > 4) { 1159 ConstantInt *FB = EndPoints[0]; 1160 ConstantInt *FE = EndPoints[1]; 1161 if (tryMergeRange(EndPoints, FB, FE)) { 1162 for (unsigned i = 0; i < Size - 2; ++i) { 1163 EndPoints[i] = EndPoints[i + 2]; 1164 } 1165 EndPoints.resize(Size - 2); 1166 } 1167 } 1168 1169 // If in the end we have a single range, it is possible that it is now the 1170 // full range. Just drop the metadata in that case. 1171 if (EndPoints.size() == 2) { 1172 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue()); 1173 if (Range.isFullSet()) 1174 return nullptr; 1175 } 1176 1177 SmallVector<Metadata *, 4> MDs; 1178 MDs.reserve(EndPoints.size()); 1179 for (auto *I : EndPoints) 1180 MDs.push_back(ConstantAsMetadata::get(I)); 1181 return MDNode::get(A->getContext(), MDs); 1182 } 1183 1184 MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) { 1185 if (!A || !B) 1186 return nullptr; 1187 1188 ConstantInt *AVal = mdconst::extract<ConstantInt>(A->getOperand(0)); 1189 ConstantInt *BVal = mdconst::extract<ConstantInt>(B->getOperand(0)); 1190 if (AVal->getZExtValue() < BVal->getZExtValue()) 1191 return A; 1192 return B; 1193 } 1194 1195 //===----------------------------------------------------------------------===// 1196 // NamedMDNode implementation. 1197 // 1198 1199 static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) { 1200 return *(SmallVector<TrackingMDRef, 4> *)Operands; 1201 } 1202 1203 NamedMDNode::NamedMDNode(const Twine &N) 1204 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {} 1205 1206 NamedMDNode::~NamedMDNode() { 1207 dropAllReferences(); 1208 delete &getNMDOps(Operands); 1209 } 1210 1211 unsigned NamedMDNode::getNumOperands() const { 1212 return (unsigned)getNMDOps(Operands).size(); 1213 } 1214 1215 MDNode *NamedMDNode::getOperand(unsigned i) const { 1216 assert(i < getNumOperands() && "Invalid Operand number!"); 1217 auto *N = getNMDOps(Operands)[i].get(); 1218 return cast_or_null<MDNode>(N); 1219 } 1220 1221 void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(M); } 1222 1223 void NamedMDNode::setOperand(unsigned I, MDNode *New) { 1224 assert(I < getNumOperands() && "Invalid operand number"); 1225 getNMDOps(Operands)[I].reset(New); 1226 } 1227 1228 void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(this); } 1229 1230 void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); } 1231 1232 StringRef NamedMDNode::getName() const { return StringRef(Name); } 1233 1234 //===----------------------------------------------------------------------===// 1235 // Instruction Metadata method implementations. 1236 // 1237 1238 MDNode *MDAttachments::lookup(unsigned ID) const { 1239 for (const auto &A : Attachments) 1240 if (A.MDKind == ID) 1241 return A.Node; 1242 return nullptr; 1243 } 1244 1245 void MDAttachments::get(unsigned ID, SmallVectorImpl<MDNode *> &Result) const { 1246 for (const auto &A : Attachments) 1247 if (A.MDKind == ID) 1248 Result.push_back(A.Node); 1249 } 1250 1251 void MDAttachments::getAll( 1252 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1253 for (const auto &A : Attachments) 1254 Result.emplace_back(A.MDKind, A.Node); 1255 1256 // Sort the resulting array so it is stable with respect to metadata IDs. We 1257 // need to preserve the original insertion order though. 1258 if (Result.size() > 1) 1259 llvm::stable_sort(Result, less_first()); 1260 } 1261 1262 void MDAttachments::set(unsigned ID, MDNode *MD) { 1263 erase(ID); 1264 if (MD) 1265 insert(ID, *MD); 1266 } 1267 1268 void MDAttachments::insert(unsigned ID, MDNode &MD) { 1269 Attachments.push_back({ID, TrackingMDNodeRef(&MD)}); 1270 } 1271 1272 bool MDAttachments::erase(unsigned ID) { 1273 if (empty()) 1274 return false; 1275 1276 // Common case is one value. 1277 if (Attachments.size() == 1 && Attachments.back().MDKind == ID) { 1278 Attachments.pop_back(); 1279 return true; 1280 } 1281 1282 auto OldSize = Attachments.size(); 1283 llvm::erase_if(Attachments, 1284 [ID](const Attachment &A) { return A.MDKind == ID; }); 1285 return OldSize != Attachments.size(); 1286 } 1287 1288 MDNode *Value::getMetadata(unsigned KindID) const { 1289 if (!hasMetadata()) 1290 return nullptr; 1291 const auto &Info = getContext().pImpl->ValueMetadata[this]; 1292 assert(!Info.empty() && "bit out of sync with hash table"); 1293 return Info.lookup(KindID); 1294 } 1295 1296 MDNode *Value::getMetadata(StringRef Kind) const { 1297 if (!hasMetadata()) 1298 return nullptr; 1299 const auto &Info = getContext().pImpl->ValueMetadata[this]; 1300 assert(!Info.empty() && "bit out of sync with hash table"); 1301 return Info.lookup(getContext().getMDKindID(Kind)); 1302 } 1303 1304 void Value::getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const { 1305 if (hasMetadata()) 1306 getContext().pImpl->ValueMetadata[this].get(KindID, MDs); 1307 } 1308 1309 void Value::getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const { 1310 if (hasMetadata()) 1311 getMetadata(getContext().getMDKindID(Kind), MDs); 1312 } 1313 1314 void Value::getAllMetadata( 1315 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const { 1316 if (hasMetadata()) { 1317 assert(getContext().pImpl->ValueMetadata.count(this) && 1318 "bit out of sync with hash table"); 1319 const auto &Info = getContext().pImpl->ValueMetadata.find(this)->second; 1320 assert(!Info.empty() && "Shouldn't have called this"); 1321 Info.getAll(MDs); 1322 } 1323 } 1324 1325 void Value::setMetadata(unsigned KindID, MDNode *Node) { 1326 assert(isa<Instruction>(this) || isa<GlobalObject>(this)); 1327 1328 // Handle the case when we're adding/updating metadata on a value. 1329 if (Node) { 1330 auto &Info = getContext().pImpl->ValueMetadata[this]; 1331 assert(!Info.empty() == HasMetadata && "bit out of sync with hash table"); 1332 if (Info.empty()) 1333 HasMetadata = true; 1334 Info.set(KindID, Node); 1335 return; 1336 } 1337 1338 // Otherwise, we're removing metadata from an instruction. 1339 assert((HasMetadata == (getContext().pImpl->ValueMetadata.count(this) > 0)) && 1340 "bit out of sync with hash table"); 1341 if (!HasMetadata) 1342 return; // Nothing to remove! 1343 auto &Info = getContext().pImpl->ValueMetadata[this]; 1344 1345 // Handle removal of an existing value. 1346 Info.erase(KindID); 1347 if (!Info.empty()) 1348 return; 1349 getContext().pImpl->ValueMetadata.erase(this); 1350 HasMetadata = false; 1351 } 1352 1353 void Value::setMetadata(StringRef Kind, MDNode *Node) { 1354 if (!Node && !HasMetadata) 1355 return; 1356 setMetadata(getContext().getMDKindID(Kind), Node); 1357 } 1358 1359 void Value::addMetadata(unsigned KindID, MDNode &MD) { 1360 assert(isa<Instruction>(this) || isa<GlobalObject>(this)); 1361 if (!HasMetadata) 1362 HasMetadata = true; 1363 getContext().pImpl->ValueMetadata[this].insert(KindID, MD); 1364 } 1365 1366 void Value::addMetadata(StringRef Kind, MDNode &MD) { 1367 addMetadata(getContext().getMDKindID(Kind), MD); 1368 } 1369 1370 bool Value::eraseMetadata(unsigned KindID) { 1371 // Nothing to unset. 1372 if (!HasMetadata) 1373 return false; 1374 1375 auto &Store = getContext().pImpl->ValueMetadata[this]; 1376 bool Changed = Store.erase(KindID); 1377 if (Store.empty()) 1378 clearMetadata(); 1379 return Changed; 1380 } 1381 1382 void Value::clearMetadata() { 1383 if (!HasMetadata) 1384 return; 1385 assert(getContext().pImpl->ValueMetadata.count(this) && 1386 "bit out of sync with hash table"); 1387 getContext().pImpl->ValueMetadata.erase(this); 1388 HasMetadata = false; 1389 } 1390 1391 void Instruction::setMetadata(StringRef Kind, MDNode *Node) { 1392 if (!Node && !hasMetadata()) 1393 return; 1394 setMetadata(getContext().getMDKindID(Kind), Node); 1395 } 1396 1397 MDNode *Instruction::getMetadataImpl(StringRef Kind) const { 1398 return getMetadataImpl(getContext().getMDKindID(Kind)); 1399 } 1400 1401 void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) { 1402 if (!Value::hasMetadata()) 1403 return; // Nothing to remove! 1404 1405 SmallSet<unsigned, 4> KnownSet; 1406 KnownSet.insert(KnownIDs.begin(), KnownIDs.end()); 1407 1408 // A DIAssignID attachment is debug metadata, don't drop it. 1409 KnownSet.insert(LLVMContext::MD_DIAssignID); 1410 1411 auto &MetadataStore = getContext().pImpl->ValueMetadata; 1412 auto &Info = MetadataStore[this]; 1413 assert(!Info.empty() && "bit out of sync with hash table"); 1414 Info.remove_if([&KnownSet](const MDAttachments::Attachment &I) { 1415 return !KnownSet.count(I.MDKind); 1416 }); 1417 1418 if (Info.empty()) { 1419 // Drop our entry at the store. 1420 clearMetadata(); 1421 } 1422 } 1423 1424 void Instruction::updateDIAssignIDMapping(DIAssignID *ID) { 1425 auto &IDToInstrs = getContext().pImpl->AssignmentIDToInstrs; 1426 if (const DIAssignID *CurrentID = 1427 cast_or_null<DIAssignID>(getMetadata(LLVMContext::MD_DIAssignID))) { 1428 // Nothing to do if the ID isn't changing. 1429 if (ID == CurrentID) 1430 return; 1431 1432 // Unmap this instruction from its current ID. 1433 auto InstrsIt = IDToInstrs.find(CurrentID); 1434 assert(InstrsIt != IDToInstrs.end() && 1435 "Expect existing attachment to be mapped"); 1436 1437 auto &InstVec = InstrsIt->second; 1438 auto *InstIt = std::find(InstVec.begin(), InstVec.end(), this); 1439 assert(InstIt != InstVec.end() && 1440 "Expect instruction to be mapped to attachment"); 1441 // The vector contains a ptr to this. If this is the only element in the 1442 // vector, remove the ID:vector entry, otherwise just remove the 1443 // instruction from the vector. 1444 if (InstVec.size() == 1) 1445 IDToInstrs.erase(InstrsIt); 1446 else 1447 InstVec.erase(InstIt); 1448 } 1449 1450 // Map this instruction to the new ID. 1451 if (ID) 1452 IDToInstrs[ID].push_back(this); 1453 } 1454 1455 void Instruction::setMetadata(unsigned KindID, MDNode *Node) { 1456 if (!Node && !hasMetadata()) 1457 return; 1458 1459 // Handle 'dbg' as a special case since it is not stored in the hash table. 1460 if (KindID == LLVMContext::MD_dbg) { 1461 DbgLoc = DebugLoc(Node); 1462 return; 1463 } 1464 1465 // Update DIAssignID to Instruction(s) mapping. 1466 if (KindID == LLVMContext::MD_DIAssignID) { 1467 // The DIAssignID tracking infrastructure doesn't support RAUWing temporary 1468 // nodes with DIAssignIDs. The cast_or_null below would also catch this, but 1469 // having a dedicated assert helps make this obvious. 1470 assert((!Node || !Node->isTemporary()) && 1471 "Temporary DIAssignIDs are invalid"); 1472 updateDIAssignIDMapping(cast_or_null<DIAssignID>(Node)); 1473 } 1474 1475 Value::setMetadata(KindID, Node); 1476 } 1477 1478 void Instruction::addAnnotationMetadata(StringRef Name) { 1479 MDBuilder MDB(getContext()); 1480 1481 auto *Existing = getMetadata(LLVMContext::MD_annotation); 1482 SmallVector<Metadata *, 4> Names; 1483 bool AppendName = true; 1484 if (Existing) { 1485 auto *Tuple = cast<MDTuple>(Existing); 1486 for (auto &N : Tuple->operands()) { 1487 if (cast<MDString>(N.get())->getString() == Name) 1488 AppendName = false; 1489 Names.push_back(N.get()); 1490 } 1491 } 1492 if (AppendName) 1493 Names.push_back(MDB.createString(Name)); 1494 1495 MDNode *MD = MDTuple::get(getContext(), Names); 1496 setMetadata(LLVMContext::MD_annotation, MD); 1497 } 1498 1499 AAMDNodes Instruction::getAAMetadata() const { 1500 AAMDNodes Result; 1501 // Not using Instruction::hasMetadata() because we're not interested in 1502 // DebugInfoMetadata. 1503 if (Value::hasMetadata()) { 1504 const auto &Info = getContext().pImpl->ValueMetadata[this]; 1505 Result.TBAA = Info.lookup(LLVMContext::MD_tbaa); 1506 Result.TBAAStruct = Info.lookup(LLVMContext::MD_tbaa_struct); 1507 Result.Scope = Info.lookup(LLVMContext::MD_alias_scope); 1508 Result.NoAlias = Info.lookup(LLVMContext::MD_noalias); 1509 } 1510 return Result; 1511 } 1512 1513 void Instruction::setAAMetadata(const AAMDNodes &N) { 1514 setMetadata(LLVMContext::MD_tbaa, N.TBAA); 1515 setMetadata(LLVMContext::MD_tbaa_struct, N.TBAAStruct); 1516 setMetadata(LLVMContext::MD_alias_scope, N.Scope); 1517 setMetadata(LLVMContext::MD_noalias, N.NoAlias); 1518 } 1519 1520 MDNode *Instruction::getMetadataImpl(unsigned KindID) const { 1521 // Handle 'dbg' as a special case since it is not stored in the hash table. 1522 if (KindID == LLVMContext::MD_dbg) 1523 return DbgLoc.getAsMDNode(); 1524 return Value::getMetadata(KindID); 1525 } 1526 1527 void Instruction::getAllMetadataImpl( 1528 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1529 Result.clear(); 1530 1531 // Handle 'dbg' as a special case since it is not stored in the hash table. 1532 if (DbgLoc) { 1533 Result.push_back( 1534 std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode())); 1535 } 1536 Value::getAllMetadata(Result); 1537 } 1538 1539 bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const { 1540 assert( 1541 (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select || 1542 getOpcode() == Instruction::Call || getOpcode() == Instruction::Invoke || 1543 getOpcode() == Instruction::IndirectBr || 1544 getOpcode() == Instruction::Switch) && 1545 "Looking for branch weights on something besides branch"); 1546 1547 return ::extractProfTotalWeight(*this, TotalVal); 1548 } 1549 1550 void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) { 1551 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs; 1552 Other->getAllMetadata(MDs); 1553 for (auto &MD : MDs) { 1554 // We need to adjust the type metadata offset. 1555 if (Offset != 0 && MD.first == LLVMContext::MD_type) { 1556 auto *OffsetConst = cast<ConstantInt>( 1557 cast<ConstantAsMetadata>(MD.second->getOperand(0))->getValue()); 1558 Metadata *TypeId = MD.second->getOperand(1); 1559 auto *NewOffsetMD = ConstantAsMetadata::get(ConstantInt::get( 1560 OffsetConst->getType(), OffsetConst->getValue() + Offset)); 1561 addMetadata(LLVMContext::MD_type, 1562 *MDNode::get(getContext(), {NewOffsetMD, TypeId})); 1563 continue; 1564 } 1565 // If an offset adjustment was specified we need to modify the DIExpression 1566 // to prepend the adjustment: 1567 // !DIExpression(DW_OP_plus, Offset, [original expr]) 1568 auto *Attachment = MD.second; 1569 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) { 1570 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Attachment); 1571 DIExpression *E = nullptr; 1572 if (!GV) { 1573 auto *GVE = cast<DIGlobalVariableExpression>(Attachment); 1574 GV = GVE->getVariable(); 1575 E = GVE->getExpression(); 1576 } 1577 ArrayRef<uint64_t> OrigElements; 1578 if (E) 1579 OrigElements = E->getElements(); 1580 std::vector<uint64_t> Elements(OrigElements.size() + 2); 1581 Elements[0] = dwarf::DW_OP_plus_uconst; 1582 Elements[1] = Offset; 1583 llvm::copy(OrigElements, Elements.begin() + 2); 1584 E = DIExpression::get(getContext(), Elements); 1585 Attachment = DIGlobalVariableExpression::get(getContext(), GV, E); 1586 } 1587 addMetadata(MD.first, *Attachment); 1588 } 1589 } 1590 1591 void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) { 1592 addMetadata( 1593 LLVMContext::MD_type, 1594 *MDTuple::get(getContext(), 1595 {ConstantAsMetadata::get(ConstantInt::get( 1596 Type::getInt64Ty(getContext()), Offset)), 1597 TypeID})); 1598 } 1599 1600 void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) { 1601 // Remove any existing vcall visibility metadata first in case we are 1602 // updating. 1603 eraseMetadata(LLVMContext::MD_vcall_visibility); 1604 addMetadata(LLVMContext::MD_vcall_visibility, 1605 *MDNode::get(getContext(), 1606 {ConstantAsMetadata::get(ConstantInt::get( 1607 Type::getInt64Ty(getContext()), Visibility))})); 1608 } 1609 1610 GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const { 1611 if (MDNode *MD = getMetadata(LLVMContext::MD_vcall_visibility)) { 1612 uint64_t Val = cast<ConstantInt>( 1613 cast<ConstantAsMetadata>(MD->getOperand(0))->getValue()) 1614 ->getZExtValue(); 1615 assert(Val <= 2 && "unknown vcall visibility!"); 1616 return (VCallVisibility)Val; 1617 } 1618 return VCallVisibility::VCallVisibilityPublic; 1619 } 1620 1621 void Function::setSubprogram(DISubprogram *SP) { 1622 setMetadata(LLVMContext::MD_dbg, SP); 1623 } 1624 1625 DISubprogram *Function::getSubprogram() const { 1626 return cast_or_null<DISubprogram>(getMetadata(LLVMContext::MD_dbg)); 1627 } 1628 1629 bool Function::shouldEmitDebugInfoForProfiling() const { 1630 if (DISubprogram *SP = getSubprogram()) { 1631 if (DICompileUnit *CU = SP->getUnit()) { 1632 return CU->getDebugInfoForProfiling(); 1633 } 1634 } 1635 return false; 1636 } 1637 1638 void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) { 1639 addMetadata(LLVMContext::MD_dbg, *GV); 1640 } 1641 1642 void GlobalVariable::getDebugInfo( 1643 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const { 1644 SmallVector<MDNode *, 1> MDs; 1645 getMetadata(LLVMContext::MD_dbg, MDs); 1646 for (MDNode *MD : MDs) 1647 GVs.push_back(cast<DIGlobalVariableExpression>(MD)); 1648 } 1649