1 //===- Metadata.cpp - Implement Metadata classes --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the Metadata classes. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/IR/Metadata.h" 14 #include "LLVMContextImpl.h" 15 #include "MetadataImpl.h" 16 #include "SymbolTableListTraitsImpl.h" 17 #include "llvm/ADT/APFloat.h" 18 #include "llvm/ADT/APInt.h" 19 #include "llvm/ADT/ArrayRef.h" 20 #include "llvm/ADT/DenseSet.h" 21 #include "llvm/ADT/None.h" 22 #include "llvm/ADT/STLExtras.h" 23 #include "llvm/ADT/SetVector.h" 24 #include "llvm/ADT/SmallPtrSet.h" 25 #include "llvm/ADT/SmallSet.h" 26 #include "llvm/ADT/SmallVector.h" 27 #include "llvm/ADT/StringMap.h" 28 #include "llvm/ADT/StringRef.h" 29 #include "llvm/ADT/Twine.h" 30 #include "llvm/IR/Argument.h" 31 #include "llvm/IR/BasicBlock.h" 32 #include "llvm/IR/Constant.h" 33 #include "llvm/IR/ConstantRange.h" 34 #include "llvm/IR/Constants.h" 35 #include "llvm/IR/DebugInfoMetadata.h" 36 #include "llvm/IR/DebugLoc.h" 37 #include "llvm/IR/Function.h" 38 #include "llvm/IR/GlobalObject.h" 39 #include "llvm/IR/GlobalVariable.h" 40 #include "llvm/IR/Instruction.h" 41 #include "llvm/IR/LLVMContext.h" 42 #include "llvm/IR/MDBuilder.h" 43 #include "llvm/IR/Module.h" 44 #include "llvm/IR/TrackingMDRef.h" 45 #include "llvm/IR/Type.h" 46 #include "llvm/IR/Value.h" 47 #include "llvm/IR/ValueHandle.h" 48 #include "llvm/Support/Casting.h" 49 #include "llvm/Support/ErrorHandling.h" 50 #include "llvm/Support/MathExtras.h" 51 #include <algorithm> 52 #include <cassert> 53 #include <cstddef> 54 #include <cstdint> 55 #include <iterator> 56 #include <tuple> 57 #include <type_traits> 58 #include <utility> 59 #include <vector> 60 61 using namespace llvm; 62 63 MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD) 64 : Value(Ty, MetadataAsValueVal), MD(MD) { 65 track(); 66 } 67 68 MetadataAsValue::~MetadataAsValue() { 69 getType()->getContext().pImpl->MetadataAsValues.erase(MD); 70 untrack(); 71 } 72 73 /// Canonicalize metadata arguments to intrinsics. 74 /// 75 /// To support bitcode upgrades (and assembly semantic sugar) for \a 76 /// MetadataAsValue, we need to canonicalize certain metadata. 77 /// 78 /// - nullptr is replaced by an empty MDNode. 79 /// - An MDNode with a single null operand is replaced by an empty MDNode. 80 /// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped. 81 /// 82 /// This maintains readability of bitcode from when metadata was a type of 83 /// value, and these bridges were unnecessary. 84 static Metadata *canonicalizeMetadataForValue(LLVMContext &Context, 85 Metadata *MD) { 86 if (!MD) 87 // !{} 88 return MDNode::get(Context, None); 89 90 // Return early if this isn't a single-operand MDNode. 91 auto *N = dyn_cast<MDNode>(MD); 92 if (!N || N->getNumOperands() != 1) 93 return MD; 94 95 if (!N->getOperand(0)) 96 // !{} 97 return MDNode::get(Context, None); 98 99 if (auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(0))) 100 // Look through the MDNode. 101 return C; 102 103 return MD; 104 } 105 106 MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) { 107 MD = canonicalizeMetadataForValue(Context, MD); 108 auto *&Entry = Context.pImpl->MetadataAsValues[MD]; 109 if (!Entry) 110 Entry = new MetadataAsValue(Type::getMetadataTy(Context), MD); 111 return Entry; 112 } 113 114 MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context, 115 Metadata *MD) { 116 MD = canonicalizeMetadataForValue(Context, MD); 117 auto &Store = Context.pImpl->MetadataAsValues; 118 return Store.lookup(MD); 119 } 120 121 void MetadataAsValue::handleChangedMetadata(Metadata *MD) { 122 LLVMContext &Context = getContext(); 123 MD = canonicalizeMetadataForValue(Context, MD); 124 auto &Store = Context.pImpl->MetadataAsValues; 125 126 // Stop tracking the old metadata. 127 Store.erase(this->MD); 128 untrack(); 129 this->MD = nullptr; 130 131 // Start tracking MD, or RAUW if necessary. 132 auto *&Entry = Store[MD]; 133 if (Entry) { 134 replaceAllUsesWith(Entry); 135 delete this; 136 return; 137 } 138 139 this->MD = MD; 140 track(); 141 Entry = this; 142 } 143 144 void MetadataAsValue::track() { 145 if (MD) 146 MetadataTracking::track(&MD, *MD, *this); 147 } 148 149 void MetadataAsValue::untrack() { 150 if (MD) 151 MetadataTracking::untrack(MD); 152 } 153 154 bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) { 155 assert(Ref && "Expected live reference"); 156 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) && 157 "Reference without owner must be direct"); 158 if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) { 159 R->addRef(Ref, Owner); 160 return true; 161 } 162 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) { 163 assert(!PH->Use && "Placeholders can only be used once"); 164 assert(!Owner && "Unexpected callback to owner"); 165 PH->Use = static_cast<Metadata **>(Ref); 166 return true; 167 } 168 return false; 169 } 170 171 void MetadataTracking::untrack(void *Ref, Metadata &MD) { 172 assert(Ref && "Expected live reference"); 173 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) 174 R->dropRef(Ref); 175 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) 176 PH->Use = nullptr; 177 } 178 179 bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) { 180 assert(Ref && "Expected live reference"); 181 assert(New && "Expected live reference"); 182 assert(Ref != New && "Expected change"); 183 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) { 184 R->moveRef(Ref, New, MD); 185 return true; 186 } 187 assert(!isa<DistinctMDOperandPlaceholder>(MD) && 188 "Unexpected move of an MDOperand"); 189 assert(!isReplaceable(MD) && 190 "Expected un-replaceable metadata, since we didn't move a reference"); 191 return false; 192 } 193 194 bool MetadataTracking::isReplaceable(const Metadata &MD) { 195 return ReplaceableMetadataImpl::isReplaceable(MD); 196 } 197 198 void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) { 199 bool WasInserted = 200 UseMap.insert(std::make_pair(Ref, std::make_pair(Owner, NextIndex))) 201 .second; 202 (void)WasInserted; 203 assert(WasInserted && "Expected to add a reference"); 204 205 ++NextIndex; 206 assert(NextIndex != 0 && "Unexpected overflow"); 207 } 208 209 void ReplaceableMetadataImpl::dropRef(void *Ref) { 210 bool WasErased = UseMap.erase(Ref); 211 (void)WasErased; 212 assert(WasErased && "Expected to drop a reference"); 213 } 214 215 void ReplaceableMetadataImpl::moveRef(void *Ref, void *New, 216 const Metadata &MD) { 217 auto I = UseMap.find(Ref); 218 assert(I != UseMap.end() && "Expected to move a reference"); 219 auto OwnerAndIndex = I->second; 220 UseMap.erase(I); 221 bool WasInserted = UseMap.insert(std::make_pair(New, OwnerAndIndex)).second; 222 (void)WasInserted; 223 assert(WasInserted && "Expected to add a reference"); 224 225 // Check that the references are direct if there's no owner. 226 (void)MD; 227 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) && 228 "Reference without owner must be direct"); 229 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) && 230 "Reference without owner must be direct"); 231 } 232 233 void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) { 234 if (UseMap.empty()) 235 return; 236 237 // Copy out uses since UseMap will get touched below. 238 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; 239 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); 240 llvm::sort(Uses, [](const UseTy &L, const UseTy &R) { 241 return L.second.second < R.second.second; 242 }); 243 for (const auto &Pair : Uses) { 244 // Check that this Ref hasn't disappeared after RAUW (when updating a 245 // previous Ref). 246 if (!UseMap.count(Pair.first)) 247 continue; 248 249 OwnerTy Owner = Pair.second.first; 250 if (!Owner) { 251 // Update unowned tracking references directly. 252 Metadata *&Ref = *static_cast<Metadata **>(Pair.first); 253 Ref = MD; 254 if (MD) 255 MetadataTracking::track(Ref); 256 UseMap.erase(Pair.first); 257 continue; 258 } 259 260 // Check for MetadataAsValue. 261 if (Owner.is<MetadataAsValue *>()) { 262 Owner.get<MetadataAsValue *>()->handleChangedMetadata(MD); 263 continue; 264 } 265 266 // There's a Metadata owner -- dispatch. 267 Metadata *OwnerMD = Owner.get<Metadata *>(); 268 switch (OwnerMD->getMetadataID()) { 269 #define HANDLE_METADATA_LEAF(CLASS) \ 270 case Metadata::CLASS##Kind: \ 271 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \ 272 continue; 273 #include "llvm/IR/Metadata.def" 274 default: 275 llvm_unreachable("Invalid metadata subclass"); 276 } 277 } 278 assert(UseMap.empty() && "Expected all uses to be replaced"); 279 } 280 281 void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) { 282 if (UseMap.empty()) 283 return; 284 285 if (!ResolveUsers) { 286 UseMap.clear(); 287 return; 288 } 289 290 // Copy out uses since UseMap could get touched below. 291 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; 292 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); 293 llvm::sort(Uses, [](const UseTy &L, const UseTy &R) { 294 return L.second.second < R.second.second; 295 }); 296 UseMap.clear(); 297 for (const auto &Pair : Uses) { 298 auto Owner = Pair.second.first; 299 if (!Owner) 300 continue; 301 if (Owner.is<MetadataAsValue *>()) 302 continue; 303 304 // Resolve MDNodes that point at this. 305 auto *OwnerMD = dyn_cast<MDNode>(Owner.get<Metadata *>()); 306 if (!OwnerMD) 307 continue; 308 if (OwnerMD->isResolved()) 309 continue; 310 OwnerMD->decrementUnresolvedOperandCount(); 311 } 312 } 313 314 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) { 315 if (auto *N = dyn_cast<MDNode>(&MD)) 316 return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses(); 317 return dyn_cast<ValueAsMetadata>(&MD); 318 } 319 320 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) { 321 if (auto *N = dyn_cast<MDNode>(&MD)) 322 return N->isResolved() ? nullptr : N->Context.getReplaceableUses(); 323 return dyn_cast<ValueAsMetadata>(&MD); 324 } 325 326 bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) { 327 if (auto *N = dyn_cast<MDNode>(&MD)) 328 return !N->isResolved(); 329 return dyn_cast<ValueAsMetadata>(&MD); 330 } 331 332 static DISubprogram *getLocalFunctionMetadata(Value *V) { 333 assert(V && "Expected value"); 334 if (auto *A = dyn_cast<Argument>(V)) { 335 if (auto *Fn = A->getParent()) 336 return Fn->getSubprogram(); 337 return nullptr; 338 } 339 340 if (BasicBlock *BB = cast<Instruction>(V)->getParent()) { 341 if (auto *Fn = BB->getParent()) 342 return Fn->getSubprogram(); 343 return nullptr; 344 } 345 346 return nullptr; 347 } 348 349 ValueAsMetadata *ValueAsMetadata::get(Value *V) { 350 assert(V && "Unexpected null Value"); 351 352 auto &Context = V->getContext(); 353 auto *&Entry = Context.pImpl->ValuesAsMetadata[V]; 354 if (!Entry) { 355 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) && 356 "Expected constant or function-local value"); 357 assert(!V->IsUsedByMD && "Expected this to be the only metadata use"); 358 V->IsUsedByMD = true; 359 if (auto *C = dyn_cast<Constant>(V)) 360 Entry = new ConstantAsMetadata(C); 361 else 362 Entry = new LocalAsMetadata(V); 363 } 364 365 return Entry; 366 } 367 368 ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) { 369 assert(V && "Unexpected null Value"); 370 return V->getContext().pImpl->ValuesAsMetadata.lookup(V); 371 } 372 373 void ValueAsMetadata::handleDeletion(Value *V) { 374 assert(V && "Expected valid value"); 375 376 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata; 377 auto I = Store.find(V); 378 if (I == Store.end()) 379 return; 380 381 // Remove old entry from the map. 382 ValueAsMetadata *MD = I->second; 383 assert(MD && "Expected valid metadata"); 384 assert(MD->getValue() == V && "Expected valid mapping"); 385 Store.erase(I); 386 387 // Delete the metadata. 388 MD->replaceAllUsesWith(nullptr); 389 delete MD; 390 } 391 392 void ValueAsMetadata::handleRAUW(Value *From, Value *To) { 393 assert(From && "Expected valid value"); 394 assert(To && "Expected valid value"); 395 assert(From != To && "Expected changed value"); 396 assert(From->getType() == To->getType() && "Unexpected type change"); 397 398 LLVMContext &Context = From->getType()->getContext(); 399 auto &Store = Context.pImpl->ValuesAsMetadata; 400 auto I = Store.find(From); 401 if (I == Store.end()) { 402 assert(!From->IsUsedByMD && "Expected From not to be used by metadata"); 403 return; 404 } 405 406 // Remove old entry from the map. 407 assert(From->IsUsedByMD && "Expected From to be used by metadata"); 408 From->IsUsedByMD = false; 409 ValueAsMetadata *MD = I->second; 410 assert(MD && "Expected valid metadata"); 411 assert(MD->getValue() == From && "Expected valid mapping"); 412 Store.erase(I); 413 414 if (isa<LocalAsMetadata>(MD)) { 415 if (auto *C = dyn_cast<Constant>(To)) { 416 // Local became a constant. 417 MD->replaceAllUsesWith(ConstantAsMetadata::get(C)); 418 delete MD; 419 return; 420 } 421 if (getLocalFunctionMetadata(From) && getLocalFunctionMetadata(To) && 422 getLocalFunctionMetadata(From) != getLocalFunctionMetadata(To)) { 423 // DISubprogram changed. 424 MD->replaceAllUsesWith(nullptr); 425 delete MD; 426 return; 427 } 428 } else if (!isa<Constant>(To)) { 429 // Changed to function-local value. 430 MD->replaceAllUsesWith(nullptr); 431 delete MD; 432 return; 433 } 434 435 auto *&Entry = Store[To]; 436 if (Entry) { 437 // The target already exists. 438 MD->replaceAllUsesWith(Entry); 439 delete MD; 440 return; 441 } 442 443 // Update MD in place (and update the map entry). 444 assert(!To->IsUsedByMD && "Expected this to be the only metadata use"); 445 To->IsUsedByMD = true; 446 MD->V = To; 447 Entry = MD; 448 } 449 450 //===----------------------------------------------------------------------===// 451 // MDString implementation. 452 // 453 454 MDString *MDString::get(LLVMContext &Context, StringRef Str) { 455 auto &Store = Context.pImpl->MDStringCache; 456 auto I = Store.try_emplace(Str); 457 auto &MapEntry = I.first->getValue(); 458 if (!I.second) 459 return &MapEntry; 460 MapEntry.Entry = &*I.first; 461 return &MapEntry; 462 } 463 464 StringRef MDString::getString() const { 465 assert(Entry && "Expected to find string map entry"); 466 return Entry->first(); 467 } 468 469 //===----------------------------------------------------------------------===// 470 // MDNode implementation. 471 // 472 473 // Assert that the MDNode types will not be unaligned by the objects 474 // prepended to them. 475 #define HANDLE_MDNODE_LEAF(CLASS) \ 476 static_assert( \ 477 alignof(uint64_t) >= alignof(CLASS), \ 478 "Alignment is insufficient after objects prepended to " #CLASS); 479 #include "llvm/IR/Metadata.def" 480 481 void *MDNode::operator new(size_t Size, unsigned NumOps) { 482 size_t OpSize = NumOps * sizeof(MDOperand); 483 // uint64_t is the most aligned type we need support (ensured by static_assert 484 // above) 485 OpSize = alignTo(OpSize, alignof(uint64_t)); 486 void *Ptr = reinterpret_cast<char *>(::operator new(OpSize + Size)) + OpSize; 487 MDOperand *O = static_cast<MDOperand *>(Ptr); 488 for (MDOperand *E = O - NumOps; O != E; --O) 489 (void)new (O - 1) MDOperand; 490 return Ptr; 491 } 492 493 // Repress memory sanitization, due to use-after-destroy by operator 494 // delete. Bug report 24578 identifies this issue. 495 LLVM_NO_SANITIZE_MEMORY_ATTRIBUTE void MDNode::operator delete(void *Mem) { 496 MDNode *N = static_cast<MDNode *>(Mem); 497 size_t OpSize = N->NumOperands * sizeof(MDOperand); 498 OpSize = alignTo(OpSize, alignof(uint64_t)); 499 500 MDOperand *O = static_cast<MDOperand *>(Mem); 501 for (MDOperand *E = O - N->NumOperands; O != E; --O) 502 (O - 1)->~MDOperand(); 503 ::operator delete(reinterpret_cast<char *>(Mem) - OpSize); 504 } 505 506 MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage, 507 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2) 508 : Metadata(ID, Storage), NumOperands(Ops1.size() + Ops2.size()), 509 NumUnresolved(0), Context(Context) { 510 unsigned Op = 0; 511 for (Metadata *MD : Ops1) 512 setOperand(Op++, MD); 513 for (Metadata *MD : Ops2) 514 setOperand(Op++, MD); 515 516 if (!isUniqued()) 517 return; 518 519 // Count the unresolved operands. If there are any, RAUW support will be 520 // added lazily on first reference. 521 countUnresolvedOperands(); 522 } 523 524 TempMDNode MDNode::clone() const { 525 switch (getMetadataID()) { 526 default: 527 llvm_unreachable("Invalid MDNode subclass"); 528 #define HANDLE_MDNODE_LEAF(CLASS) \ 529 case CLASS##Kind: \ 530 return cast<CLASS>(this)->cloneImpl(); 531 #include "llvm/IR/Metadata.def" 532 } 533 } 534 535 static bool isOperandUnresolved(Metadata *Op) { 536 if (auto *N = dyn_cast_or_null<MDNode>(Op)) 537 return !N->isResolved(); 538 return false; 539 } 540 541 void MDNode::countUnresolvedOperands() { 542 assert(NumUnresolved == 0 && "Expected unresolved ops to be uncounted"); 543 assert(isUniqued() && "Expected this to be uniqued"); 544 NumUnresolved = count_if(operands(), isOperandUnresolved); 545 } 546 547 void MDNode::makeUniqued() { 548 assert(isTemporary() && "Expected this to be temporary"); 549 assert(!isResolved() && "Expected this to be unresolved"); 550 551 // Enable uniquing callbacks. 552 for (auto &Op : mutable_operands()) 553 Op.reset(Op.get(), this); 554 555 // Make this 'uniqued'. 556 Storage = Uniqued; 557 countUnresolvedOperands(); 558 if (!NumUnresolved) { 559 dropReplaceableUses(); 560 assert(isResolved() && "Expected this to be resolved"); 561 } 562 563 assert(isUniqued() && "Expected this to be uniqued"); 564 } 565 566 void MDNode::makeDistinct() { 567 assert(isTemporary() && "Expected this to be temporary"); 568 assert(!isResolved() && "Expected this to be unresolved"); 569 570 // Drop RAUW support and store as a distinct node. 571 dropReplaceableUses(); 572 storeDistinctInContext(); 573 574 assert(isDistinct() && "Expected this to be distinct"); 575 assert(isResolved() && "Expected this to be resolved"); 576 } 577 578 void MDNode::resolve() { 579 assert(isUniqued() && "Expected this to be uniqued"); 580 assert(!isResolved() && "Expected this to be unresolved"); 581 582 NumUnresolved = 0; 583 dropReplaceableUses(); 584 585 assert(isResolved() && "Expected this to be resolved"); 586 } 587 588 void MDNode::dropReplaceableUses() { 589 assert(!NumUnresolved && "Unexpected unresolved operand"); 590 591 // Drop any RAUW support. 592 if (Context.hasReplaceableUses()) 593 Context.takeReplaceableUses()->resolveAllUses(); 594 } 595 596 void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) { 597 assert(isUniqued() && "Expected this to be uniqued"); 598 assert(NumUnresolved != 0 && "Expected unresolved operands"); 599 600 // Check if an operand was resolved. 601 if (!isOperandUnresolved(Old)) { 602 if (isOperandUnresolved(New)) 603 // An operand was un-resolved! 604 ++NumUnresolved; 605 } else if (!isOperandUnresolved(New)) 606 decrementUnresolvedOperandCount(); 607 } 608 609 void MDNode::decrementUnresolvedOperandCount() { 610 assert(!isResolved() && "Expected this to be unresolved"); 611 if (isTemporary()) 612 return; 613 614 assert(isUniqued() && "Expected this to be uniqued"); 615 if (--NumUnresolved) 616 return; 617 618 // Last unresolved operand has just been resolved. 619 dropReplaceableUses(); 620 assert(isResolved() && "Expected this to become resolved"); 621 } 622 623 void MDNode::resolveCycles() { 624 if (isResolved()) 625 return; 626 627 // Resolve this node immediately. 628 resolve(); 629 630 // Resolve all operands. 631 for (const auto &Op : operands()) { 632 auto *N = dyn_cast_or_null<MDNode>(Op); 633 if (!N) 634 continue; 635 636 assert(!N->isTemporary() && 637 "Expected all forward declarations to be resolved"); 638 if (!N->isResolved()) 639 N->resolveCycles(); 640 } 641 } 642 643 static bool hasSelfReference(MDNode *N) { 644 return llvm::is_contained(N->operands(), N); 645 } 646 647 MDNode *MDNode::replaceWithPermanentImpl() { 648 switch (getMetadataID()) { 649 default: 650 // If this type isn't uniquable, replace with a distinct node. 651 return replaceWithDistinctImpl(); 652 653 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 654 case CLASS##Kind: \ 655 break; 656 #include "llvm/IR/Metadata.def" 657 } 658 659 // Even if this type is uniquable, self-references have to be distinct. 660 if (hasSelfReference(this)) 661 return replaceWithDistinctImpl(); 662 return replaceWithUniquedImpl(); 663 } 664 665 MDNode *MDNode::replaceWithUniquedImpl() { 666 // Try to uniquify in place. 667 MDNode *UniquedNode = uniquify(); 668 669 if (UniquedNode == this) { 670 makeUniqued(); 671 return this; 672 } 673 674 // Collision, so RAUW instead. 675 replaceAllUsesWith(UniquedNode); 676 deleteAsSubclass(); 677 return UniquedNode; 678 } 679 680 MDNode *MDNode::replaceWithDistinctImpl() { 681 makeDistinct(); 682 return this; 683 } 684 685 void MDTuple::recalculateHash() { 686 setHash(MDTupleInfo::KeyTy::calculateHash(this)); 687 } 688 689 void MDNode::dropAllReferences() { 690 for (unsigned I = 0, E = NumOperands; I != E; ++I) 691 setOperand(I, nullptr); 692 if (Context.hasReplaceableUses()) { 693 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false); 694 (void)Context.takeReplaceableUses(); 695 } 696 } 697 698 void MDNode::handleChangedOperand(void *Ref, Metadata *New) { 699 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin(); 700 assert(Op < getNumOperands() && "Expected valid operand"); 701 702 if (!isUniqued()) { 703 // This node is not uniqued. Just set the operand and be done with it. 704 setOperand(Op, New); 705 return; 706 } 707 708 // This node is uniqued. 709 eraseFromStore(); 710 711 Metadata *Old = getOperand(Op); 712 setOperand(Op, New); 713 714 // Drop uniquing for self-reference cycles and deleted constants. 715 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Old))) { 716 if (!isResolved()) 717 resolve(); 718 storeDistinctInContext(); 719 return; 720 } 721 722 // Re-unique the node. 723 auto *Uniqued = uniquify(); 724 if (Uniqued == this) { 725 if (!isResolved()) 726 resolveAfterOperandChange(Old, New); 727 return; 728 } 729 730 // Collision. 731 if (!isResolved()) { 732 // Still unresolved, so RAUW. 733 // 734 // First, clear out all operands to prevent any recursion (similar to 735 // dropAllReferences(), but we still need the use-list). 736 for (unsigned O = 0, E = getNumOperands(); O != E; ++O) 737 setOperand(O, nullptr); 738 if (Context.hasReplaceableUses()) 739 Context.getReplaceableUses()->replaceAllUsesWith(Uniqued); 740 deleteAsSubclass(); 741 return; 742 } 743 744 // Store in non-uniqued form if RAUW isn't possible. 745 storeDistinctInContext(); 746 } 747 748 void MDNode::deleteAsSubclass() { 749 switch (getMetadataID()) { 750 default: 751 llvm_unreachable("Invalid subclass of MDNode"); 752 #define HANDLE_MDNODE_LEAF(CLASS) \ 753 case CLASS##Kind: \ 754 delete cast<CLASS>(this); \ 755 break; 756 #include "llvm/IR/Metadata.def" 757 } 758 } 759 760 template <class T, class InfoT> 761 static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) { 762 if (T *U = getUniqued(Store, N)) 763 return U; 764 765 Store.insert(N); 766 return N; 767 } 768 769 template <class NodeTy> struct MDNode::HasCachedHash { 770 using Yes = char[1]; 771 using No = char[2]; 772 template <class U, U Val> struct SFINAE {}; 773 774 template <class U> 775 static Yes &check(SFINAE<void (U::*)(unsigned), &U::setHash> *); 776 template <class U> static No &check(...); 777 778 static const bool value = sizeof(check<NodeTy>(nullptr)) == sizeof(Yes); 779 }; 780 781 MDNode *MDNode::uniquify() { 782 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node"); 783 784 // Try to insert into uniquing store. 785 switch (getMetadataID()) { 786 default: 787 llvm_unreachable("Invalid or non-uniquable subclass of MDNode"); 788 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 789 case CLASS##Kind: { \ 790 CLASS *SubclassThis = cast<CLASS>(this); \ 791 std::integral_constant<bool, HasCachedHash<CLASS>::value> \ 792 ShouldRecalculateHash; \ 793 dispatchRecalculateHash(SubclassThis, ShouldRecalculateHash); \ 794 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \ 795 } 796 #include "llvm/IR/Metadata.def" 797 } 798 } 799 800 void MDNode::eraseFromStore() { 801 switch (getMetadataID()) { 802 default: 803 llvm_unreachable("Invalid or non-uniquable subclass of MDNode"); 804 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 805 case CLASS##Kind: \ 806 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \ 807 break; 808 #include "llvm/IR/Metadata.def" 809 } 810 } 811 812 MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs, 813 StorageType Storage, bool ShouldCreate) { 814 unsigned Hash = 0; 815 if (Storage == Uniqued) { 816 MDTupleInfo::KeyTy Key(MDs); 817 if (auto *N = getUniqued(Context.pImpl->MDTuples, Key)) 818 return N; 819 if (!ShouldCreate) 820 return nullptr; 821 Hash = Key.getHash(); 822 } else { 823 assert(ShouldCreate && "Expected non-uniqued nodes to always be created"); 824 } 825 826 return storeImpl(new (MDs.size()) MDTuple(Context, Storage, Hash, MDs), 827 Storage, Context.pImpl->MDTuples); 828 } 829 830 void MDNode::deleteTemporary(MDNode *N) { 831 assert(N->isTemporary() && "Expected temporary node"); 832 N->replaceAllUsesWith(nullptr); 833 N->deleteAsSubclass(); 834 } 835 836 void MDNode::storeDistinctInContext() { 837 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses"); 838 assert(!NumUnresolved && "Unexpected unresolved nodes"); 839 Storage = Distinct; 840 assert(isResolved() && "Expected this to be resolved"); 841 842 // Reset the hash. 843 switch (getMetadataID()) { 844 default: 845 llvm_unreachable("Invalid subclass of MDNode"); 846 #define HANDLE_MDNODE_LEAF(CLASS) \ 847 case CLASS##Kind: { \ 848 std::integral_constant<bool, HasCachedHash<CLASS>::value> ShouldResetHash; \ 849 dispatchResetHash(cast<CLASS>(this), ShouldResetHash); \ 850 break; \ 851 } 852 #include "llvm/IR/Metadata.def" 853 } 854 855 getContext().pImpl->DistinctMDNodes.push_back(this); 856 } 857 858 void MDNode::replaceOperandWith(unsigned I, Metadata *New) { 859 if (getOperand(I) == New) 860 return; 861 862 if (!isUniqued()) { 863 setOperand(I, New); 864 return; 865 } 866 867 handleChangedOperand(mutable_begin() + I, New); 868 } 869 870 void MDNode::setOperand(unsigned I, Metadata *New) { 871 assert(I < NumOperands); 872 mutable_begin()[I].reset(New, isUniqued() ? this : nullptr); 873 } 874 875 /// Get a node or a self-reference that looks like it. 876 /// 877 /// Special handling for finding self-references, for use by \a 878 /// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from 879 /// when self-referencing nodes were still uniqued. If the first operand has 880 /// the same operands as \c Ops, return the first operand instead. 881 static MDNode *getOrSelfReference(LLVMContext &Context, 882 ArrayRef<Metadata *> Ops) { 883 if (!Ops.empty()) 884 if (MDNode *N = dyn_cast_or_null<MDNode>(Ops[0])) 885 if (N->getNumOperands() == Ops.size() && N == N->getOperand(0)) { 886 for (unsigned I = 1, E = Ops.size(); I != E; ++I) 887 if (Ops[I] != N->getOperand(I)) 888 return MDNode::get(Context, Ops); 889 return N; 890 } 891 892 return MDNode::get(Context, Ops); 893 } 894 895 MDNode *MDNode::concatenate(MDNode *A, MDNode *B) { 896 if (!A) 897 return B; 898 if (!B) 899 return A; 900 901 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); 902 MDs.insert(B->op_begin(), B->op_end()); 903 904 // FIXME: This preserves long-standing behaviour, but is it really the right 905 // behaviour? Or was that an unintended side-effect of node uniquing? 906 return getOrSelfReference(A->getContext(), MDs.getArrayRef()); 907 } 908 909 MDNode *MDNode::intersect(MDNode *A, MDNode *B) { 910 if (!A || !B) 911 return nullptr; 912 913 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); 914 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end()); 915 MDs.remove_if([&](Metadata *MD) { return !BSet.count(MD); }); 916 917 // FIXME: This preserves long-standing behaviour, but is it really the right 918 // behaviour? Or was that an unintended side-effect of node uniquing? 919 return getOrSelfReference(A->getContext(), MDs.getArrayRef()); 920 } 921 922 MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) { 923 if (!A || !B) 924 return nullptr; 925 926 // Take the intersection of domains then union the scopes 927 // within those domains 928 SmallPtrSet<const MDNode *, 16> ADomains; 929 SmallPtrSet<const MDNode *, 16> IntersectDomains; 930 SmallSetVector<Metadata *, 4> MDs; 931 for (const MDOperand &MDOp : A->operands()) 932 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 933 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 934 ADomains.insert(Domain); 935 936 for (const MDOperand &MDOp : B->operands()) 937 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 938 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 939 if (ADomains.contains(Domain)) { 940 IntersectDomains.insert(Domain); 941 MDs.insert(MDOp); 942 } 943 944 for (const MDOperand &MDOp : A->operands()) 945 if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp)) 946 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain()) 947 if (IntersectDomains.contains(Domain)) 948 MDs.insert(MDOp); 949 950 return MDs.empty() ? nullptr 951 : getOrSelfReference(A->getContext(), MDs.getArrayRef()); 952 } 953 954 MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) { 955 if (!A || !B) 956 return nullptr; 957 958 APFloat AVal = mdconst::extract<ConstantFP>(A->getOperand(0))->getValueAPF(); 959 APFloat BVal = mdconst::extract<ConstantFP>(B->getOperand(0))->getValueAPF(); 960 if (AVal < BVal) 961 return A; 962 return B; 963 } 964 965 static bool isContiguous(const ConstantRange &A, const ConstantRange &B) { 966 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper(); 967 } 968 969 static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) { 970 return !A.intersectWith(B).isEmptySet() || isContiguous(A, B); 971 } 972 973 static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints, 974 ConstantInt *Low, ConstantInt *High) { 975 ConstantRange NewRange(Low->getValue(), High->getValue()); 976 unsigned Size = EndPoints.size(); 977 APInt LB = EndPoints[Size - 2]->getValue(); 978 APInt LE = EndPoints[Size - 1]->getValue(); 979 ConstantRange LastRange(LB, LE); 980 if (canBeMerged(NewRange, LastRange)) { 981 ConstantRange Union = LastRange.unionWith(NewRange); 982 Type *Ty = High->getType(); 983 EndPoints[Size - 2] = 984 cast<ConstantInt>(ConstantInt::get(Ty, Union.getLower())); 985 EndPoints[Size - 1] = 986 cast<ConstantInt>(ConstantInt::get(Ty, Union.getUpper())); 987 return true; 988 } 989 return false; 990 } 991 992 static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints, 993 ConstantInt *Low, ConstantInt *High) { 994 if (!EndPoints.empty()) 995 if (tryMergeRange(EndPoints, Low, High)) 996 return; 997 998 EndPoints.push_back(Low); 999 EndPoints.push_back(High); 1000 } 1001 1002 MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) { 1003 // Given two ranges, we want to compute the union of the ranges. This 1004 // is slightly complicated by having to combine the intervals and merge 1005 // the ones that overlap. 1006 1007 if (!A || !B) 1008 return nullptr; 1009 1010 if (A == B) 1011 return A; 1012 1013 // First, walk both lists in order of the lower boundary of each interval. 1014 // At each step, try to merge the new interval to the last one we adedd. 1015 SmallVector<ConstantInt *, 4> EndPoints; 1016 int AI = 0; 1017 int BI = 0; 1018 int AN = A->getNumOperands() / 2; 1019 int BN = B->getNumOperands() / 2; 1020 while (AI < AN && BI < BN) { 1021 ConstantInt *ALow = mdconst::extract<ConstantInt>(A->getOperand(2 * AI)); 1022 ConstantInt *BLow = mdconst::extract<ConstantInt>(B->getOperand(2 * BI)); 1023 1024 if (ALow->getValue().slt(BLow->getValue())) { 1025 addRange(EndPoints, ALow, 1026 mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1))); 1027 ++AI; 1028 } else { 1029 addRange(EndPoints, BLow, 1030 mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1))); 1031 ++BI; 1032 } 1033 } 1034 while (AI < AN) { 1035 addRange(EndPoints, mdconst::extract<ConstantInt>(A->getOperand(2 * AI)), 1036 mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1))); 1037 ++AI; 1038 } 1039 while (BI < BN) { 1040 addRange(EndPoints, mdconst::extract<ConstantInt>(B->getOperand(2 * BI)), 1041 mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1))); 1042 ++BI; 1043 } 1044 1045 // If we have more than 2 ranges (4 endpoints) we have to try to merge 1046 // the last and first ones. 1047 unsigned Size = EndPoints.size(); 1048 if (Size > 4) { 1049 ConstantInt *FB = EndPoints[0]; 1050 ConstantInt *FE = EndPoints[1]; 1051 if (tryMergeRange(EndPoints, FB, FE)) { 1052 for (unsigned i = 0; i < Size - 2; ++i) { 1053 EndPoints[i] = EndPoints[i + 2]; 1054 } 1055 EndPoints.resize(Size - 2); 1056 } 1057 } 1058 1059 // If in the end we have a single range, it is possible that it is now the 1060 // full range. Just drop the metadata in that case. 1061 if (EndPoints.size() == 2) { 1062 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue()); 1063 if (Range.isFullSet()) 1064 return nullptr; 1065 } 1066 1067 SmallVector<Metadata *, 4> MDs; 1068 MDs.reserve(EndPoints.size()); 1069 for (auto *I : EndPoints) 1070 MDs.push_back(ConstantAsMetadata::get(I)); 1071 return MDNode::get(A->getContext(), MDs); 1072 } 1073 1074 MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) { 1075 if (!A || !B) 1076 return nullptr; 1077 1078 ConstantInt *AVal = mdconst::extract<ConstantInt>(A->getOperand(0)); 1079 ConstantInt *BVal = mdconst::extract<ConstantInt>(B->getOperand(0)); 1080 if (AVal->getZExtValue() < BVal->getZExtValue()) 1081 return A; 1082 return B; 1083 } 1084 1085 //===----------------------------------------------------------------------===// 1086 // NamedMDNode implementation. 1087 // 1088 1089 static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) { 1090 return *(SmallVector<TrackingMDRef, 4> *)Operands; 1091 } 1092 1093 NamedMDNode::NamedMDNode(const Twine &N) 1094 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {} 1095 1096 NamedMDNode::~NamedMDNode() { 1097 dropAllReferences(); 1098 delete &getNMDOps(Operands); 1099 } 1100 1101 unsigned NamedMDNode::getNumOperands() const { 1102 return (unsigned)getNMDOps(Operands).size(); 1103 } 1104 1105 MDNode *NamedMDNode::getOperand(unsigned i) const { 1106 assert(i < getNumOperands() && "Invalid Operand number!"); 1107 auto *N = getNMDOps(Operands)[i].get(); 1108 return cast_or_null<MDNode>(N); 1109 } 1110 1111 void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(M); } 1112 1113 void NamedMDNode::setOperand(unsigned I, MDNode *New) { 1114 assert(I < getNumOperands() && "Invalid operand number"); 1115 getNMDOps(Operands)[I].reset(New); 1116 } 1117 1118 void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(this); } 1119 1120 void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); } 1121 1122 StringRef NamedMDNode::getName() const { return StringRef(Name); } 1123 1124 //===----------------------------------------------------------------------===// 1125 // Instruction Metadata method implementations. 1126 // 1127 1128 MDNode *MDAttachments::lookup(unsigned ID) const { 1129 for (const auto &A : Attachments) 1130 if (A.MDKind == ID) 1131 return A.Node; 1132 return nullptr; 1133 } 1134 1135 void MDAttachments::get(unsigned ID, SmallVectorImpl<MDNode *> &Result) const { 1136 for (const auto &A : Attachments) 1137 if (A.MDKind == ID) 1138 Result.push_back(A.Node); 1139 } 1140 1141 void MDAttachments::getAll( 1142 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1143 for (const auto &A : Attachments) 1144 Result.emplace_back(A.MDKind, A.Node); 1145 1146 // Sort the resulting array so it is stable with respect to metadata IDs. We 1147 // need to preserve the original insertion order though. 1148 if (Result.size() > 1) 1149 llvm::stable_sort(Result, less_first()); 1150 } 1151 1152 void MDAttachments::set(unsigned ID, MDNode *MD) { 1153 erase(ID); 1154 if (MD) 1155 insert(ID, *MD); 1156 } 1157 1158 void MDAttachments::insert(unsigned ID, MDNode &MD) { 1159 Attachments.push_back({ID, TrackingMDNodeRef(&MD)}); 1160 } 1161 1162 bool MDAttachments::erase(unsigned ID) { 1163 if (empty()) 1164 return false; 1165 1166 // Common case is one value. 1167 if (Attachments.size() == 1 && Attachments.back().MDKind == ID) { 1168 Attachments.pop_back(); 1169 return true; 1170 } 1171 1172 auto OldSize = Attachments.size(); 1173 llvm::erase_if(Attachments, 1174 [ID](const Attachment &A) { return A.MDKind == ID; }); 1175 return OldSize != Attachments.size(); 1176 } 1177 1178 MDNode *Value::getMetadata(unsigned KindID) const { 1179 if (!hasMetadata()) 1180 return nullptr; 1181 const auto &Info = getContext().pImpl->ValueMetadata[this]; 1182 assert(!Info.empty() && "bit out of sync with hash table"); 1183 return Info.lookup(KindID); 1184 } 1185 1186 MDNode *Value::getMetadata(StringRef Kind) const { 1187 if (!hasMetadata()) 1188 return nullptr; 1189 const auto &Info = getContext().pImpl->ValueMetadata[this]; 1190 assert(!Info.empty() && "bit out of sync with hash table"); 1191 return Info.lookup(getContext().getMDKindID(Kind)); 1192 } 1193 1194 void Value::getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const { 1195 if (hasMetadata()) 1196 getContext().pImpl->ValueMetadata[this].get(KindID, MDs); 1197 } 1198 1199 void Value::getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const { 1200 if (hasMetadata()) 1201 getMetadata(getContext().getMDKindID(Kind), MDs); 1202 } 1203 1204 void Value::getAllMetadata( 1205 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const { 1206 if (hasMetadata()) { 1207 assert(getContext().pImpl->ValueMetadata.count(this) && 1208 "bit out of sync with hash table"); 1209 const auto &Info = getContext().pImpl->ValueMetadata.find(this)->second; 1210 assert(!Info.empty() && "Shouldn't have called this"); 1211 Info.getAll(MDs); 1212 } 1213 } 1214 1215 void Value::setMetadata(unsigned KindID, MDNode *Node) { 1216 assert(isa<Instruction>(this) || isa<GlobalObject>(this)); 1217 1218 // Handle the case when we're adding/updating metadata on a value. 1219 if (Node) { 1220 auto &Info = getContext().pImpl->ValueMetadata[this]; 1221 assert(!Info.empty() == HasMetadata && "bit out of sync with hash table"); 1222 if (Info.empty()) 1223 HasMetadata = true; 1224 Info.set(KindID, Node); 1225 return; 1226 } 1227 1228 // Otherwise, we're removing metadata from an instruction. 1229 assert((HasMetadata == (getContext().pImpl->ValueMetadata.count(this) > 0)) && 1230 "bit out of sync with hash table"); 1231 if (!HasMetadata) 1232 return; // Nothing to remove! 1233 auto &Info = getContext().pImpl->ValueMetadata[this]; 1234 1235 // Handle removal of an existing value. 1236 Info.erase(KindID); 1237 if (!Info.empty()) 1238 return; 1239 getContext().pImpl->ValueMetadata.erase(this); 1240 HasMetadata = false; 1241 } 1242 1243 void Value::setMetadata(StringRef Kind, MDNode *Node) { 1244 if (!Node && !HasMetadata) 1245 return; 1246 setMetadata(getContext().getMDKindID(Kind), Node); 1247 } 1248 1249 void Value::addMetadata(unsigned KindID, MDNode &MD) { 1250 assert(isa<Instruction>(this) || isa<GlobalObject>(this)); 1251 if (!HasMetadata) 1252 HasMetadata = true; 1253 getContext().pImpl->ValueMetadata[this].insert(KindID, MD); 1254 } 1255 1256 void Value::addMetadata(StringRef Kind, MDNode &MD) { 1257 addMetadata(getContext().getMDKindID(Kind), MD); 1258 } 1259 1260 bool Value::eraseMetadata(unsigned KindID) { 1261 // Nothing to unset. 1262 if (!HasMetadata) 1263 return false; 1264 1265 auto &Store = getContext().pImpl->ValueMetadata[this]; 1266 bool Changed = Store.erase(KindID); 1267 if (Store.empty()) 1268 clearMetadata(); 1269 return Changed; 1270 } 1271 1272 void Value::clearMetadata() { 1273 if (!HasMetadata) 1274 return; 1275 assert(getContext().pImpl->ValueMetadata.count(this) && 1276 "bit out of sync with hash table"); 1277 getContext().pImpl->ValueMetadata.erase(this); 1278 HasMetadata = false; 1279 } 1280 1281 void Instruction::setMetadata(StringRef Kind, MDNode *Node) { 1282 if (!Node && !hasMetadata()) 1283 return; 1284 setMetadata(getContext().getMDKindID(Kind), Node); 1285 } 1286 1287 MDNode *Instruction::getMetadataImpl(StringRef Kind) const { 1288 return getMetadataImpl(getContext().getMDKindID(Kind)); 1289 } 1290 1291 void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) { 1292 if (!Value::hasMetadata()) 1293 return; // Nothing to remove! 1294 1295 if (KnownIDs.empty()) { 1296 // Just drop our entry at the store. 1297 clearMetadata(); 1298 return; 1299 } 1300 1301 SmallSet<unsigned, 4> KnownSet; 1302 KnownSet.insert(KnownIDs.begin(), KnownIDs.end()); 1303 1304 auto &MetadataStore = getContext().pImpl->ValueMetadata; 1305 auto &Info = MetadataStore[this]; 1306 assert(!Info.empty() && "bit out of sync with hash table"); 1307 Info.remove_if([&KnownSet](const MDAttachments::Attachment &I) { 1308 return !KnownSet.count(I.MDKind); 1309 }); 1310 1311 if (Info.empty()) { 1312 // Drop our entry at the store. 1313 clearMetadata(); 1314 } 1315 } 1316 1317 void Instruction::setMetadata(unsigned KindID, MDNode *Node) { 1318 if (!Node && !hasMetadata()) 1319 return; 1320 1321 // Handle 'dbg' as a special case since it is not stored in the hash table. 1322 if (KindID == LLVMContext::MD_dbg) { 1323 DbgLoc = DebugLoc(Node); 1324 return; 1325 } 1326 1327 Value::setMetadata(KindID, Node); 1328 } 1329 1330 void Instruction::addAnnotationMetadata(StringRef Name) { 1331 MDBuilder MDB(getContext()); 1332 1333 auto *Existing = getMetadata(LLVMContext::MD_annotation); 1334 SmallVector<Metadata *, 4> Names; 1335 bool AppendName = true; 1336 if (Existing) { 1337 auto *Tuple = cast<MDTuple>(Existing); 1338 for (auto &N : Tuple->operands()) { 1339 if (cast<MDString>(N.get())->getString() == Name) 1340 AppendName = false; 1341 Names.push_back(N.get()); 1342 } 1343 } 1344 if (AppendName) 1345 Names.push_back(MDB.createString(Name)); 1346 1347 MDNode *MD = MDTuple::get(getContext(), Names); 1348 setMetadata(LLVMContext::MD_annotation, MD); 1349 } 1350 1351 void Instruction::setAAMetadata(const AAMDNodes &N) { 1352 setMetadata(LLVMContext::MD_tbaa, N.TBAA); 1353 setMetadata(LLVMContext::MD_tbaa_struct, N.TBAAStruct); 1354 setMetadata(LLVMContext::MD_alias_scope, N.Scope); 1355 setMetadata(LLVMContext::MD_noalias, N.NoAlias); 1356 } 1357 1358 MDNode *Instruction::getMetadataImpl(unsigned KindID) const { 1359 // Handle 'dbg' as a special case since it is not stored in the hash table. 1360 if (KindID == LLVMContext::MD_dbg) 1361 return DbgLoc.getAsMDNode(); 1362 return Value::getMetadata(KindID); 1363 } 1364 1365 void Instruction::getAllMetadataImpl( 1366 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1367 Result.clear(); 1368 1369 // Handle 'dbg' as a special case since it is not stored in the hash table. 1370 if (DbgLoc) { 1371 Result.push_back( 1372 std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode())); 1373 } 1374 Value::getAllMetadata(Result); 1375 } 1376 1377 bool Instruction::extractProfMetadata(uint64_t &TrueVal, 1378 uint64_t &FalseVal) const { 1379 assert( 1380 (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select) && 1381 "Looking for branch weights on something besides branch or select"); 1382 1383 auto *ProfileData = getMetadata(LLVMContext::MD_prof); 1384 if (!ProfileData || ProfileData->getNumOperands() != 3) 1385 return false; 1386 1387 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); 1388 if (!ProfDataName || !ProfDataName->getString().equals("branch_weights")) 1389 return false; 1390 1391 auto *CITrue = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1)); 1392 auto *CIFalse = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2)); 1393 if (!CITrue || !CIFalse) 1394 return false; 1395 1396 TrueVal = CITrue->getValue().getZExtValue(); 1397 FalseVal = CIFalse->getValue().getZExtValue(); 1398 1399 return true; 1400 } 1401 1402 bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const { 1403 assert((getOpcode() == Instruction::Br || 1404 getOpcode() == Instruction::Select || 1405 getOpcode() == Instruction::Call || 1406 getOpcode() == Instruction::Invoke || 1407 getOpcode() == Instruction::Switch) && 1408 "Looking for branch weights on something besides branch"); 1409 1410 TotalVal = 0; 1411 auto *ProfileData = getMetadata(LLVMContext::MD_prof); 1412 if (!ProfileData) 1413 return false; 1414 1415 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); 1416 if (!ProfDataName) 1417 return false; 1418 1419 if (ProfDataName->getString().equals("branch_weights")) { 1420 TotalVal = 0; 1421 for (unsigned i = 1; i < ProfileData->getNumOperands(); i++) { 1422 auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i)); 1423 if (!V) 1424 return false; 1425 TotalVal += V->getValue().getZExtValue(); 1426 } 1427 return true; 1428 } else if (ProfDataName->getString().equals("VP") && 1429 ProfileData->getNumOperands() > 3) { 1430 TotalVal = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2)) 1431 ->getValue() 1432 .getZExtValue(); 1433 return true; 1434 } 1435 return false; 1436 } 1437 1438 void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) { 1439 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs; 1440 Other->getAllMetadata(MDs); 1441 for (auto &MD : MDs) { 1442 // We need to adjust the type metadata offset. 1443 if (Offset != 0 && MD.first == LLVMContext::MD_type) { 1444 auto *OffsetConst = cast<ConstantInt>( 1445 cast<ConstantAsMetadata>(MD.second->getOperand(0))->getValue()); 1446 Metadata *TypeId = MD.second->getOperand(1); 1447 auto *NewOffsetMD = ConstantAsMetadata::get(ConstantInt::get( 1448 OffsetConst->getType(), OffsetConst->getValue() + Offset)); 1449 addMetadata(LLVMContext::MD_type, 1450 *MDNode::get(getContext(), {NewOffsetMD, TypeId})); 1451 continue; 1452 } 1453 // If an offset adjustment was specified we need to modify the DIExpression 1454 // to prepend the adjustment: 1455 // !DIExpression(DW_OP_plus, Offset, [original expr]) 1456 auto *Attachment = MD.second; 1457 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) { 1458 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Attachment); 1459 DIExpression *E = nullptr; 1460 if (!GV) { 1461 auto *GVE = cast<DIGlobalVariableExpression>(Attachment); 1462 GV = GVE->getVariable(); 1463 E = GVE->getExpression(); 1464 } 1465 ArrayRef<uint64_t> OrigElements; 1466 if (E) 1467 OrigElements = E->getElements(); 1468 std::vector<uint64_t> Elements(OrigElements.size() + 2); 1469 Elements[0] = dwarf::DW_OP_plus_uconst; 1470 Elements[1] = Offset; 1471 llvm::copy(OrigElements, Elements.begin() + 2); 1472 E = DIExpression::get(getContext(), Elements); 1473 Attachment = DIGlobalVariableExpression::get(getContext(), GV, E); 1474 } 1475 addMetadata(MD.first, *Attachment); 1476 } 1477 } 1478 1479 void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) { 1480 addMetadata( 1481 LLVMContext::MD_type, 1482 *MDTuple::get(getContext(), 1483 {ConstantAsMetadata::get(ConstantInt::get( 1484 Type::getInt64Ty(getContext()), Offset)), 1485 TypeID})); 1486 } 1487 1488 void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) { 1489 // Remove any existing vcall visibility metadata first in case we are 1490 // updating. 1491 eraseMetadata(LLVMContext::MD_vcall_visibility); 1492 addMetadata(LLVMContext::MD_vcall_visibility, 1493 *MDNode::get(getContext(), 1494 {ConstantAsMetadata::get(ConstantInt::get( 1495 Type::getInt64Ty(getContext()), Visibility))})); 1496 } 1497 1498 GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const { 1499 if (MDNode *MD = getMetadata(LLVMContext::MD_vcall_visibility)) { 1500 uint64_t Val = cast<ConstantInt>( 1501 cast<ConstantAsMetadata>(MD->getOperand(0))->getValue()) 1502 ->getZExtValue(); 1503 assert(Val <= 2 && "unknown vcall visibility!"); 1504 return (VCallVisibility)Val; 1505 } 1506 return VCallVisibility::VCallVisibilityPublic; 1507 } 1508 1509 void Function::setSubprogram(DISubprogram *SP) { 1510 setMetadata(LLVMContext::MD_dbg, SP); 1511 } 1512 1513 DISubprogram *Function::getSubprogram() const { 1514 return cast_or_null<DISubprogram>(getMetadata(LLVMContext::MD_dbg)); 1515 } 1516 1517 bool Function::isDebugInfoForProfiling() const { 1518 if (DISubprogram *SP = getSubprogram()) { 1519 if (DICompileUnit *CU = SP->getUnit()) { 1520 return CU->getDebugInfoForProfiling(); 1521 } 1522 } 1523 return false; 1524 } 1525 1526 void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) { 1527 addMetadata(LLVMContext::MD_dbg, *GV); 1528 } 1529 1530 void GlobalVariable::getDebugInfo( 1531 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const { 1532 SmallVector<MDNode *, 1> MDs; 1533 getMetadata(LLVMContext::MD_dbg, MDs); 1534 for (MDNode *MD : MDs) 1535 GVs.push_back(cast<DIGlobalVariableExpression>(MD)); 1536 } 1537