1 //===- CGSCCPassManager.cpp - Managing & running CGSCC passes -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/Analysis/CGSCCPassManager.h" 10 #include "llvm/ADT/ArrayRef.h" 11 #include "llvm/ADT/Optional.h" 12 #include "llvm/ADT/STLExtras.h" 13 #include "llvm/ADT/SetVector.h" 14 #include "llvm/ADT/SmallPtrSet.h" 15 #include "llvm/ADT/SmallVector.h" 16 #include "llvm/ADT/iterator_range.h" 17 #include "llvm/Analysis/LazyCallGraph.h" 18 #include "llvm/IR/Constant.h" 19 #include "llvm/IR/InstIterator.h" 20 #include "llvm/IR/Instruction.h" 21 #include "llvm/IR/PassManager.h" 22 #include "llvm/IR/PassManagerImpl.h" 23 #include "llvm/Support/Casting.h" 24 #include "llvm/Support/Debug.h" 25 #include "llvm/Support/raw_ostream.h" 26 #include "llvm/Support/TimeProfiler.h" 27 #include <algorithm> 28 #include <cassert> 29 #include <iterator> 30 31 #define DEBUG_TYPE "cgscc" 32 33 using namespace llvm; 34 35 // Explicit template instantiations and specialization definitions for core 36 // template typedefs. 37 namespace llvm { 38 39 // Explicit instantiations for the core proxy templates. 40 template class AllAnalysesOn<LazyCallGraph::SCC>; 41 template class AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &>; 42 template class PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, 43 LazyCallGraph &, CGSCCUpdateResult &>; 44 template class InnerAnalysisManagerProxy<CGSCCAnalysisManager, Module>; 45 template class OuterAnalysisManagerProxy<ModuleAnalysisManager, 46 LazyCallGraph::SCC, LazyCallGraph &>; 47 template class OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function>; 48 49 /// Explicitly specialize the pass manager run method to handle call graph 50 /// updates. 51 template <> 52 PreservedAnalyses 53 PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, 54 CGSCCUpdateResult &>::run(LazyCallGraph::SCC &InitialC, 55 CGSCCAnalysisManager &AM, 56 LazyCallGraph &G, CGSCCUpdateResult &UR) { 57 // Request PassInstrumentation from analysis manager, will use it to run 58 // instrumenting callbacks for the passes later. 59 PassInstrumentation PI = 60 AM.getResult<PassInstrumentationAnalysis>(InitialC, G); 61 62 PreservedAnalyses PA = PreservedAnalyses::all(); 63 64 if (DebugLogging) 65 dbgs() << "Starting CGSCC pass manager run.\n"; 66 67 // The SCC may be refined while we are running passes over it, so set up 68 // a pointer that we can update. 69 LazyCallGraph::SCC *C = &InitialC; 70 71 // Get Function analysis manager from its proxy. 72 FunctionAnalysisManager &FAM = 73 AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*C)->getManager(); 74 75 for (auto &Pass : Passes) { 76 // Check the PassInstrumentation's BeforePass callbacks before running the 77 // pass, skip its execution completely if asked to (callback returns false). 78 if (!PI.runBeforePass(*Pass, *C)) 79 continue; 80 81 if (DebugLogging) 82 dbgs() << "Running pass: " << Pass->name() << " on " << *C << "\n"; 83 84 PreservedAnalyses PassPA; 85 { 86 TimeTraceScope TimeScope(Pass->name()); 87 PassPA = Pass->run(*C, AM, G, UR); 88 } 89 90 if (UR.InvalidatedSCCs.count(C)) 91 PI.runAfterPassInvalidated<LazyCallGraph::SCC>(*Pass); 92 else 93 PI.runAfterPass<LazyCallGraph::SCC>(*Pass, *C); 94 95 // Update the SCC if necessary. 96 C = UR.UpdatedC ? UR.UpdatedC : C; 97 if (UR.UpdatedC) { 98 // If C is updated, also create a proxy and update FAM inside the result. 99 auto *ResultFAMCP = 100 &AM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, G); 101 ResultFAMCP->updateFAM(FAM); 102 } 103 104 // If the CGSCC pass wasn't able to provide a valid updated SCC, the 105 // current SCC may simply need to be skipped if invalid. 106 if (UR.InvalidatedSCCs.count(C)) { 107 LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n"); 108 break; 109 } 110 // Check that we didn't miss any update scenario. 111 assert(C->begin() != C->end() && "Cannot have an empty SCC!"); 112 113 // Update the analysis manager as each pass runs and potentially 114 // invalidates analyses. 115 AM.invalidate(*C, PassPA); 116 117 // Finally, we intersect the final preserved analyses to compute the 118 // aggregate preserved set for this pass manager. 119 PA.intersect(std::move(PassPA)); 120 121 // FIXME: Historically, the pass managers all called the LLVM context's 122 // yield function here. We don't have a generic way to acquire the 123 // context and it isn't yet clear what the right pattern is for yielding 124 // in the new pass manager so it is currently omitted. 125 // ...getContext().yield(); 126 } 127 128 // Before we mark all of *this* SCC's analyses as preserved below, intersect 129 // this with the cross-SCC preserved analysis set. This is used to allow 130 // CGSCC passes to mutate ancestor SCCs and still trigger proper invalidation 131 // for them. 132 UR.CrossSCCPA.intersect(PA); 133 134 // Invalidation was handled after each pass in the above loop for the current 135 // SCC. Therefore, the remaining analysis results in the AnalysisManager are 136 // preserved. We mark this with a set so that we don't need to inspect each 137 // one individually. 138 PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>(); 139 140 if (DebugLogging) 141 dbgs() << "Finished CGSCC pass manager run.\n"; 142 143 return PA; 144 } 145 146 bool CGSCCAnalysisManagerModuleProxy::Result::invalidate( 147 Module &M, const PreservedAnalyses &PA, 148 ModuleAnalysisManager::Invalidator &Inv) { 149 // If literally everything is preserved, we're done. 150 if (PA.areAllPreserved()) 151 return false; // This is still a valid proxy. 152 153 // If this proxy or the call graph is going to be invalidated, we also need 154 // to clear all the keys coming from that analysis. 155 // 156 // We also directly invalidate the FAM's module proxy if necessary, and if 157 // that proxy isn't preserved we can't preserve this proxy either. We rely on 158 // it to handle module -> function analysis invalidation in the face of 159 // structural changes and so if it's unavailable we conservatively clear the 160 // entire SCC layer as well rather than trying to do invalidation ourselves. 161 auto PAC = PA.getChecker<CGSCCAnalysisManagerModuleProxy>(); 162 if (!(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>()) || 163 Inv.invalidate<LazyCallGraphAnalysis>(M, PA) || 164 Inv.invalidate<FunctionAnalysisManagerModuleProxy>(M, PA)) { 165 InnerAM->clear(); 166 167 // And the proxy itself should be marked as invalid so that we can observe 168 // the new call graph. This isn't strictly necessary because we cheat 169 // above, but is still useful. 170 return true; 171 } 172 173 // Directly check if the relevant set is preserved so we can short circuit 174 // invalidating SCCs below. 175 bool AreSCCAnalysesPreserved = 176 PA.allAnalysesInSetPreserved<AllAnalysesOn<LazyCallGraph::SCC>>(); 177 178 // Ok, we have a graph, so we can propagate the invalidation down into it. 179 G->buildRefSCCs(); 180 for (auto &RC : G->postorder_ref_sccs()) 181 for (auto &C : RC) { 182 Optional<PreservedAnalyses> InnerPA; 183 184 // Check to see whether the preserved set needs to be adjusted based on 185 // module-level analysis invalidation triggering deferred invalidation 186 // for this SCC. 187 if (auto *OuterProxy = 188 InnerAM->getCachedResult<ModuleAnalysisManagerCGSCCProxy>(C)) 189 for (const auto &OuterInvalidationPair : 190 OuterProxy->getOuterInvalidations()) { 191 AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first; 192 const auto &InnerAnalysisIDs = OuterInvalidationPair.second; 193 if (Inv.invalidate(OuterAnalysisID, M, PA)) { 194 if (!InnerPA) 195 InnerPA = PA; 196 for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) 197 InnerPA->abandon(InnerAnalysisID); 198 } 199 } 200 201 // Check if we needed a custom PA set. If so we'll need to run the inner 202 // invalidation. 203 if (InnerPA) { 204 InnerAM->invalidate(C, *InnerPA); 205 continue; 206 } 207 208 // Otherwise we only need to do invalidation if the original PA set didn't 209 // preserve all SCC analyses. 210 if (!AreSCCAnalysesPreserved) 211 InnerAM->invalidate(C, PA); 212 } 213 214 // Return false to indicate that this result is still a valid proxy. 215 return false; 216 } 217 218 template <> 219 CGSCCAnalysisManagerModuleProxy::Result 220 CGSCCAnalysisManagerModuleProxy::run(Module &M, ModuleAnalysisManager &AM) { 221 // Force the Function analysis manager to also be available so that it can 222 // be accessed in an SCC analysis and proxied onward to function passes. 223 // FIXME: It is pretty awkward to just drop the result here and assert that 224 // we can find it again later. 225 (void)AM.getResult<FunctionAnalysisManagerModuleProxy>(M); 226 227 return Result(*InnerAM, AM.getResult<LazyCallGraphAnalysis>(M)); 228 } 229 230 AnalysisKey FunctionAnalysisManagerCGSCCProxy::Key; 231 232 FunctionAnalysisManagerCGSCCProxy::Result 233 FunctionAnalysisManagerCGSCCProxy::run(LazyCallGraph::SCC &C, 234 CGSCCAnalysisManager &AM, 235 LazyCallGraph &CG) { 236 // Note: unconditionally getting checking that the proxy exists may get it at 237 // this point. There are cases when this is being run unnecessarily, but 238 // it is cheap and having the assertion in place is more valuable. 239 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerCGSCCProxy>(C, CG); 240 Module &M = *C.begin()->getFunction().getParent(); 241 bool ProxyExists = 242 MAMProxy.cachedResultExists<FunctionAnalysisManagerModuleProxy>(M); 243 assert(ProxyExists && 244 "The CGSCC pass manager requires that the FAM module proxy is run " 245 "on the module prior to entering the CGSCC walk"); 246 (void)ProxyExists; 247 248 // We just return an empty result. The caller will use the updateFAM interface 249 // to correctly register the relevant FunctionAnalysisManager based on the 250 // context in which this proxy is run. 251 return Result(); 252 } 253 254 bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate( 255 LazyCallGraph::SCC &C, const PreservedAnalyses &PA, 256 CGSCCAnalysisManager::Invalidator &Inv) { 257 // If literally everything is preserved, we're done. 258 if (PA.areAllPreserved()) 259 return false; // This is still a valid proxy. 260 261 // All updates to preserve valid results are done below, so we don't need to 262 // invalidate this proxy. 263 // 264 // Note that in order to preserve this proxy, a module pass must ensure that 265 // the FAM has been completely updated to handle the deletion of functions. 266 // Specifically, any FAM-cached results for those functions need to have been 267 // forcibly cleared. When preserved, this proxy will only invalidate results 268 // cached on functions *still in the module* at the end of the module pass. 269 auto PAC = PA.getChecker<FunctionAnalysisManagerCGSCCProxy>(); 270 if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<LazyCallGraph::SCC>>()) { 271 for (LazyCallGraph::Node &N : C) 272 FAM->clear(N.getFunction(), N.getFunction().getName()); 273 274 return false; 275 } 276 277 // Directly check if the relevant set is preserved. 278 bool AreFunctionAnalysesPreserved = 279 PA.allAnalysesInSetPreserved<AllAnalysesOn<Function>>(); 280 281 // Now walk all the functions to see if any inner analysis invalidation is 282 // necessary. 283 for (LazyCallGraph::Node &N : C) { 284 Function &F = N.getFunction(); 285 Optional<PreservedAnalyses> FunctionPA; 286 287 // Check to see whether the preserved set needs to be pruned based on 288 // SCC-level analysis invalidation that triggers deferred invalidation 289 // registered with the outer analysis manager proxy for this function. 290 if (auto *OuterProxy = 291 FAM->getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F)) 292 for (const auto &OuterInvalidationPair : 293 OuterProxy->getOuterInvalidations()) { 294 AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first; 295 const auto &InnerAnalysisIDs = OuterInvalidationPair.second; 296 if (Inv.invalidate(OuterAnalysisID, C, PA)) { 297 if (!FunctionPA) 298 FunctionPA = PA; 299 for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) 300 FunctionPA->abandon(InnerAnalysisID); 301 } 302 } 303 304 // Check if we needed a custom PA set, and if so we'll need to run the 305 // inner invalidation. 306 if (FunctionPA) { 307 FAM->invalidate(F, *FunctionPA); 308 continue; 309 } 310 311 // Otherwise we only need to do invalidation if the original PA set didn't 312 // preserve all function analyses. 313 if (!AreFunctionAnalysesPreserved) 314 FAM->invalidate(F, PA); 315 } 316 317 // Return false to indicate that this result is still a valid proxy. 318 return false; 319 } 320 321 } // end namespace llvm 322 323 /// When a new SCC is created for the graph we first update the 324 /// FunctionAnalysisManager in the Proxy's result. 325 /// As there might be function analysis results cached for the functions now in 326 /// that SCC, two forms of updates are required. 327 /// 328 /// First, a proxy from the SCC to the FunctionAnalysisManager needs to be 329 /// created so that any subsequent invalidation events to the SCC are 330 /// propagated to the function analysis results cached for functions within it. 331 /// 332 /// Second, if any of the functions within the SCC have analysis results with 333 /// outer analysis dependencies, then those dependencies would point to the 334 /// *wrong* SCC's analysis result. We forcibly invalidate the necessary 335 /// function analyses so that they don't retain stale handles. 336 static void updateNewSCCFunctionAnalyses(LazyCallGraph::SCC &C, 337 LazyCallGraph &G, 338 CGSCCAnalysisManager &AM, 339 FunctionAnalysisManager &FAM) { 340 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, G).updateFAM(FAM); 341 342 // Now walk the functions in this SCC and invalidate any function analysis 343 // results that might have outer dependencies on an SCC analysis. 344 for (LazyCallGraph::Node &N : C) { 345 Function &F = N.getFunction(); 346 347 auto *OuterProxy = 348 FAM.getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F); 349 if (!OuterProxy) 350 // No outer analyses were queried, nothing to do. 351 continue; 352 353 // Forcibly abandon all the inner analyses with dependencies, but 354 // invalidate nothing else. 355 auto PA = PreservedAnalyses::all(); 356 for (const auto &OuterInvalidationPair : 357 OuterProxy->getOuterInvalidations()) { 358 const auto &InnerAnalysisIDs = OuterInvalidationPair.second; 359 for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) 360 PA.abandon(InnerAnalysisID); 361 } 362 363 // Now invalidate anything we found. 364 FAM.invalidate(F, PA); 365 } 366 } 367 368 /// Helper function to update both the \c CGSCCAnalysisManager \p AM and the \c 369 /// CGSCCPassManager's \c CGSCCUpdateResult \p UR based on a range of newly 370 /// added SCCs. 371 /// 372 /// The range of new SCCs must be in postorder already. The SCC they were split 373 /// out of must be provided as \p C. The current node being mutated and 374 /// triggering updates must be passed as \p N. 375 /// 376 /// This function returns the SCC containing \p N. This will be either \p C if 377 /// no new SCCs have been split out, or it will be the new SCC containing \p N. 378 template <typename SCCRangeT> 379 static LazyCallGraph::SCC * 380 incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G, 381 LazyCallGraph::Node &N, LazyCallGraph::SCC *C, 382 CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR) { 383 using SCC = LazyCallGraph::SCC; 384 385 if (NewSCCRange.begin() == NewSCCRange.end()) 386 return C; 387 388 // Add the current SCC to the worklist as its shape has changed. 389 UR.CWorklist.insert(C); 390 LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist:" << *C 391 << "\n"); 392 393 SCC *OldC = C; 394 395 // Update the current SCC. Note that if we have new SCCs, this must actually 396 // change the SCC. 397 assert(C != &*NewSCCRange.begin() && 398 "Cannot insert new SCCs without changing current SCC!"); 399 C = &*NewSCCRange.begin(); 400 assert(G.lookupSCC(N) == C && "Failed to update current SCC!"); 401 402 // If we had a cached FAM proxy originally, we will want to create more of 403 // them for each SCC that was split off. 404 FunctionAnalysisManager *FAM = nullptr; 405 if (auto *FAMProxy = 406 AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*OldC)) 407 FAM = &FAMProxy->getManager(); 408 409 // We need to propagate an invalidation call to all but the newly current SCC 410 // because the outer pass manager won't do that for us after splitting them. 411 // FIXME: We should accept a PreservedAnalysis from the CG updater so that if 412 // there are preserved analysis we can avoid invalidating them here for 413 // split-off SCCs. 414 // We know however that this will preserve any FAM proxy so go ahead and mark 415 // that. 416 PreservedAnalyses PA; 417 PA.preserve<FunctionAnalysisManagerCGSCCProxy>(); 418 AM.invalidate(*OldC, PA); 419 420 // Ensure the now-current SCC's function analyses are updated. 421 if (FAM) 422 updateNewSCCFunctionAnalyses(*C, G, AM, *FAM); 423 424 for (SCC &NewC : llvm::reverse(make_range(std::next(NewSCCRange.begin()), 425 NewSCCRange.end()))) { 426 assert(C != &NewC && "No need to re-visit the current SCC!"); 427 assert(OldC != &NewC && "Already handled the original SCC!"); 428 UR.CWorklist.insert(&NewC); 429 LLVM_DEBUG(dbgs() << "Enqueuing a newly formed SCC:" << NewC << "\n"); 430 431 // Ensure new SCCs' function analyses are updated. 432 if (FAM) 433 updateNewSCCFunctionAnalyses(NewC, G, AM, *FAM); 434 435 // Also propagate a normal invalidation to the new SCC as only the current 436 // will get one from the pass manager infrastructure. 437 AM.invalidate(NewC, PA); 438 } 439 return C; 440 } 441 442 static LazyCallGraph::SCC &updateCGAndAnalysisManagerForPass( 443 LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N, 444 CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, 445 FunctionAnalysisManager &FAM, bool FunctionPass) { 446 using Node = LazyCallGraph::Node; 447 using Edge = LazyCallGraph::Edge; 448 using SCC = LazyCallGraph::SCC; 449 using RefSCC = LazyCallGraph::RefSCC; 450 451 RefSCC &InitialRC = InitialC.getOuterRefSCC(); 452 SCC *C = &InitialC; 453 RefSCC *RC = &InitialRC; 454 Function &F = N.getFunction(); 455 456 // Walk the function body and build up the set of retained, promoted, and 457 // demoted edges. 458 SmallVector<Constant *, 16> Worklist; 459 SmallPtrSet<Constant *, 16> Visited; 460 SmallPtrSet<Node *, 16> RetainedEdges; 461 SmallSetVector<Node *, 4> PromotedRefTargets; 462 SmallSetVector<Node *, 4> DemotedCallTargets; 463 SmallSetVector<Node *, 4> NewCallEdges; 464 SmallSetVector<Node *, 4> NewRefEdges; 465 466 // First walk the function and handle all called functions. We do this first 467 // because if there is a single call edge, whether there are ref edges is 468 // irrelevant. 469 for (Instruction &I : instructions(F)) 470 if (auto *CB = dyn_cast<CallBase>(&I)) 471 if (Function *Callee = CB->getCalledFunction()) 472 if (Visited.insert(Callee).second && !Callee->isDeclaration()) { 473 Node &CalleeN = *G.lookup(*Callee); 474 Edge *E = N->lookup(CalleeN); 475 assert((E || !FunctionPass) && 476 "No function transformations should introduce *new* " 477 "call edges! Any new calls should be modeled as " 478 "promoted existing ref edges!"); 479 bool Inserted = RetainedEdges.insert(&CalleeN).second; 480 (void)Inserted; 481 assert(Inserted && "We should never visit a function twice."); 482 if (!E) 483 NewCallEdges.insert(&CalleeN); 484 else if (!E->isCall()) 485 PromotedRefTargets.insert(&CalleeN); 486 } 487 488 // Now walk all references. 489 for (Instruction &I : instructions(F)) 490 for (Value *Op : I.operand_values()) 491 if (auto *C = dyn_cast<Constant>(Op)) 492 if (Visited.insert(C).second) 493 Worklist.push_back(C); 494 495 auto VisitRef = [&](Function &Referee) { 496 Node &RefereeN = *G.lookup(Referee); 497 Edge *E = N->lookup(RefereeN); 498 assert((E || !FunctionPass) && 499 "No function transformations should introduce *new* ref " 500 "edges! Any new ref edges would require IPO which " 501 "function passes aren't allowed to do!"); 502 bool Inserted = RetainedEdges.insert(&RefereeN).second; 503 (void)Inserted; 504 assert(Inserted && "We should never visit a function twice."); 505 if (!E) 506 NewRefEdges.insert(&RefereeN); 507 else if (E->isCall()) 508 DemotedCallTargets.insert(&RefereeN); 509 }; 510 LazyCallGraph::visitReferences(Worklist, Visited, VisitRef); 511 512 // Handle new ref edges. 513 for (Node *RefTarget : NewRefEdges) { 514 SCC &TargetC = *G.lookupSCC(*RefTarget); 515 RefSCC &TargetRC = TargetC.getOuterRefSCC(); 516 (void)TargetRC; 517 // TODO: This only allows trivial edges to be added for now. 518 assert((RC == &TargetRC || 519 RC->isAncestorOf(TargetRC)) && "New ref edge is not trivial!"); 520 RC->insertTrivialRefEdge(N, *RefTarget); 521 } 522 523 // Handle new call edges. 524 for (Node *CallTarget : NewCallEdges) { 525 SCC &TargetC = *G.lookupSCC(*CallTarget); 526 RefSCC &TargetRC = TargetC.getOuterRefSCC(); 527 (void)TargetRC; 528 // TODO: This only allows trivial edges to be added for now. 529 assert((RC == &TargetRC || 530 RC->isAncestorOf(TargetRC)) && "New call edge is not trivial!"); 531 RC->insertTrivialCallEdge(N, *CallTarget); 532 } 533 534 // Include synthetic reference edges to known, defined lib functions. 535 for (auto *F : G.getLibFunctions()) 536 // While the list of lib functions doesn't have repeats, don't re-visit 537 // anything handled above. 538 if (!Visited.count(F)) 539 VisitRef(*F); 540 541 // First remove all of the edges that are no longer present in this function. 542 // The first step makes these edges uniformly ref edges and accumulates them 543 // into a separate data structure so removal doesn't invalidate anything. 544 SmallVector<Node *, 4> DeadTargets; 545 for (Edge &E : *N) { 546 if (RetainedEdges.count(&E.getNode())) 547 continue; 548 549 SCC &TargetC = *G.lookupSCC(E.getNode()); 550 RefSCC &TargetRC = TargetC.getOuterRefSCC(); 551 if (&TargetRC == RC && E.isCall()) { 552 if (C != &TargetC) { 553 // For separate SCCs this is trivial. 554 RC->switchTrivialInternalEdgeToRef(N, E.getNode()); 555 } else { 556 // Now update the call graph. 557 C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, E.getNode()), 558 G, N, C, AM, UR); 559 } 560 } 561 562 // Now that this is ready for actual removal, put it into our list. 563 DeadTargets.push_back(&E.getNode()); 564 } 565 // Remove the easy cases quickly and actually pull them out of our list. 566 DeadTargets.erase( 567 llvm::remove_if(DeadTargets, 568 [&](Node *TargetN) { 569 SCC &TargetC = *G.lookupSCC(*TargetN); 570 RefSCC &TargetRC = TargetC.getOuterRefSCC(); 571 572 // We can't trivially remove internal targets, so skip 573 // those. 574 if (&TargetRC == RC) 575 return false; 576 577 RC->removeOutgoingEdge(N, *TargetN); 578 LLVM_DEBUG(dbgs() << "Deleting outgoing edge from '" 579 << N << "' to '" << TargetN << "'\n"); 580 return true; 581 }), 582 DeadTargets.end()); 583 584 // Now do a batch removal of the internal ref edges left. 585 auto NewRefSCCs = RC->removeInternalRefEdge(N, DeadTargets); 586 if (!NewRefSCCs.empty()) { 587 // The old RefSCC is dead, mark it as such. 588 UR.InvalidatedRefSCCs.insert(RC); 589 590 // Note that we don't bother to invalidate analyses as ref-edge 591 // connectivity is not really observable in any way and is intended 592 // exclusively to be used for ordering of transforms rather than for 593 // analysis conclusions. 594 595 // Update RC to the "bottom". 596 assert(G.lookupSCC(N) == C && "Changed the SCC when splitting RefSCCs!"); 597 RC = &C->getOuterRefSCC(); 598 assert(G.lookupRefSCC(N) == RC && "Failed to update current RefSCC!"); 599 600 // The RC worklist is in reverse postorder, so we enqueue the new ones in 601 // RPO except for the one which contains the source node as that is the 602 // "bottom" we will continue processing in the bottom-up walk. 603 assert(NewRefSCCs.front() == RC && 604 "New current RefSCC not first in the returned list!"); 605 for (RefSCC *NewRC : llvm::reverse(make_range(std::next(NewRefSCCs.begin()), 606 NewRefSCCs.end()))) { 607 assert(NewRC != RC && "Should not encounter the current RefSCC further " 608 "in the postorder list of new RefSCCs."); 609 UR.RCWorklist.insert(NewRC); 610 LLVM_DEBUG(dbgs() << "Enqueuing a new RefSCC in the update worklist: " 611 << *NewRC << "\n"); 612 } 613 } 614 615 // Next demote all the call edges that are now ref edges. This helps make 616 // the SCCs small which should minimize the work below as we don't want to 617 // form cycles that this would break. 618 for (Node *RefTarget : DemotedCallTargets) { 619 SCC &TargetC = *G.lookupSCC(*RefTarget); 620 RefSCC &TargetRC = TargetC.getOuterRefSCC(); 621 622 // The easy case is when the target RefSCC is not this RefSCC. This is 623 // only supported when the target RefSCC is a child of this RefSCC. 624 if (&TargetRC != RC) { 625 assert(RC->isAncestorOf(TargetRC) && 626 "Cannot potentially form RefSCC cycles here!"); 627 RC->switchOutgoingEdgeToRef(N, *RefTarget); 628 LLVM_DEBUG(dbgs() << "Switch outgoing call edge to a ref edge from '" << N 629 << "' to '" << *RefTarget << "'\n"); 630 continue; 631 } 632 633 // We are switching an internal call edge to a ref edge. This may split up 634 // some SCCs. 635 if (C != &TargetC) { 636 // For separate SCCs this is trivial. 637 RC->switchTrivialInternalEdgeToRef(N, *RefTarget); 638 continue; 639 } 640 641 // Now update the call graph. 642 C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, *RefTarget), G, N, 643 C, AM, UR); 644 } 645 646 // Now promote ref edges into call edges. 647 for (Node *CallTarget : PromotedRefTargets) { 648 SCC &TargetC = *G.lookupSCC(*CallTarget); 649 RefSCC &TargetRC = TargetC.getOuterRefSCC(); 650 651 // The easy case is when the target RefSCC is not this RefSCC. This is 652 // only supported when the target RefSCC is a child of this RefSCC. 653 if (&TargetRC != RC) { 654 assert(RC->isAncestorOf(TargetRC) && 655 "Cannot potentially form RefSCC cycles here!"); 656 RC->switchOutgoingEdgeToCall(N, *CallTarget); 657 LLVM_DEBUG(dbgs() << "Switch outgoing ref edge to a call edge from '" << N 658 << "' to '" << *CallTarget << "'\n"); 659 continue; 660 } 661 LLVM_DEBUG(dbgs() << "Switch an internal ref edge to a call edge from '" 662 << N << "' to '" << *CallTarget << "'\n"); 663 664 // Otherwise we are switching an internal ref edge to a call edge. This 665 // may merge away some SCCs, and we add those to the UpdateResult. We also 666 // need to make sure to update the worklist in the event SCCs have moved 667 // before the current one in the post-order sequence 668 bool HasFunctionAnalysisProxy = false; 669 auto InitialSCCIndex = RC->find(*C) - RC->begin(); 670 bool FormedCycle = RC->switchInternalEdgeToCall( 671 N, *CallTarget, [&](ArrayRef<SCC *> MergedSCCs) { 672 for (SCC *MergedC : MergedSCCs) { 673 assert(MergedC != &TargetC && "Cannot merge away the target SCC!"); 674 675 HasFunctionAnalysisProxy |= 676 AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>( 677 *MergedC) != nullptr; 678 679 // Mark that this SCC will no longer be valid. 680 UR.InvalidatedSCCs.insert(MergedC); 681 682 // FIXME: We should really do a 'clear' here to forcibly release 683 // memory, but we don't have a good way of doing that and 684 // preserving the function analyses. 685 auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>(); 686 PA.preserve<FunctionAnalysisManagerCGSCCProxy>(); 687 AM.invalidate(*MergedC, PA); 688 } 689 }); 690 691 // If we formed a cycle by creating this call, we need to update more data 692 // structures. 693 if (FormedCycle) { 694 C = &TargetC; 695 assert(G.lookupSCC(N) == C && "Failed to update current SCC!"); 696 697 // If one of the invalidated SCCs had a cached proxy to a function 698 // analysis manager, we need to create a proxy in the new current SCC as 699 // the invalidated SCCs had their functions moved. 700 if (HasFunctionAnalysisProxy) 701 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, G).updateFAM(FAM); 702 703 // Any analyses cached for this SCC are no longer precise as the shape 704 // has changed by introducing this cycle. However, we have taken care to 705 // update the proxies so it remains valide. 706 auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>(); 707 PA.preserve<FunctionAnalysisManagerCGSCCProxy>(); 708 AM.invalidate(*C, PA); 709 } 710 auto NewSCCIndex = RC->find(*C) - RC->begin(); 711 // If we have actually moved an SCC to be topologically "below" the current 712 // one due to merging, we will need to revisit the current SCC after 713 // visiting those moved SCCs. 714 // 715 // It is critical that we *do not* revisit the current SCC unless we 716 // actually move SCCs in the process of merging because otherwise we may 717 // form a cycle where an SCC is split apart, merged, split, merged and so 718 // on infinitely. 719 if (InitialSCCIndex < NewSCCIndex) { 720 // Put our current SCC back onto the worklist as we'll visit other SCCs 721 // that are now definitively ordered prior to the current one in the 722 // post-order sequence, and may end up observing more precise context to 723 // optimize the current SCC. 724 UR.CWorklist.insert(C); 725 LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist: " << *C 726 << "\n"); 727 // Enqueue in reverse order as we pop off the back of the worklist. 728 for (SCC &MovedC : llvm::reverse(make_range(RC->begin() + InitialSCCIndex, 729 RC->begin() + NewSCCIndex))) { 730 UR.CWorklist.insert(&MovedC); 731 LLVM_DEBUG(dbgs() << "Enqueuing a newly earlier in post-order SCC: " 732 << MovedC << "\n"); 733 } 734 } 735 } 736 737 assert(!UR.InvalidatedSCCs.count(C) && "Invalidated the current SCC!"); 738 assert(!UR.InvalidatedRefSCCs.count(RC) && "Invalidated the current RefSCC!"); 739 assert(&C->getOuterRefSCC() == RC && "Current SCC not in current RefSCC!"); 740 741 // Record the current RefSCC and SCC for higher layers of the CGSCC pass 742 // manager now that all the updates have been applied. 743 if (RC != &InitialRC) 744 UR.UpdatedRC = RC; 745 if (C != &InitialC) 746 UR.UpdatedC = C; 747 748 return *C; 749 } 750 751 LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForFunctionPass( 752 LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N, 753 CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, 754 FunctionAnalysisManager &FAM) { 755 return updateCGAndAnalysisManagerForPass(G, InitialC, N, AM, UR, FAM, 756 /* FunctionPass */ true); 757 } 758 LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForCGSCCPass( 759 LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N, 760 CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, 761 FunctionAnalysisManager &FAM) { 762 return updateCGAndAnalysisManagerForPass(G, InitialC, N, AM, UR, FAM, 763 /* FunctionPass */ false); 764 } 765