1 //===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements Function import based on summaries. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Transforms/IPO/FunctionImport.h" 14 #include "llvm/ADT/ArrayRef.h" 15 #include "llvm/ADT/STLExtras.h" 16 #include "llvm/ADT/SetVector.h" 17 #include "llvm/ADT/SmallVector.h" 18 #include "llvm/ADT/Statistic.h" 19 #include "llvm/ADT/StringMap.h" 20 #include "llvm/ADT/StringRef.h" 21 #include "llvm/Bitcode/BitcodeReader.h" 22 #include "llvm/IR/AutoUpgrade.h" 23 #include "llvm/IR/Constants.h" 24 #include "llvm/IR/Function.h" 25 #include "llvm/IR/GlobalAlias.h" 26 #include "llvm/IR/GlobalObject.h" 27 #include "llvm/IR/GlobalValue.h" 28 #include "llvm/IR/GlobalVariable.h" 29 #include "llvm/IR/Metadata.h" 30 #include "llvm/IR/Module.h" 31 #include "llvm/IR/ModuleSummaryIndex.h" 32 #include "llvm/IRReader/IRReader.h" 33 #include "llvm/InitializePasses.h" 34 #include "llvm/Linker/IRMover.h" 35 #include "llvm/Pass.h" 36 #include "llvm/Support/Casting.h" 37 #include "llvm/Support/CommandLine.h" 38 #include "llvm/Support/Debug.h" 39 #include "llvm/Support/Errc.h" 40 #include "llvm/Support/Error.h" 41 #include "llvm/Support/ErrorHandling.h" 42 #include "llvm/Support/FileSystem.h" 43 #include "llvm/Support/SourceMgr.h" 44 #include "llvm/Support/raw_ostream.h" 45 #include "llvm/Transforms/IPO/Internalize.h" 46 #include "llvm/Transforms/Utils/Cloning.h" 47 #include "llvm/Transforms/Utils/FunctionImportUtils.h" 48 #include "llvm/Transforms/Utils/ValueMapper.h" 49 #include <cassert> 50 #include <memory> 51 #include <set> 52 #include <string> 53 #include <system_error> 54 #include <tuple> 55 #include <utility> 56 57 using namespace llvm; 58 59 #define DEBUG_TYPE "function-import" 60 61 STATISTIC(NumImportedFunctionsThinLink, 62 "Number of functions thin link decided to import"); 63 STATISTIC(NumImportedHotFunctionsThinLink, 64 "Number of hot functions thin link decided to import"); 65 STATISTIC(NumImportedCriticalFunctionsThinLink, 66 "Number of critical functions thin link decided to import"); 67 STATISTIC(NumImportedGlobalVarsThinLink, 68 "Number of global variables thin link decided to import"); 69 STATISTIC(NumImportedFunctions, "Number of functions imported in backend"); 70 STATISTIC(NumImportedGlobalVars, 71 "Number of global variables imported in backend"); 72 STATISTIC(NumImportedModules, "Number of modules imported from"); 73 STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index"); 74 STATISTIC(NumLiveSymbols, "Number of live symbols in index"); 75 76 /// Limit on instruction count of imported functions. 77 static cl::opt<unsigned> ImportInstrLimit( 78 "import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"), 79 cl::desc("Only import functions with less than N instructions")); 80 81 static cl::opt<int> ImportCutoff( 82 "import-cutoff", cl::init(-1), cl::Hidden, cl::value_desc("N"), 83 cl::desc("Only import first N functions if N>=0 (default -1)")); 84 85 static cl::opt<bool> 86 ForceImportAll("force-import-all", cl::init(false), cl::Hidden, 87 cl::desc("Import functions with noinline attribute")); 88 89 static cl::opt<float> 90 ImportInstrFactor("import-instr-evolution-factor", cl::init(0.7), 91 cl::Hidden, cl::value_desc("x"), 92 cl::desc("As we import functions, multiply the " 93 "`import-instr-limit` threshold by this factor " 94 "before processing newly imported functions")); 95 96 static cl::opt<float> ImportHotInstrFactor( 97 "import-hot-evolution-factor", cl::init(1.0), cl::Hidden, 98 cl::value_desc("x"), 99 cl::desc("As we import functions called from hot callsite, multiply the " 100 "`import-instr-limit` threshold by this factor " 101 "before processing newly imported functions")); 102 103 static cl::opt<float> ImportHotMultiplier( 104 "import-hot-multiplier", cl::init(10.0), cl::Hidden, cl::value_desc("x"), 105 cl::desc("Multiply the `import-instr-limit` threshold for hot callsites")); 106 107 static cl::opt<float> ImportCriticalMultiplier( 108 "import-critical-multiplier", cl::init(100.0), cl::Hidden, 109 cl::value_desc("x"), 110 cl::desc( 111 "Multiply the `import-instr-limit` threshold for critical callsites")); 112 113 // FIXME: This multiplier was not really tuned up. 114 static cl::opt<float> ImportColdMultiplier( 115 "import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"), 116 cl::desc("Multiply the `import-instr-limit` threshold for cold callsites")); 117 118 static cl::opt<bool> PrintImports("print-imports", cl::init(false), cl::Hidden, 119 cl::desc("Print imported functions")); 120 121 static cl::opt<bool> PrintImportFailures( 122 "print-import-failures", cl::init(false), cl::Hidden, 123 cl::desc("Print information for functions rejected for importing")); 124 125 static cl::opt<bool> ComputeDead("compute-dead", cl::init(true), cl::Hidden, 126 cl::desc("Compute dead symbols")); 127 128 static cl::opt<bool> EnableImportMetadata( 129 "enable-import-metadata", cl::init(false), cl::Hidden, 130 cl::desc("Enable import metadata like 'thinlto_src_module'")); 131 132 /// Summary file to use for function importing when using -function-import from 133 /// the command line. 134 static cl::opt<std::string> 135 SummaryFile("summary-file", 136 cl::desc("The summary file to use for function importing.")); 137 138 /// Used when testing importing from distributed indexes via opt 139 // -function-import. 140 static cl::opt<bool> 141 ImportAllIndex("import-all-index", 142 cl::desc("Import all external functions in index.")); 143 144 // Load lazily a module from \p FileName in \p Context. 145 static std::unique_ptr<Module> loadFile(const std::string &FileName, 146 LLVMContext &Context) { 147 SMDiagnostic Err; 148 LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n"); 149 // Metadata isn't loaded until functions are imported, to minimize 150 // the memory overhead. 151 std::unique_ptr<Module> Result = 152 getLazyIRFileModule(FileName, Err, Context, 153 /* ShouldLazyLoadMetadata = */ true); 154 if (!Result) { 155 Err.print("function-import", errs()); 156 report_fatal_error("Abort"); 157 } 158 159 return Result; 160 } 161 162 /// Given a list of possible callee implementation for a call site, select one 163 /// that fits the \p Threshold. 164 /// 165 /// FIXME: select "best" instead of first that fits. But what is "best"? 166 /// - The smallest: more likely to be inlined. 167 /// - The one with the least outgoing edges (already well optimized). 168 /// - One from a module already being imported from in order to reduce the 169 /// number of source modules parsed/linked. 170 /// - One that has PGO data attached. 171 /// - [insert you fancy metric here] 172 static const GlobalValueSummary * 173 selectCallee(const ModuleSummaryIndex &Index, 174 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList, 175 unsigned Threshold, StringRef CallerModulePath, 176 FunctionImporter::ImportFailureReason &Reason, 177 GlobalValue::GUID GUID) { 178 Reason = FunctionImporter::ImportFailureReason::None; 179 auto It = llvm::find_if( 180 CalleeSummaryList, 181 [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) { 182 auto *GVSummary = SummaryPtr.get(); 183 if (!Index.isGlobalValueLive(GVSummary)) { 184 Reason = FunctionImporter::ImportFailureReason::NotLive; 185 return false; 186 } 187 188 if (GlobalValue::isInterposableLinkage(GVSummary->linkage())) { 189 Reason = FunctionImporter::ImportFailureReason::InterposableLinkage; 190 // There is no point in importing these, we can't inline them 191 return false; 192 } 193 194 auto *Summary = cast<FunctionSummary>(GVSummary->getBaseObject()); 195 196 // If this is a local function, make sure we import the copy 197 // in the caller's module. The only time a local function can 198 // share an entry in the index is if there is a local with the same name 199 // in another module that had the same source file name (in a different 200 // directory), where each was compiled in their own directory so there 201 // was not distinguishing path. 202 // However, do the import from another module if there is only one 203 // entry in the list - in that case this must be a reference due 204 // to indirect call profile data, since a function pointer can point to 205 // a local in another module. 206 if (GlobalValue::isLocalLinkage(Summary->linkage()) && 207 CalleeSummaryList.size() > 1 && 208 Summary->modulePath() != CallerModulePath) { 209 Reason = 210 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule; 211 return false; 212 } 213 214 if ((Summary->instCount() > Threshold) && 215 !Summary->fflags().AlwaysInline && !ForceImportAll) { 216 Reason = FunctionImporter::ImportFailureReason::TooLarge; 217 return false; 218 } 219 220 // Skip if it isn't legal to import (e.g. may reference unpromotable 221 // locals). 222 if (Summary->notEligibleToImport()) { 223 Reason = FunctionImporter::ImportFailureReason::NotEligible; 224 return false; 225 } 226 227 // Don't bother importing if we can't inline it anyway. 228 if (Summary->fflags().NoInline && !ForceImportAll) { 229 Reason = FunctionImporter::ImportFailureReason::NoInline; 230 return false; 231 } 232 233 return true; 234 }); 235 if (It == CalleeSummaryList.end()) 236 return nullptr; 237 238 return cast<GlobalValueSummary>(It->get()); 239 } 240 241 namespace { 242 243 using EdgeInfo = 244 std::tuple<const GlobalValueSummary *, unsigned /* Threshold */>; 245 246 } // anonymous namespace 247 248 static bool shouldImportGlobal(const ValueInfo &VI, 249 const GVSummaryMapTy &DefinedGVSummaries) { 250 const auto &GVS = DefinedGVSummaries.find(VI.getGUID()); 251 if (GVS == DefinedGVSummaries.end()) 252 return true; 253 // We should not skip import if the module contains a definition with 254 // interposable linkage type. This is required for correctness in 255 // the situation with two following conditions: 256 // * the def with interposable linkage is non-prevailing, 257 // * there is a prevailing def available for import and marked read-only. 258 // In this case, the non-prevailing def will be converted to a declaration, 259 // while the prevailing one becomes internal, thus no definitions will be 260 // available for linking. In order to prevent undefined symbol link error, 261 // the prevailing definition must be imported. 262 // FIXME: Consider adding a check that the suitable prevailing definition 263 // exists and marked read-only. 264 if (VI.getSummaryList().size() > 1 && 265 GlobalValue::isInterposableLinkage(GVS->second->linkage())) 266 return true; 267 268 return false; 269 } 270 271 static void computeImportForReferencedGlobals( 272 const GlobalValueSummary &Summary, const ModuleSummaryIndex &Index, 273 const GVSummaryMapTy &DefinedGVSummaries, 274 SmallVectorImpl<EdgeInfo> &Worklist, 275 FunctionImporter::ImportMapTy &ImportList, 276 StringMap<FunctionImporter::ExportSetTy> *ExportLists) { 277 for (const auto &VI : Summary.refs()) { 278 if (!shouldImportGlobal(VI, DefinedGVSummaries)) { 279 LLVM_DEBUG( 280 dbgs() << "Ref ignored! Target already in destination module.\n"); 281 continue; 282 } 283 284 LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n"); 285 286 // If this is a local variable, make sure we import the copy 287 // in the caller's module. The only time a local variable can 288 // share an entry in the index is if there is a local with the same name 289 // in another module that had the same source file name (in a different 290 // directory), where each was compiled in their own directory so there 291 // was not distinguishing path. 292 auto LocalNotInModule = [&](const GlobalValueSummary *RefSummary) -> bool { 293 return GlobalValue::isLocalLinkage(RefSummary->linkage()) && 294 RefSummary->modulePath() != Summary.modulePath(); 295 }; 296 297 for (const auto &RefSummary : VI.getSummaryList()) 298 if (isa<GlobalVarSummary>(RefSummary.get()) && 299 Index.canImportGlobalVar(RefSummary.get(), /* AnalyzeRefs */ true) && 300 !LocalNotInModule(RefSummary.get())) { 301 auto ILI = ImportList[RefSummary->modulePath()].insert(VI.getGUID()); 302 // Only update stat and exports if we haven't already imported this 303 // variable. 304 if (!ILI.second) 305 break; 306 NumImportedGlobalVarsThinLink++; 307 // Any references made by this variable will be marked exported later, 308 // in ComputeCrossModuleImport, after import decisions are complete, 309 // which is more efficient than adding them here. 310 if (ExportLists) 311 (*ExportLists)[RefSummary->modulePath()].insert(VI); 312 313 // If variable is not writeonly we attempt to recursively analyze 314 // its references in order to import referenced constants. 315 if (!Index.isWriteOnly(cast<GlobalVarSummary>(RefSummary.get()))) 316 Worklist.emplace_back(RefSummary.get(), 0); 317 break; 318 } 319 } 320 } 321 322 static const char * 323 getFailureName(FunctionImporter::ImportFailureReason Reason) { 324 switch (Reason) { 325 case FunctionImporter::ImportFailureReason::None: 326 return "None"; 327 case FunctionImporter::ImportFailureReason::GlobalVar: 328 return "GlobalVar"; 329 case FunctionImporter::ImportFailureReason::NotLive: 330 return "NotLive"; 331 case FunctionImporter::ImportFailureReason::TooLarge: 332 return "TooLarge"; 333 case FunctionImporter::ImportFailureReason::InterposableLinkage: 334 return "InterposableLinkage"; 335 case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule: 336 return "LocalLinkageNotInModule"; 337 case FunctionImporter::ImportFailureReason::NotEligible: 338 return "NotEligible"; 339 case FunctionImporter::ImportFailureReason::NoInline: 340 return "NoInline"; 341 } 342 llvm_unreachable("invalid reason"); 343 } 344 345 /// Compute the list of functions to import for a given caller. Mark these 346 /// imported functions and the symbols they reference in their source module as 347 /// exported from their source module. 348 static void computeImportForFunction( 349 const FunctionSummary &Summary, const ModuleSummaryIndex &Index, 350 const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries, 351 SmallVectorImpl<EdgeInfo> &Worklist, 352 FunctionImporter::ImportMapTy &ImportList, 353 StringMap<FunctionImporter::ExportSetTy> *ExportLists, 354 FunctionImporter::ImportThresholdsTy &ImportThresholds) { 355 computeImportForReferencedGlobals(Summary, Index, DefinedGVSummaries, 356 Worklist, ImportList, ExportLists); 357 static int ImportCount = 0; 358 for (const auto &Edge : Summary.calls()) { 359 ValueInfo VI = Edge.first; 360 LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold 361 << "\n"); 362 363 if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) { 364 LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff 365 << " reached.\n"); 366 continue; 367 } 368 369 if (DefinedGVSummaries.count(VI.getGUID())) { 370 // FIXME: Consider not skipping import if the module contains 371 // a non-prevailing def with interposable linkage. The prevailing copy 372 // can safely be imported (see shouldImportGlobal()). 373 LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n"); 374 continue; 375 } 376 377 auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float { 378 if (Hotness == CalleeInfo::HotnessType::Hot) 379 return ImportHotMultiplier; 380 if (Hotness == CalleeInfo::HotnessType::Cold) 381 return ImportColdMultiplier; 382 if (Hotness == CalleeInfo::HotnessType::Critical) 383 return ImportCriticalMultiplier; 384 return 1.0; 385 }; 386 387 const auto NewThreshold = 388 Threshold * GetBonusMultiplier(Edge.second.getHotness()); 389 390 auto IT = ImportThresholds.insert(std::make_pair( 391 VI.getGUID(), std::make_tuple(NewThreshold, nullptr, nullptr))); 392 bool PreviouslyVisited = !IT.second; 393 auto &ProcessedThreshold = std::get<0>(IT.first->second); 394 auto &CalleeSummary = std::get<1>(IT.first->second); 395 auto &FailureInfo = std::get<2>(IT.first->second); 396 397 bool IsHotCallsite = 398 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot; 399 bool IsCriticalCallsite = 400 Edge.second.getHotness() == CalleeInfo::HotnessType::Critical; 401 402 const FunctionSummary *ResolvedCalleeSummary = nullptr; 403 if (CalleeSummary) { 404 assert(PreviouslyVisited); 405 // Since the traversal of the call graph is DFS, we can revisit a function 406 // a second time with a higher threshold. In this case, it is added back 407 // to the worklist with the new threshold (so that its own callee chains 408 // can be considered with the higher threshold). 409 if (NewThreshold <= ProcessedThreshold) { 410 LLVM_DEBUG( 411 dbgs() << "ignored! Target was already imported with Threshold " 412 << ProcessedThreshold << "\n"); 413 continue; 414 } 415 // Update with new larger threshold. 416 ProcessedThreshold = NewThreshold; 417 ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary); 418 } else { 419 // If we already rejected importing a callee at the same or higher 420 // threshold, don't waste time calling selectCallee. 421 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) { 422 LLVM_DEBUG( 423 dbgs() << "ignored! Target was already rejected with Threshold " 424 << ProcessedThreshold << "\n"); 425 if (PrintImportFailures) { 426 assert(FailureInfo && 427 "Expected FailureInfo for previously rejected candidate"); 428 FailureInfo->Attempts++; 429 } 430 continue; 431 } 432 433 FunctionImporter::ImportFailureReason Reason; 434 CalleeSummary = selectCallee(Index, VI.getSummaryList(), NewThreshold, 435 Summary.modulePath(), Reason, VI.getGUID()); 436 if (!CalleeSummary) { 437 // Update with new larger threshold if this was a retry (otherwise 438 // we would have already inserted with NewThreshold above). Also 439 // update failure info if requested. 440 if (PreviouslyVisited) { 441 ProcessedThreshold = NewThreshold; 442 if (PrintImportFailures) { 443 assert(FailureInfo && 444 "Expected FailureInfo for previously rejected candidate"); 445 FailureInfo->Reason = Reason; 446 FailureInfo->Attempts++; 447 FailureInfo->MaxHotness = 448 std::max(FailureInfo->MaxHotness, Edge.second.getHotness()); 449 } 450 } else if (PrintImportFailures) { 451 assert(!FailureInfo && 452 "Expected no FailureInfo for newly rejected candidate"); 453 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>( 454 VI, Edge.second.getHotness(), Reason, 1); 455 } 456 if (ForceImportAll) { 457 std::string Msg = std::string("Failed to import function ") + 458 VI.name().str() + " due to " + 459 getFailureName(Reason); 460 auto Error = make_error<StringError>( 461 Msg, make_error_code(errc::not_supported)); 462 logAllUnhandledErrors(std::move(Error), errs(), 463 "Error importing module: "); 464 break; 465 } else { 466 LLVM_DEBUG(dbgs() 467 << "ignored! No qualifying callee with summary found.\n"); 468 continue; 469 } 470 } 471 472 // "Resolve" the summary 473 CalleeSummary = CalleeSummary->getBaseObject(); 474 ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary); 475 476 assert((ResolvedCalleeSummary->fflags().AlwaysInline || ForceImportAll || 477 (ResolvedCalleeSummary->instCount() <= NewThreshold)) && 478 "selectCallee() didn't honor the threshold"); 479 480 auto ExportModulePath = ResolvedCalleeSummary->modulePath(); 481 auto ILI = ImportList[ExportModulePath].insert(VI.getGUID()); 482 // We previously decided to import this GUID definition if it was already 483 // inserted in the set of imports from the exporting module. 484 bool PreviouslyImported = !ILI.second; 485 if (!PreviouslyImported) { 486 NumImportedFunctionsThinLink++; 487 if (IsHotCallsite) 488 NumImportedHotFunctionsThinLink++; 489 if (IsCriticalCallsite) 490 NumImportedCriticalFunctionsThinLink++; 491 } 492 493 // Any calls/references made by this function will be marked exported 494 // later, in ComputeCrossModuleImport, after import decisions are 495 // complete, which is more efficient than adding them here. 496 if (ExportLists) 497 (*ExportLists)[ExportModulePath].insert(VI); 498 } 499 500 auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) { 501 // Adjust the threshold for next level of imported functions. 502 // The threshold is different for hot callsites because we can then 503 // inline chains of hot calls. 504 if (IsHotCallsite) 505 return Threshold * ImportHotInstrFactor; 506 return Threshold * ImportInstrFactor; 507 }; 508 509 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite); 510 511 ImportCount++; 512 513 // Insert the newly imported function to the worklist. 514 Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold); 515 } 516 } 517 518 /// Given the list of globals defined in a module, compute the list of imports 519 /// as well as the list of "exports", i.e. the list of symbols referenced from 520 /// another module (that may require promotion). 521 static void ComputeImportForModule( 522 const GVSummaryMapTy &DefinedGVSummaries, const ModuleSummaryIndex &Index, 523 StringRef ModName, FunctionImporter::ImportMapTy &ImportList, 524 StringMap<FunctionImporter::ExportSetTy> *ExportLists = nullptr) { 525 // Worklist contains the list of function imported in this module, for which 526 // we will analyse the callees and may import further down the callgraph. 527 SmallVector<EdgeInfo, 128> Worklist; 528 FunctionImporter::ImportThresholdsTy ImportThresholds; 529 530 // Populate the worklist with the import for the functions in the current 531 // module 532 for (const auto &GVSummary : DefinedGVSummaries) { 533 #ifndef NDEBUG 534 // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID 535 // so this map look up (and possibly others) can be avoided. 536 auto VI = Index.getValueInfo(GVSummary.first); 537 #endif 538 if (!Index.isGlobalValueLive(GVSummary.second)) { 539 LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n"); 540 continue; 541 } 542 auto *FuncSummary = 543 dyn_cast<FunctionSummary>(GVSummary.second->getBaseObject()); 544 if (!FuncSummary) 545 // Skip import for global variables 546 continue; 547 LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n"); 548 computeImportForFunction(*FuncSummary, Index, ImportInstrLimit, 549 DefinedGVSummaries, Worklist, ImportList, 550 ExportLists, ImportThresholds); 551 } 552 553 // Process the newly imported functions and add callees to the worklist. 554 while (!Worklist.empty()) { 555 auto GVInfo = Worklist.pop_back_val(); 556 auto *Summary = std::get<0>(GVInfo); 557 auto Threshold = std::get<1>(GVInfo); 558 559 if (auto *FS = dyn_cast<FunctionSummary>(Summary)) 560 computeImportForFunction(*FS, Index, Threshold, DefinedGVSummaries, 561 Worklist, ImportList, ExportLists, 562 ImportThresholds); 563 else 564 computeImportForReferencedGlobals(*Summary, Index, DefinedGVSummaries, 565 Worklist, ImportList, ExportLists); 566 } 567 568 // Print stats about functions considered but rejected for importing 569 // when requested. 570 if (PrintImportFailures) { 571 dbgs() << "Missed imports into module " << ModName << "\n"; 572 for (auto &I : ImportThresholds) { 573 auto &ProcessedThreshold = std::get<0>(I.second); 574 auto &CalleeSummary = std::get<1>(I.second); 575 auto &FailureInfo = std::get<2>(I.second); 576 if (CalleeSummary) 577 continue; // We are going to import. 578 assert(FailureInfo); 579 FunctionSummary *FS = nullptr; 580 if (!FailureInfo->VI.getSummaryList().empty()) 581 FS = dyn_cast<FunctionSummary>( 582 FailureInfo->VI.getSummaryList()[0]->getBaseObject()); 583 dbgs() << FailureInfo->VI 584 << ": Reason = " << getFailureName(FailureInfo->Reason) 585 << ", Threshold = " << ProcessedThreshold 586 << ", Size = " << (FS ? (int)FS->instCount() : -1) 587 << ", MaxHotness = " << getHotnessName(FailureInfo->MaxHotness) 588 << ", Attempts = " << FailureInfo->Attempts << "\n"; 589 } 590 } 591 } 592 593 #ifndef NDEBUG 594 static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI) { 595 auto SL = VI.getSummaryList(); 596 return SL.empty() 597 ? false 598 : SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind; 599 } 600 601 static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, 602 GlobalValue::GUID G) { 603 if (const auto &VI = Index.getValueInfo(G)) 604 return isGlobalVarSummary(Index, VI); 605 return false; 606 } 607 608 template <class T> 609 static unsigned numGlobalVarSummaries(const ModuleSummaryIndex &Index, 610 T &Cont) { 611 unsigned NumGVS = 0; 612 for (auto &V : Cont) 613 if (isGlobalVarSummary(Index, V)) 614 ++NumGVS; 615 return NumGVS; 616 } 617 #endif 618 619 #ifndef NDEBUG 620 static bool 621 checkVariableImport(const ModuleSummaryIndex &Index, 622 StringMap<FunctionImporter::ImportMapTy> &ImportLists, 623 StringMap<FunctionImporter::ExportSetTy> &ExportLists) { 624 625 DenseSet<GlobalValue::GUID> FlattenedImports; 626 627 for (auto &ImportPerModule : ImportLists) 628 for (auto &ExportPerModule : ImportPerModule.second) 629 FlattenedImports.insert(ExportPerModule.second.begin(), 630 ExportPerModule.second.end()); 631 632 // Checks that all GUIDs of read/writeonly vars we see in export lists 633 // are also in the import lists. Otherwise we my face linker undefs, 634 // because readonly and writeonly vars are internalized in their 635 // source modules. 636 auto IsReadOrWriteOnlyVar = [&](StringRef ModulePath, const ValueInfo &VI) { 637 auto *GVS = dyn_cast_or_null<GlobalVarSummary>( 638 Index.findSummaryInModule(VI, ModulePath)); 639 return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)); 640 }; 641 642 for (auto &ExportPerModule : ExportLists) 643 for (auto &VI : ExportPerModule.second) 644 if (!FlattenedImports.count(VI.getGUID()) && 645 IsReadOrWriteOnlyVar(ExportPerModule.first(), VI)) 646 return false; 647 648 return true; 649 } 650 #endif 651 652 /// Compute all the import and export for every module using the Index. 653 void llvm::ComputeCrossModuleImport( 654 const ModuleSummaryIndex &Index, 655 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, 656 StringMap<FunctionImporter::ImportMapTy> &ImportLists, 657 StringMap<FunctionImporter::ExportSetTy> &ExportLists) { 658 // For each module that has function defined, compute the import/export lists. 659 for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) { 660 auto &ImportList = ImportLists[DefinedGVSummaries.first()]; 661 LLVM_DEBUG(dbgs() << "Computing import for Module '" 662 << DefinedGVSummaries.first() << "'\n"); 663 ComputeImportForModule(DefinedGVSummaries.second, Index, 664 DefinedGVSummaries.first(), ImportList, 665 &ExportLists); 666 } 667 668 // When computing imports we only added the variables and functions being 669 // imported to the export list. We also need to mark any references and calls 670 // they make as exported as well. We do this here, as it is more efficient 671 // since we may import the same values multiple times into different modules 672 // during the import computation. 673 for (auto &ELI : ExportLists) { 674 FunctionImporter::ExportSetTy NewExports; 675 const auto &DefinedGVSummaries = 676 ModuleToDefinedGVSummaries.lookup(ELI.first()); 677 for (auto &EI : ELI.second) { 678 // Find the copy defined in the exporting module so that we can mark the 679 // values it references in that specific definition as exported. 680 // Below we will add all references and called values, without regard to 681 // whether they are also defined in this module. We subsequently prune the 682 // list to only include those defined in the exporting module, see comment 683 // there as to why. 684 auto DS = DefinedGVSummaries.find(EI.getGUID()); 685 // Anything marked exported during the import computation must have been 686 // defined in the exporting module. 687 assert(DS != DefinedGVSummaries.end()); 688 auto *S = DS->getSecond(); 689 S = S->getBaseObject(); 690 if (auto *GVS = dyn_cast<GlobalVarSummary>(S)) { 691 // Export referenced functions and variables. We don't export/promote 692 // objects referenced by writeonly variable initializer, because 693 // we convert such variables initializers to "zeroinitializer". 694 // See processGlobalForThinLTO. 695 if (!Index.isWriteOnly(GVS)) 696 for (const auto &VI : GVS->refs()) 697 NewExports.insert(VI); 698 } else { 699 auto *FS = cast<FunctionSummary>(S); 700 for (const auto &Edge : FS->calls()) 701 NewExports.insert(Edge.first); 702 for (const auto &Ref : FS->refs()) 703 NewExports.insert(Ref); 704 } 705 } 706 // Prune list computed above to only include values defined in the exporting 707 // module. We do this after the above insertion since we may hit the same 708 // ref/call target multiple times in above loop, and it is more efficient to 709 // avoid a set lookup each time. 710 for (auto EI = NewExports.begin(); EI != NewExports.end();) { 711 if (!DefinedGVSummaries.count(EI->getGUID())) 712 NewExports.erase(EI++); 713 else 714 ++EI; 715 } 716 ELI.second.insert(NewExports.begin(), NewExports.end()); 717 } 718 719 assert(checkVariableImport(Index, ImportLists, ExportLists)); 720 #ifndef NDEBUG 721 LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size() 722 << " modules:\n"); 723 for (auto &ModuleImports : ImportLists) { 724 auto ModName = ModuleImports.first(); 725 auto &Exports = ExportLists[ModName]; 726 unsigned NumGVS = numGlobalVarSummaries(Index, Exports); 727 LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports " 728 << Exports.size() - NumGVS << " functions and " << NumGVS 729 << " vars. Imports from " << ModuleImports.second.size() 730 << " modules.\n"); 731 for (auto &Src : ModuleImports.second) { 732 auto SrcModName = Src.first(); 733 unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second); 734 LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod 735 << " functions imported from " << SrcModName << "\n"); 736 LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod 737 << " global vars imported from " << SrcModName << "\n"); 738 } 739 } 740 #endif 741 } 742 743 #ifndef NDEBUG 744 static void dumpImportListForModule(const ModuleSummaryIndex &Index, 745 StringRef ModulePath, 746 FunctionImporter::ImportMapTy &ImportList) { 747 LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from " 748 << ImportList.size() << " modules.\n"); 749 for (auto &Src : ImportList) { 750 auto SrcModName = Src.first(); 751 unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second); 752 LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod 753 << " functions imported from " << SrcModName << "\n"); 754 LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod << " vars imported from " 755 << SrcModName << "\n"); 756 } 757 } 758 #endif 759 760 /// Compute all the imports for the given module in the Index. 761 void llvm::ComputeCrossModuleImportForModule( 762 StringRef ModulePath, const ModuleSummaryIndex &Index, 763 FunctionImporter::ImportMapTy &ImportList) { 764 // Collect the list of functions this module defines. 765 // GUID -> Summary 766 GVSummaryMapTy FunctionSummaryMap; 767 Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap); 768 769 // Compute the import list for this module. 770 LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n"); 771 ComputeImportForModule(FunctionSummaryMap, Index, ModulePath, ImportList); 772 773 #ifndef NDEBUG 774 dumpImportListForModule(Index, ModulePath, ImportList); 775 #endif 776 } 777 778 // Mark all external summaries in Index for import into the given module. 779 // Used for distributed builds using a distributed index. 780 void llvm::ComputeCrossModuleImportForModuleFromIndex( 781 StringRef ModulePath, const ModuleSummaryIndex &Index, 782 FunctionImporter::ImportMapTy &ImportList) { 783 for (const auto &GlobalList : Index) { 784 // Ignore entries for undefined references. 785 if (GlobalList.second.SummaryList.empty()) 786 continue; 787 788 auto GUID = GlobalList.first; 789 assert(GlobalList.second.SummaryList.size() == 1 && 790 "Expected individual combined index to have one summary per GUID"); 791 auto &Summary = GlobalList.second.SummaryList[0]; 792 // Skip the summaries for the importing module. These are included to 793 // e.g. record required linkage changes. 794 if (Summary->modulePath() == ModulePath) 795 continue; 796 // Add an entry to provoke importing by thinBackend. 797 ImportList[Summary->modulePath()].insert(GUID); 798 } 799 #ifndef NDEBUG 800 dumpImportListForModule(Index, ModulePath, ImportList); 801 #endif 802 } 803 804 // For SamplePGO, the indirect call targets for local functions will 805 // have its original name annotated in profile. We try to find the 806 // corresponding PGOFuncName as the GUID, and fix up the edges 807 // accordingly. 808 void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index, 809 FunctionSummary *FS) { 810 for (auto &EI : FS->mutableCalls()) { 811 if (!EI.first.getSummaryList().empty()) 812 continue; 813 auto GUID = Index.getGUIDFromOriginalID(EI.first.getGUID()); 814 if (GUID == 0) 815 continue; 816 // Update the edge to point directly to the correct GUID. 817 auto VI = Index.getValueInfo(GUID); 818 if (llvm::any_of( 819 VI.getSummaryList(), 820 [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) { 821 // The mapping from OriginalId to GUID may return a GUID 822 // that corresponds to a static variable. Filter it out here. 823 // This can happen when 824 // 1) There is a call to a library function which is not defined 825 // in the index. 826 // 2) There is a static variable with the OriginalGUID identical 827 // to the GUID of the library function in 1); 828 // When this happens the static variable in 2) will be found, 829 // which needs to be filtered out. 830 return SummaryPtr->getSummaryKind() == 831 GlobalValueSummary::GlobalVarKind; 832 })) 833 continue; 834 EI.first = VI; 835 } 836 } 837 838 void llvm::updateIndirectCalls(ModuleSummaryIndex &Index) { 839 for (const auto &Entry : Index) { 840 for (const auto &S : Entry.second.SummaryList) { 841 if (auto *FS = dyn_cast<FunctionSummary>(S.get())) 842 updateValueInfoForIndirectCalls(Index, FS); 843 } 844 } 845 } 846 847 void llvm::computeDeadSymbolsAndUpdateIndirectCalls( 848 ModuleSummaryIndex &Index, 849 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols, 850 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) { 851 assert(!Index.withGlobalValueDeadStripping()); 852 if (!ComputeDead || 853 // Don't do anything when nothing is live, this is friendly with tests. 854 GUIDPreservedSymbols.empty()) { 855 // Still need to update indirect calls. 856 updateIndirectCalls(Index); 857 return; 858 } 859 unsigned LiveSymbols = 0; 860 SmallVector<ValueInfo, 128> Worklist; 861 Worklist.reserve(GUIDPreservedSymbols.size() * 2); 862 for (auto GUID : GUIDPreservedSymbols) { 863 ValueInfo VI = Index.getValueInfo(GUID); 864 if (!VI) 865 continue; 866 for (const auto &S : VI.getSummaryList()) 867 S->setLive(true); 868 } 869 870 // Add values flagged in the index as live roots to the worklist. 871 for (const auto &Entry : Index) { 872 auto VI = Index.getValueInfo(Entry); 873 for (const auto &S : Entry.second.SummaryList) { 874 if (auto *FS = dyn_cast<FunctionSummary>(S.get())) 875 updateValueInfoForIndirectCalls(Index, FS); 876 if (S->isLive()) { 877 LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n"); 878 Worklist.push_back(VI); 879 ++LiveSymbols; 880 break; 881 } 882 } 883 } 884 885 // Make value live and add it to the worklist if it was not live before. 886 auto visit = [&](ValueInfo VI, bool IsAliasee) { 887 // FIXME: If we knew which edges were created for indirect call profiles, 888 // we could skip them here. Any that are live should be reached via 889 // other edges, e.g. reference edges. Otherwise, using a profile collected 890 // on a slightly different binary might provoke preserving, importing 891 // and ultimately promoting calls to functions not linked into this 892 // binary, which increases the binary size unnecessarily. Note that 893 // if this code changes, the importer needs to change so that edges 894 // to functions marked dead are skipped. 895 896 if (llvm::any_of(VI.getSummaryList(), 897 [](const std::unique_ptr<llvm::GlobalValueSummary> &S) { 898 return S->isLive(); 899 })) 900 return; 901 902 // We only keep live symbols that are known to be non-prevailing if any are 903 // available_externally, linkonceodr, weakodr. Those symbols are discarded 904 // later in the EliminateAvailableExternally pass and setting them to 905 // not-live could break downstreams users of liveness information (PR36483) 906 // or limit optimization opportunities. 907 if (isPrevailing(VI.getGUID()) == PrevailingType::No) { 908 bool KeepAliveLinkage = false; 909 bool Interposable = false; 910 for (const auto &S : VI.getSummaryList()) { 911 if (S->linkage() == GlobalValue::AvailableExternallyLinkage || 912 S->linkage() == GlobalValue::WeakODRLinkage || 913 S->linkage() == GlobalValue::LinkOnceODRLinkage) 914 KeepAliveLinkage = true; 915 else if (GlobalValue::isInterposableLinkage(S->linkage())) 916 Interposable = true; 917 } 918 919 if (!IsAliasee) { 920 if (!KeepAliveLinkage) 921 return; 922 923 if (Interposable) 924 report_fatal_error( 925 "Interposable and available_externally/linkonce_odr/weak_odr " 926 "symbol"); 927 } 928 } 929 930 for (const auto &S : VI.getSummaryList()) 931 S->setLive(true); 932 ++LiveSymbols; 933 Worklist.push_back(VI); 934 }; 935 936 while (!Worklist.empty()) { 937 auto VI = Worklist.pop_back_val(); 938 for (const auto &Summary : VI.getSummaryList()) { 939 if (auto *AS = dyn_cast<AliasSummary>(Summary.get())) { 940 // If this is an alias, visit the aliasee VI to ensure that all copies 941 // are marked live and it is added to the worklist for further 942 // processing of its references. 943 visit(AS->getAliaseeVI(), true); 944 continue; 945 } 946 for (auto Ref : Summary->refs()) 947 visit(Ref, false); 948 if (auto *FS = dyn_cast<FunctionSummary>(Summary.get())) 949 for (auto Call : FS->calls()) 950 visit(Call.first, false); 951 } 952 } 953 Index.setWithGlobalValueDeadStripping(); 954 955 unsigned DeadSymbols = Index.size() - LiveSymbols; 956 LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols 957 << " symbols Dead \n"); 958 NumDeadSymbols += DeadSymbols; 959 NumLiveSymbols += LiveSymbols; 960 } 961 962 // Compute dead symbols and propagate constants in combined index. 963 void llvm::computeDeadSymbolsWithConstProp( 964 ModuleSummaryIndex &Index, 965 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols, 966 function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing, 967 bool ImportEnabled) { 968 computeDeadSymbolsAndUpdateIndirectCalls(Index, GUIDPreservedSymbols, 969 isPrevailing); 970 if (ImportEnabled) 971 Index.propagateAttributes(GUIDPreservedSymbols); 972 } 973 974 /// Compute the set of summaries needed for a ThinLTO backend compilation of 975 /// \p ModulePath. 976 void llvm::gatherImportedSummariesForModule( 977 StringRef ModulePath, 978 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, 979 const FunctionImporter::ImportMapTy &ImportList, 980 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) { 981 // Include all summaries from the importing module. 982 ModuleToSummariesForIndex[std::string(ModulePath)] = 983 ModuleToDefinedGVSummaries.lookup(ModulePath); 984 // Include summaries for imports. 985 for (const auto &ILI : ImportList) { 986 auto &SummariesForIndex = 987 ModuleToSummariesForIndex[std::string(ILI.first())]; 988 const auto &DefinedGVSummaries = 989 ModuleToDefinedGVSummaries.lookup(ILI.first()); 990 for (const auto &GI : ILI.second) { 991 const auto &DS = DefinedGVSummaries.find(GI); 992 assert(DS != DefinedGVSummaries.end() && 993 "Expected a defined summary for imported global value"); 994 SummariesForIndex[GI] = DS->second; 995 } 996 } 997 } 998 999 /// Emit the files \p ModulePath will import from into \p OutputFilename. 1000 std::error_code llvm::EmitImportsFiles( 1001 StringRef ModulePath, StringRef OutputFilename, 1002 const std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) { 1003 std::error_code EC; 1004 raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::OF_None); 1005 if (EC) 1006 return EC; 1007 for (const auto &ILI : ModuleToSummariesForIndex) 1008 // The ModuleToSummariesForIndex map includes an entry for the current 1009 // Module (needed for writing out the index files). We don't want to 1010 // include it in the imports file, however, so filter it out. 1011 if (ILI.first != ModulePath) 1012 ImportsOS << ILI.first << "\n"; 1013 return std::error_code(); 1014 } 1015 1016 bool llvm::convertToDeclaration(GlobalValue &GV) { 1017 LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName() 1018 << "\n"); 1019 if (Function *F = dyn_cast<Function>(&GV)) { 1020 F->deleteBody(); 1021 F->clearMetadata(); 1022 F->setComdat(nullptr); 1023 } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(&GV)) { 1024 V->setInitializer(nullptr); 1025 V->setLinkage(GlobalValue::ExternalLinkage); 1026 V->clearMetadata(); 1027 V->setComdat(nullptr); 1028 } else { 1029 GlobalValue *NewGV; 1030 if (GV.getValueType()->isFunctionTy()) 1031 NewGV = 1032 Function::Create(cast<FunctionType>(GV.getValueType()), 1033 GlobalValue::ExternalLinkage, GV.getAddressSpace(), 1034 "", GV.getParent()); 1035 else 1036 NewGV = 1037 new GlobalVariable(*GV.getParent(), GV.getValueType(), 1038 /*isConstant*/ false, GlobalValue::ExternalLinkage, 1039 /*init*/ nullptr, "", 1040 /*insertbefore*/ nullptr, GV.getThreadLocalMode(), 1041 GV.getType()->getAddressSpace()); 1042 NewGV->takeName(&GV); 1043 GV.replaceAllUsesWith(NewGV); 1044 return false; 1045 } 1046 if (!GV.isImplicitDSOLocal()) 1047 GV.setDSOLocal(false); 1048 return true; 1049 } 1050 1051 void llvm::thinLTOFinalizeInModule(Module &TheModule, 1052 const GVSummaryMapTy &DefinedGlobals, 1053 bool PropagateAttrs) { 1054 DenseSet<Comdat *> NonPrevailingComdats; 1055 auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) { 1056 // See if the global summary analysis computed a new resolved linkage. 1057 const auto &GS = DefinedGlobals.find(GV.getGUID()); 1058 if (GS == DefinedGlobals.end()) 1059 return; 1060 1061 if (Propagate) 1062 if (FunctionSummary *FS = dyn_cast<FunctionSummary>(GS->second)) { 1063 if (Function *F = dyn_cast<Function>(&GV)) { 1064 // TODO: propagate ReadNone and ReadOnly. 1065 if (FS->fflags().ReadNone && !F->doesNotAccessMemory()) 1066 F->setDoesNotAccessMemory(); 1067 1068 if (FS->fflags().ReadOnly && !F->onlyReadsMemory()) 1069 F->setOnlyReadsMemory(); 1070 1071 if (FS->fflags().NoRecurse && !F->doesNotRecurse()) 1072 F->setDoesNotRecurse(); 1073 1074 if (FS->fflags().NoUnwind && !F->doesNotThrow()) 1075 F->setDoesNotThrow(); 1076 } 1077 } 1078 1079 auto NewLinkage = GS->second->linkage(); 1080 if (GlobalValue::isLocalLinkage(GV.getLinkage()) || 1081 // Don't internalize anything here, because the code below 1082 // lacks necessary correctness checks. Leave this job to 1083 // LLVM 'internalize' pass. 1084 GlobalValue::isLocalLinkage(NewLinkage) || 1085 // In case it was dead and already converted to declaration. 1086 GV.isDeclaration()) 1087 return; 1088 1089 // Set the potentially more constraining visibility computed from summaries. 1090 // The DefaultVisibility condition is because older GlobalValueSummary does 1091 // not record DefaultVisibility and we don't want to change protected/hidden 1092 // to default. 1093 if (GS->second->getVisibility() != GlobalValue::DefaultVisibility) 1094 GV.setVisibility(GS->second->getVisibility()); 1095 1096 if (NewLinkage == GV.getLinkage()) 1097 return; 1098 1099 // Check for a non-prevailing def that has interposable linkage 1100 // (e.g. non-odr weak or linkonce). In that case we can't simply 1101 // convert to available_externally, since it would lose the 1102 // interposable property and possibly get inlined. Simply drop 1103 // the definition in that case. 1104 if (GlobalValue::isAvailableExternallyLinkage(NewLinkage) && 1105 GlobalValue::isInterposableLinkage(GV.getLinkage())) { 1106 if (!convertToDeclaration(GV)) 1107 // FIXME: Change this to collect replaced GVs and later erase 1108 // them from the parent module once thinLTOResolvePrevailingGUID is 1109 // changed to enable this for aliases. 1110 llvm_unreachable("Expected GV to be converted"); 1111 } else { 1112 // If all copies of the original symbol had global unnamed addr and 1113 // linkonce_odr linkage, or if all of them had local unnamed addr linkage 1114 // and are constants, then it should be an auto hide symbol. In that case 1115 // the thin link would have marked it as CanAutoHide. Add hidden 1116 // visibility to the symbol to preserve the property. 1117 if (NewLinkage == GlobalValue::WeakODRLinkage && 1118 GS->second->canAutoHide()) { 1119 assert(GV.canBeOmittedFromSymbolTable()); 1120 GV.setVisibility(GlobalValue::HiddenVisibility); 1121 } 1122 1123 LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() 1124 << "` from " << GV.getLinkage() << " to " << NewLinkage 1125 << "\n"); 1126 GV.setLinkage(NewLinkage); 1127 } 1128 // Remove declarations from comdats, including available_externally 1129 // as this is a declaration for the linker, and will be dropped eventually. 1130 // It is illegal for comdats to contain declarations. 1131 auto *GO = dyn_cast_or_null<GlobalObject>(&GV); 1132 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) { 1133 if (GO->getComdat()->getName() == GO->getName()) 1134 NonPrevailingComdats.insert(GO->getComdat()); 1135 GO->setComdat(nullptr); 1136 } 1137 }; 1138 1139 // Process functions and global now 1140 for (auto &GV : TheModule) 1141 FinalizeInModule(GV, PropagateAttrs); 1142 for (auto &GV : TheModule.globals()) 1143 FinalizeInModule(GV); 1144 for (auto &GV : TheModule.aliases()) 1145 FinalizeInModule(GV); 1146 1147 // For a non-prevailing comdat, all its members must be available_externally. 1148 // FinalizeInModule has handled non-local-linkage GlobalValues. Here we handle 1149 // local linkage GlobalValues. 1150 if (NonPrevailingComdats.empty()) 1151 return; 1152 for (auto &GO : TheModule.global_objects()) { 1153 if (auto *C = GO.getComdat(); C && NonPrevailingComdats.count(C)) { 1154 GO.setComdat(nullptr); 1155 GO.setLinkage(GlobalValue::AvailableExternallyLinkage); 1156 } 1157 } 1158 bool Changed; 1159 do { 1160 Changed = false; 1161 // If an alias references a GlobalValue in a non-prevailing comdat, change 1162 // it to available_externally. For simplicity we only handle GlobalValue and 1163 // ConstantExpr with a base object. ConstantExpr without a base object is 1164 // unlikely used in a COMDAT. 1165 for (auto &GA : TheModule.aliases()) { 1166 if (GA.hasAvailableExternallyLinkage()) 1167 continue; 1168 GlobalObject *Obj = GA.getAliaseeObject(); 1169 assert(Obj && "aliasee without an base object is unimplemented"); 1170 if (Obj->hasAvailableExternallyLinkage()) { 1171 GA.setLinkage(GlobalValue::AvailableExternallyLinkage); 1172 Changed = true; 1173 } 1174 } 1175 } while (Changed); 1176 } 1177 1178 /// Run internalization on \p TheModule based on symmary analysis. 1179 void llvm::thinLTOInternalizeModule(Module &TheModule, 1180 const GVSummaryMapTy &DefinedGlobals) { 1181 // Declare a callback for the internalize pass that will ask for every 1182 // candidate GlobalValue if it can be internalized or not. 1183 auto MustPreserveGV = [&](const GlobalValue &GV) -> bool { 1184 // It may be the case that GV is on a chain of an ifunc, its alias and 1185 // subsequent aliases. In this case, the summary for the value is not 1186 // available. 1187 if (isa<GlobalIFunc>(&GV) || 1188 (isa<GlobalAlias>(&GV) && 1189 isa<GlobalIFunc>(cast<GlobalAlias>(&GV)->getAliaseeObject()))) 1190 return true; 1191 1192 // Lookup the linkage recorded in the summaries during global analysis. 1193 auto GS = DefinedGlobals.find(GV.getGUID()); 1194 if (GS == DefinedGlobals.end()) { 1195 // Must have been promoted (possibly conservatively). Find original 1196 // name so that we can access the correct summary and see if it can 1197 // be internalized again. 1198 // FIXME: Eventually we should control promotion instead of promoting 1199 // and internalizing again. 1200 StringRef OrigName = 1201 ModuleSummaryIndex::getOriginalNameBeforePromote(GV.getName()); 1202 std::string OrigId = GlobalValue::getGlobalIdentifier( 1203 OrigName, GlobalValue::InternalLinkage, 1204 TheModule.getSourceFileName()); 1205 GS = DefinedGlobals.find(GlobalValue::getGUID(OrigId)); 1206 if (GS == DefinedGlobals.end()) { 1207 // Also check the original non-promoted non-globalized name. In some 1208 // cases a preempted weak value is linked in as a local copy because 1209 // it is referenced by an alias (IRLinker::linkGlobalValueProto). 1210 // In that case, since it was originally not a local value, it was 1211 // recorded in the index using the original name. 1212 // FIXME: This may not be needed once PR27866 is fixed. 1213 GS = DefinedGlobals.find(GlobalValue::getGUID(OrigName)); 1214 assert(GS != DefinedGlobals.end()); 1215 } 1216 } 1217 return !GlobalValue::isLocalLinkage(GS->second->linkage()); 1218 }; 1219 1220 // FIXME: See if we can just internalize directly here via linkage changes 1221 // based on the index, rather than invoking internalizeModule. 1222 internalizeModule(TheModule, MustPreserveGV); 1223 } 1224 1225 /// Make alias a clone of its aliasee. 1226 static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) { 1227 Function *Fn = cast<Function>(GA->getAliaseeObject()); 1228 1229 ValueToValueMapTy VMap; 1230 Function *NewFn = CloneFunction(Fn, VMap); 1231 // Clone should use the original alias's linkage, visibility and name, and we 1232 // ensure all uses of alias instead use the new clone (casted if necessary). 1233 NewFn->setLinkage(GA->getLinkage()); 1234 NewFn->setVisibility(GA->getVisibility()); 1235 GA->replaceAllUsesWith(ConstantExpr::getBitCast(NewFn, GA->getType())); 1236 NewFn->takeName(GA); 1237 return NewFn; 1238 } 1239 1240 // Internalize values that we marked with specific attribute 1241 // in processGlobalForThinLTO. 1242 static void internalizeGVsAfterImport(Module &M) { 1243 for (auto &GV : M.globals()) 1244 // Skip GVs which have been converted to declarations 1245 // by dropDeadSymbols. 1246 if (!GV.isDeclaration() && GV.hasAttribute("thinlto-internalize")) { 1247 GV.setLinkage(GlobalValue::InternalLinkage); 1248 GV.setVisibility(GlobalValue::DefaultVisibility); 1249 } 1250 } 1251 1252 // Automatically import functions in Module \p DestModule based on the summaries 1253 // index. 1254 Expected<bool> FunctionImporter::importFunctions( 1255 Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) { 1256 LLVM_DEBUG(dbgs() << "Starting import for Module " 1257 << DestModule.getModuleIdentifier() << "\n"); 1258 unsigned ImportedCount = 0, ImportedGVCount = 0; 1259 1260 IRMover Mover(DestModule); 1261 // Do the actual import of functions now, one Module at a time 1262 std::set<StringRef> ModuleNameOrderedList; 1263 for (const auto &FunctionsToImportPerModule : ImportList) { 1264 ModuleNameOrderedList.insert(FunctionsToImportPerModule.first()); 1265 } 1266 for (const auto &Name : ModuleNameOrderedList) { 1267 // Get the module for the import 1268 const auto &FunctionsToImportPerModule = ImportList.find(Name); 1269 assert(FunctionsToImportPerModule != ImportList.end()); 1270 Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(Name); 1271 if (!SrcModuleOrErr) 1272 return SrcModuleOrErr.takeError(); 1273 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr); 1274 assert(&DestModule.getContext() == &SrcModule->getContext() && 1275 "Context mismatch"); 1276 1277 // If modules were created with lazy metadata loading, materialize it 1278 // now, before linking it (otherwise this will be a noop). 1279 if (Error Err = SrcModule->materializeMetadata()) 1280 return std::move(Err); 1281 1282 auto &ImportGUIDs = FunctionsToImportPerModule->second; 1283 // Find the globals to import 1284 SetVector<GlobalValue *> GlobalsToImport; 1285 for (Function &F : *SrcModule) { 1286 if (!F.hasName()) 1287 continue; 1288 auto GUID = F.getGUID(); 1289 auto Import = ImportGUIDs.count(GUID); 1290 LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing function " 1291 << GUID << " " << F.getName() << " from " 1292 << SrcModule->getSourceFileName() << "\n"); 1293 if (Import) { 1294 if (Error Err = F.materialize()) 1295 return std::move(Err); 1296 if (EnableImportMetadata) { 1297 // Add 'thinlto_src_module' metadata for statistics and debugging. 1298 F.setMetadata( 1299 "thinlto_src_module", 1300 MDNode::get(DestModule.getContext(), 1301 {MDString::get(DestModule.getContext(), 1302 SrcModule->getSourceFileName())})); 1303 } 1304 GlobalsToImport.insert(&F); 1305 } 1306 } 1307 for (GlobalVariable &GV : SrcModule->globals()) { 1308 if (!GV.hasName()) 1309 continue; 1310 auto GUID = GV.getGUID(); 1311 auto Import = ImportGUIDs.count(GUID); 1312 LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing global " 1313 << GUID << " " << GV.getName() << " from " 1314 << SrcModule->getSourceFileName() << "\n"); 1315 if (Import) { 1316 if (Error Err = GV.materialize()) 1317 return std::move(Err); 1318 ImportedGVCount += GlobalsToImport.insert(&GV); 1319 } 1320 } 1321 for (GlobalAlias &GA : SrcModule->aliases()) { 1322 if (!GA.hasName() || isa<GlobalIFunc>(GA.getAliaseeObject())) 1323 continue; 1324 auto GUID = GA.getGUID(); 1325 auto Import = ImportGUIDs.count(GUID); 1326 LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing alias " 1327 << GUID << " " << GA.getName() << " from " 1328 << SrcModule->getSourceFileName() << "\n"); 1329 if (Import) { 1330 if (Error Err = GA.materialize()) 1331 return std::move(Err); 1332 // Import alias as a copy of its aliasee. 1333 GlobalObject *GO = GA.getAliaseeObject(); 1334 if (Error Err = GO->materialize()) 1335 return std::move(Err); 1336 auto *Fn = replaceAliasWithAliasee(SrcModule.get(), &GA); 1337 LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << GO->getGUID() << " " 1338 << GO->getName() << " from " 1339 << SrcModule->getSourceFileName() << "\n"); 1340 if (EnableImportMetadata) { 1341 // Add 'thinlto_src_module' metadata for statistics and debugging. 1342 Fn->setMetadata( 1343 "thinlto_src_module", 1344 MDNode::get(DestModule.getContext(), 1345 {MDString::get(DestModule.getContext(), 1346 SrcModule->getSourceFileName())})); 1347 } 1348 GlobalsToImport.insert(Fn); 1349 } 1350 } 1351 1352 // Upgrade debug info after we're done materializing all the globals and we 1353 // have loaded all the required metadata! 1354 UpgradeDebugInfo(*SrcModule); 1355 1356 // Set the partial sample profile ratio in the profile summary module flag 1357 // of the imported source module, if applicable, so that the profile summary 1358 // module flag will match with that of the destination module when it's 1359 // imported. 1360 SrcModule->setPartialSampleProfileRatio(Index); 1361 1362 // Link in the specified functions. 1363 if (renameModuleForThinLTO(*SrcModule, Index, ClearDSOLocalOnDeclarations, 1364 &GlobalsToImport)) 1365 return true; 1366 1367 if (PrintImports) { 1368 for (const auto *GV : GlobalsToImport) 1369 dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName() 1370 << " from " << SrcModule->getSourceFileName() << "\n"; 1371 } 1372 1373 if (Error Err = Mover.move(std::move(SrcModule), 1374 GlobalsToImport.getArrayRef(), nullptr, 1375 /*IsPerformingImport=*/true)) 1376 report_fatal_error(Twine("Function Import: link error: ") + 1377 toString(std::move(Err))); 1378 1379 ImportedCount += GlobalsToImport.size(); 1380 NumImportedModules++; 1381 } 1382 1383 internalizeGVsAfterImport(DestModule); 1384 1385 NumImportedFunctions += (ImportedCount - ImportedGVCount); 1386 NumImportedGlobalVars += ImportedGVCount; 1387 1388 LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount 1389 << " functions for Module " 1390 << DestModule.getModuleIdentifier() << "\n"); 1391 LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount 1392 << " global variables for Module " 1393 << DestModule.getModuleIdentifier() << "\n"); 1394 return ImportedCount; 1395 } 1396 1397 static bool doImportingForModule(Module &M) { 1398 if (SummaryFile.empty()) 1399 report_fatal_error("error: -function-import requires -summary-file\n"); 1400 Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr = 1401 getModuleSummaryIndexForFile(SummaryFile); 1402 if (!IndexPtrOrErr) { 1403 logAllUnhandledErrors(IndexPtrOrErr.takeError(), errs(), 1404 "Error loading file '" + SummaryFile + "': "); 1405 return false; 1406 } 1407 std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr); 1408 1409 // First step is collecting the import list. 1410 FunctionImporter::ImportMapTy ImportList; 1411 // If requested, simply import all functions in the index. This is used 1412 // when testing distributed backend handling via the opt tool, when 1413 // we have distributed indexes containing exactly the summaries to import. 1414 if (ImportAllIndex) 1415 ComputeCrossModuleImportForModuleFromIndex(M.getModuleIdentifier(), *Index, 1416 ImportList); 1417 else 1418 ComputeCrossModuleImportForModule(M.getModuleIdentifier(), *Index, 1419 ImportList); 1420 1421 // Conservatively mark all internal values as promoted. This interface is 1422 // only used when doing importing via the function importing pass. The pass 1423 // is only enabled when testing importing via the 'opt' tool, which does 1424 // not do the ThinLink that would normally determine what values to promote. 1425 for (auto &I : *Index) { 1426 for (auto &S : I.second.SummaryList) { 1427 if (GlobalValue::isLocalLinkage(S->linkage())) 1428 S->setLinkage(GlobalValue::ExternalLinkage); 1429 } 1430 } 1431 1432 // Next we need to promote to global scope and rename any local values that 1433 // are potentially exported to other modules. 1434 if (renameModuleForThinLTO(M, *Index, /*ClearDSOLocalOnDeclarations=*/false, 1435 /*GlobalsToImport=*/nullptr)) { 1436 errs() << "Error renaming module\n"; 1437 return false; 1438 } 1439 1440 // Perform the import now. 1441 auto ModuleLoader = [&M](StringRef Identifier) { 1442 return loadFile(std::string(Identifier), M.getContext()); 1443 }; 1444 FunctionImporter Importer(*Index, ModuleLoader, 1445 /*ClearDSOLocalOnDeclarations=*/false); 1446 Expected<bool> Result = Importer.importFunctions(M, ImportList); 1447 1448 // FIXME: Probably need to propagate Errors through the pass manager. 1449 if (!Result) { 1450 logAllUnhandledErrors(Result.takeError(), errs(), 1451 "Error importing module: "); 1452 return false; 1453 } 1454 1455 return *Result; 1456 } 1457 1458 PreservedAnalyses FunctionImportPass::run(Module &M, 1459 ModuleAnalysisManager &AM) { 1460 if (!doImportingForModule(M)) 1461 return PreservedAnalyses::all(); 1462 1463 return PreservedAnalyses::none(); 1464 } 1465