1 //===- LTO.cpp ------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "LTO.h" 10 #include "Config.h" 11 #include "InputFiles.h" 12 #include "SymbolTable.h" 13 #include "Symbols.h" 14 #include "lld/Common/Args.h" 15 #include "lld/Common/ErrorHandler.h" 16 #include "lld/Common/Strings.h" 17 #include "lld/Common/TargetOptionsCommandFlags.h" 18 #include "llvm/ADT/SmallString.h" 19 #include "llvm/ADT/StringRef.h" 20 #include "llvm/ADT/Twine.h" 21 #include "llvm/BinaryFormat/ELF.h" 22 #include "llvm/Bitcode/BitcodeWriter.h" 23 #include "llvm/LTO/Config.h" 24 #include "llvm/LTO/LTO.h" 25 #include "llvm/Support/Caching.h" 26 #include "llvm/Support/CodeGen.h" 27 #include "llvm/Support/Error.h" 28 #include "llvm/Support/FileSystem.h" 29 #include "llvm/Support/MemoryBuffer.h" 30 #include <algorithm> 31 #include <cstddef> 32 #include <memory> 33 #include <string> 34 #include <system_error> 35 #include <vector> 36 37 using namespace llvm; 38 using namespace llvm::object; 39 using namespace llvm::ELF; 40 using namespace lld; 41 using namespace lld::elf; 42 43 // Creates an empty file to store a list of object files for final 44 // linking of distributed ThinLTO. 45 static std::unique_ptr<raw_fd_ostream> openFile(StringRef file) { 46 std::error_code ec; 47 auto ret = 48 std::make_unique<raw_fd_ostream>(file, ec, sys::fs::OpenFlags::OF_None); 49 if (ec) { 50 error("cannot open " + file + ": " + ec.message()); 51 return nullptr; 52 } 53 return ret; 54 } 55 56 // The merged bitcode after LTO is large. Try opening a file stream that 57 // supports reading, seeking and writing. Such a file allows BitcodeWriter to 58 // flush buffered data to reduce memory consumption. If this fails, open a file 59 // stream that supports only write. 60 static std::unique_ptr<raw_fd_ostream> openLTOOutputFile(StringRef file) { 61 std::error_code ec; 62 std::unique_ptr<raw_fd_ostream> fs = 63 std::make_unique<raw_fd_stream>(file, ec); 64 if (!ec) 65 return fs; 66 return openFile(file); 67 } 68 69 static std::string getThinLTOOutputFile(StringRef modulePath) { 70 return lto::getThinLTOOutputFile(modulePath, config->thinLTOPrefixReplaceOld, 71 config->thinLTOPrefixReplaceNew); 72 } 73 74 static lto::Config createConfig() { 75 lto::Config c; 76 77 // LLD supports the new relocations and address-significance tables. 78 c.Options = initTargetOptionsFromCodeGenFlags(); 79 c.Options.EmitAddrsig = true; 80 for (StringRef C : config->mllvmOpts) 81 c.MllvmArgs.emplace_back(C.str()); 82 83 // Always emit a section per function/datum with LTO. 84 c.Options.FunctionSections = true; 85 c.Options.DataSections = true; 86 87 // Check if basic block sections must be used. 88 // Allowed values for --lto-basic-block-sections are "all", "labels", 89 // "<file name specifying basic block ids>", or none. This is the equivalent 90 // of -fbasic-block-sections= flag in clang. 91 if (!config->ltoBasicBlockSections.empty()) { 92 if (config->ltoBasicBlockSections == "all") { 93 c.Options.BBSections = BasicBlockSection::All; 94 } else if (config->ltoBasicBlockSections == "labels") { 95 c.Options.BBSections = BasicBlockSection::Labels; 96 } else if (config->ltoBasicBlockSections == "none") { 97 c.Options.BBSections = BasicBlockSection::None; 98 } else { 99 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = 100 MemoryBuffer::getFile(config->ltoBasicBlockSections.str()); 101 if (!MBOrErr) { 102 error("cannot open " + config->ltoBasicBlockSections + ":" + 103 MBOrErr.getError().message()); 104 } else { 105 c.Options.BBSectionsFuncListBuf = std::move(*MBOrErr); 106 } 107 c.Options.BBSections = BasicBlockSection::List; 108 } 109 } 110 111 c.Options.UniqueBasicBlockSectionNames = 112 config->ltoUniqueBasicBlockSectionNames; 113 114 if (auto relocModel = getRelocModelFromCMModel()) 115 c.RelocModel = *relocModel; 116 else if (config->relocatable) 117 c.RelocModel = std::nullopt; 118 else if (config->isPic) 119 c.RelocModel = Reloc::PIC_; 120 else 121 c.RelocModel = Reloc::Static; 122 123 c.CodeModel = getCodeModelFromCMModel(); 124 c.DisableVerify = config->disableVerify; 125 c.DiagHandler = diagnosticHandler; 126 c.OptLevel = config->ltoo; 127 c.CPU = getCPUStr(); 128 c.MAttrs = getMAttrs(); 129 c.CGOptLevel = config->ltoCgo; 130 131 c.PTO.LoopVectorization = c.OptLevel > 1; 132 c.PTO.SLPVectorization = c.OptLevel > 1; 133 134 // Set up a custom pipeline if we've been asked to. 135 c.OptPipeline = std::string(config->ltoNewPmPasses); 136 c.AAPipeline = std::string(config->ltoAAPipeline); 137 138 // Set up optimization remarks if we've been asked to. 139 c.RemarksFilename = std::string(config->optRemarksFilename); 140 c.RemarksPasses = std::string(config->optRemarksPasses); 141 c.RemarksWithHotness = config->optRemarksWithHotness; 142 c.RemarksHotnessThreshold = config->optRemarksHotnessThreshold; 143 c.RemarksFormat = std::string(config->optRemarksFormat); 144 145 // Set up output file to emit statistics. 146 c.StatsFile = std::string(config->optStatsFilename); 147 148 c.SampleProfile = std::string(config->ltoSampleProfile); 149 for (StringRef pluginFn : config->passPlugins) 150 c.PassPlugins.push_back(std::string(pluginFn)); 151 c.DebugPassManager = config->ltoDebugPassManager; 152 c.DwoDir = std::string(config->dwoDir); 153 154 c.HasWholeProgramVisibility = config->ltoWholeProgramVisibility; 155 c.AlwaysEmitRegularLTOObj = !config->ltoObjPath.empty(); 156 157 for (const llvm::StringRef &name : config->thinLTOModulesToCompile) 158 c.ThinLTOModulesToCompile.emplace_back(name); 159 160 c.TimeTraceEnabled = config->timeTraceEnabled; 161 c.TimeTraceGranularity = config->timeTraceGranularity; 162 163 c.CSIRProfile = std::string(config->ltoCSProfileFile); 164 c.RunCSIRInstr = config->ltoCSProfileGenerate; 165 c.PGOWarnMismatch = config->ltoPGOWarnMismatch; 166 167 if (config->emitLLVM) { 168 c.PostInternalizeModuleHook = [](size_t task, const Module &m) { 169 if (std::unique_ptr<raw_fd_ostream> os = 170 openLTOOutputFile(config->outputFile)) 171 WriteBitcodeToFile(m, *os, false); 172 return false; 173 }; 174 } 175 176 if (config->ltoEmitAsm) { 177 c.CGFileType = CGFT_AssemblyFile; 178 c.Options.MCOptions.AsmVerbose = true; 179 } 180 181 if (!config->saveTempsArgs.empty()) 182 checkError(c.addSaveTemps(config->outputFile.str() + ".", 183 /*UseInputModulePath*/ true, 184 config->saveTempsArgs)); 185 return c; 186 } 187 188 BitcodeCompiler::BitcodeCompiler() { 189 // Initialize indexFile. 190 if (!config->thinLTOIndexOnlyArg.empty()) 191 indexFile = openFile(config->thinLTOIndexOnlyArg); 192 193 // Initialize ltoObj. 194 lto::ThinBackend backend; 195 auto onIndexWrite = [&](StringRef s) { thinIndices.erase(s); }; 196 if (config->thinLTOIndexOnly) { 197 backend = lto::createWriteIndexesThinBackend( 198 std::string(config->thinLTOPrefixReplaceOld), 199 std::string(config->thinLTOPrefixReplaceNew), 200 std::string(config->thinLTOPrefixReplaceNativeObject), 201 config->thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite); 202 } else { 203 backend = lto::createInProcessThinBackend( 204 llvm::heavyweight_hardware_concurrency(config->thinLTOJobs), 205 onIndexWrite, config->thinLTOEmitIndexFiles, 206 config->thinLTOEmitImportsFiles); 207 } 208 209 constexpr llvm::lto::LTO::LTOKind ltoModes[3] = 210 {llvm::lto::LTO::LTOKind::LTOK_UnifiedThin, 211 llvm::lto::LTO::LTOKind::LTOK_UnifiedRegular, 212 llvm::lto::LTO::LTOKind::LTOK_Default}; 213 ltoObj = std::make_unique<lto::LTO>( 214 createConfig(), backend, config->ltoPartitions, 215 ltoModes[config->ltoKind]); 216 217 // Initialize usedStartStop. 218 if (ctx.bitcodeFiles.empty()) 219 return; 220 for (Symbol *sym : symtab.getSymbols()) { 221 if (sym->isPlaceholder()) 222 continue; 223 StringRef s = sym->getName(); 224 for (StringRef prefix : {"__start_", "__stop_"}) 225 if (s.starts_with(prefix)) 226 usedStartStop.insert(s.substr(prefix.size())); 227 } 228 } 229 230 BitcodeCompiler::~BitcodeCompiler() = default; 231 232 void BitcodeCompiler::add(BitcodeFile &f) { 233 lto::InputFile &obj = *f.obj; 234 bool isExec = !config->shared && !config->relocatable; 235 236 if (config->thinLTOEmitIndexFiles) 237 thinIndices.insert(obj.getName()); 238 239 ArrayRef<Symbol *> syms = f.getSymbols(); 240 ArrayRef<lto::InputFile::Symbol> objSyms = obj.symbols(); 241 std::vector<lto::SymbolResolution> resols(syms.size()); 242 243 // Provide a resolution to the LTO API for each symbol. 244 for (size_t i = 0, e = syms.size(); i != e; ++i) { 245 Symbol *sym = syms[i]; 246 const lto::InputFile::Symbol &objSym = objSyms[i]; 247 lto::SymbolResolution &r = resols[i]; 248 249 // Ideally we shouldn't check for SF_Undefined but currently IRObjectFile 250 // reports two symbols for module ASM defined. Without this check, lld 251 // flags an undefined in IR with a definition in ASM as prevailing. 252 // Once IRObjectFile is fixed to report only one symbol this hack can 253 // be removed. 254 r.Prevailing = !objSym.isUndefined() && sym->file == &f; 255 256 // We ask LTO to preserve following global symbols: 257 // 1) All symbols when doing relocatable link, so that them can be used 258 // for doing final link. 259 // 2) Symbols that are used in regular objects. 260 // 3) C named sections if we have corresponding __start_/__stop_ symbol. 261 // 4) Symbols that are defined in bitcode files and used for dynamic 262 // linking. 263 // 5) Symbols that will be referenced after linker wrapping is performed. 264 r.VisibleToRegularObj = config->relocatable || sym->isUsedInRegularObj || 265 sym->referencedAfterWrap || 266 (r.Prevailing && sym->includeInDynsym()) || 267 usedStartStop.count(objSym.getSectionName()); 268 // Identify symbols exported dynamically, and that therefore could be 269 // referenced by a shared library not visible to the linker. 270 r.ExportDynamic = 271 sym->computeBinding() != STB_LOCAL && 272 (config->exportDynamic || sym->exportDynamic || sym->inDynamicList); 273 const auto *dr = dyn_cast<Defined>(sym); 274 r.FinalDefinitionInLinkageUnit = 275 (isExec || sym->visibility() != STV_DEFAULT) && dr && 276 // Skip absolute symbols from ELF objects, otherwise PC-rel relocations 277 // will be generated by for them, triggering linker errors. 278 // Symbol section is always null for bitcode symbols, hence the check 279 // for isElf(). Skip linker script defined symbols as well: they have 280 // no File defined. 281 !(dr->section == nullptr && (!sym->file || sym->file->isElf())); 282 283 if (r.Prevailing) 284 Undefined(nullptr, StringRef(), STB_GLOBAL, STV_DEFAULT, sym->type) 285 .overwrite(*sym); 286 287 // We tell LTO to not apply interprocedural optimization for wrapped 288 // (with --wrap) symbols because otherwise LTO would inline them while 289 // their values are still not final. 290 r.LinkerRedefined = sym->scriptDefined; 291 } 292 checkError(ltoObj->add(std::move(f.obj), resols)); 293 } 294 295 // If LazyObjFile has not been added to link, emit empty index files. 296 // This is needed because this is what GNU gold plugin does and we have a 297 // distributed build system that depends on that behavior. 298 static void thinLTOCreateEmptyIndexFiles() { 299 DenseSet<StringRef> linkedBitCodeFiles; 300 for (BitcodeFile *f : ctx.bitcodeFiles) 301 linkedBitCodeFiles.insert(f->getName()); 302 303 for (BitcodeFile *f : ctx.lazyBitcodeFiles) { 304 if (!f->lazy) 305 continue; 306 if (linkedBitCodeFiles.contains(f->getName())) 307 continue; 308 std::string path = 309 replaceThinLTOSuffix(getThinLTOOutputFile(f->obj->getName())); 310 std::unique_ptr<raw_fd_ostream> os = openFile(path + ".thinlto.bc"); 311 if (!os) 312 continue; 313 314 ModuleSummaryIndex m(/*HaveGVs*/ false); 315 m.setSkipModuleByDistributedBackend(); 316 writeIndexToFile(m, *os); 317 if (config->thinLTOEmitImportsFiles) 318 openFile(path + ".imports"); 319 } 320 } 321 322 // Merge all the bitcode files we have seen, codegen the result 323 // and return the resulting ObjectFile(s). 324 std::vector<InputFile *> BitcodeCompiler::compile() { 325 unsigned maxTasks = ltoObj->getMaxTasks(); 326 buf.resize(maxTasks); 327 files.resize(maxTasks); 328 329 // The --thinlto-cache-dir option specifies the path to a directory in which 330 // to cache native object files for ThinLTO incremental builds. If a path was 331 // specified, configure LTO to use it as the cache directory. 332 FileCache cache; 333 if (!config->thinLTOCacheDir.empty()) 334 cache = check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir, 335 [&](size_t task, const Twine &moduleName, 336 std::unique_ptr<MemoryBuffer> mb) { 337 files[task] = std::move(mb); 338 })); 339 340 if (!ctx.bitcodeFiles.empty()) 341 checkError(ltoObj->run( 342 [&](size_t task, const Twine &moduleName) { 343 return std::make_unique<CachedFileStream>( 344 std::make_unique<raw_svector_ostream>(buf[task])); 345 }, 346 cache)); 347 348 // Emit empty index files for non-indexed files but not in single-module mode. 349 if (config->thinLTOModulesToCompile.empty()) { 350 for (StringRef s : thinIndices) { 351 std::string path = getThinLTOOutputFile(s); 352 openFile(path + ".thinlto.bc"); 353 if (config->thinLTOEmitImportsFiles) 354 openFile(path + ".imports"); 355 } 356 } 357 358 if (config->thinLTOEmitIndexFiles) 359 thinLTOCreateEmptyIndexFiles(); 360 361 if (config->thinLTOIndexOnly) { 362 if (!config->ltoObjPath.empty()) 363 saveBuffer(buf[0], config->ltoObjPath); 364 365 // ThinLTO with index only option is required to generate only the index 366 // files. After that, we exit from linker and ThinLTO backend runs in a 367 // distributed environment. 368 if (indexFile) 369 indexFile->close(); 370 return {}; 371 } 372 373 if (!config->thinLTOCacheDir.empty()) 374 pruneCache(config->thinLTOCacheDir, config->thinLTOCachePolicy, files); 375 376 if (!config->ltoObjPath.empty()) { 377 saveBuffer(buf[0], config->ltoObjPath); 378 for (unsigned i = 1; i != maxTasks; ++i) 379 saveBuffer(buf[i], config->ltoObjPath + Twine(i)); 380 } 381 382 if (config->saveTempsArgs.contains("prelink")) { 383 if (!buf[0].empty()) 384 saveBuffer(buf[0], config->outputFile + ".lto.o"); 385 for (unsigned i = 1; i != maxTasks; ++i) 386 saveBuffer(buf[i], config->outputFile + Twine(i) + ".lto.o"); 387 } 388 389 if (config->ltoEmitAsm) { 390 saveBuffer(buf[0], config->outputFile); 391 for (unsigned i = 1; i != maxTasks; ++i) 392 saveBuffer(buf[i], config->outputFile + Twine(i)); 393 return {}; 394 } 395 396 std::vector<InputFile *> ret; 397 for (unsigned i = 0; i != maxTasks; ++i) 398 if (!buf[i].empty()) 399 ret.push_back(createObjFile(MemoryBufferRef(buf[i], "lto.tmp"))); 400 401 for (std::unique_ptr<MemoryBuffer> &file : files) 402 if (file) 403 ret.push_back(createObjFile(*file)); 404 return ret; 405 } 406