1 //===-- llvm-lto2: test harness for the resolution-based LTO interface ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This program takes in a list of bitcode files, links them and performs 10 // link-time optimization according to the provided symbol resolutions using the 11 // resolution-based LTO interface, and outputs one or more object files. 12 // 13 // This program is intended to eventually replace llvm-lto which uses the legacy 14 // LTO interface. 15 // 16 //===----------------------------------------------------------------------===// 17 18 #include "llvm/Bitcode/BitcodeReader.h" 19 #include "llvm/CodeGen/CommandFlags.inc" 20 #include "llvm/IR/DiagnosticPrinter.h" 21 #include "llvm/LTO/Caching.h" 22 #include "llvm/LTO/LTO.h" 23 #include "llvm/Support/CommandLine.h" 24 #include "llvm/Support/FileSystem.h" 25 #include "llvm/Support/InitLLVM.h" 26 #include "llvm/Support/TargetSelect.h" 27 #include "llvm/Support/Threading.h" 28 29 using namespace llvm; 30 using namespace lto; 31 32 static cl::opt<char> 33 OptLevel("O", cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] " 34 "(default = '-O2')"), 35 cl::Prefix, cl::ZeroOrMore, cl::init('2')); 36 37 static cl::opt<char> CGOptLevel( 38 "cg-opt-level", 39 cl::desc("Codegen optimization level (0, 1, 2 or 3, default = '2')"), 40 cl::init('2')); 41 42 static cl::list<std::string> InputFilenames(cl::Positional, cl::OneOrMore, 43 cl::desc("<input bitcode files>")); 44 45 static cl::opt<std::string> OutputFilename("o", cl::Required, 46 cl::desc("Output filename"), 47 cl::value_desc("filename")); 48 49 static cl::opt<std::string> CacheDir("cache-dir", cl::desc("Cache Directory"), 50 cl::value_desc("directory")); 51 52 static cl::opt<std::string> OptPipeline("opt-pipeline", 53 cl::desc("Optimizer Pipeline"), 54 cl::value_desc("pipeline")); 55 56 static cl::opt<std::string> AAPipeline("aa-pipeline", 57 cl::desc("Alias Analysis Pipeline"), 58 cl::value_desc("aapipeline")); 59 60 static cl::opt<bool> SaveTemps("save-temps", cl::desc("Save temporary files")); 61 62 static cl::opt<bool> 63 ThinLTODistributedIndexes("thinlto-distributed-indexes", cl::init(false), 64 cl::desc("Write out individual index and " 65 "import files for the " 66 "distributed backend case")); 67 68 static cl::opt<int> Threads("thinlto-threads", 69 cl::init(llvm::heavyweight_hardware_concurrency())); 70 71 static cl::list<std::string> SymbolResolutions( 72 "r", 73 cl::desc("Specify a symbol resolution: filename,symbolname,resolution\n" 74 "where \"resolution\" is a sequence (which may be empty) of the\n" 75 "following characters:\n" 76 " p - prevailing: the linker has chosen this definition of the\n" 77 " symbol\n" 78 " l - local: the definition of this symbol is unpreemptable at\n" 79 " runtime and is known to be in this linkage unit\n" 80 " x - externally visible: the definition of this symbol is\n" 81 " visible outside of the LTO unit\n" 82 "A resolution for each symbol must be specified."), 83 cl::ZeroOrMore); 84 85 static cl::opt<std::string> OverrideTriple( 86 "override-triple", 87 cl::desc("Replace target triples in input files with this triple")); 88 89 static cl::opt<std::string> DefaultTriple( 90 "default-triple", 91 cl::desc( 92 "Replace unspecified target triples in input files with this triple")); 93 94 static cl::opt<bool> RemarksWithHotness( 95 "pass-remarks-with-hotness", 96 cl::desc("With PGO, include profile count in optimization remarks"), 97 cl::Hidden); 98 99 static cl::opt<std::string> 100 RemarksFilename("pass-remarks-output", 101 cl::desc("Output filename for pass remarks"), 102 cl::value_desc("filename")); 103 104 static cl::opt<std::string> 105 RemarksPasses("pass-remarks-filter", 106 cl::desc("Only record optimization remarks from passes whose " 107 "names match the given regular expression"), 108 cl::value_desc("regex")); 109 110 static cl::opt<std::string> RemarksFormat( 111 "pass-remarks-format", 112 cl::desc("The format used for serializing remarks (default: YAML)"), 113 cl::value_desc("format"), cl::init("yaml")); 114 115 static cl::opt<std::string> 116 SamplePGOFile("lto-sample-profile-file", 117 cl::desc("Specify a SamplePGO profile file")); 118 119 static cl::opt<std::string> 120 CSPGOFile("lto-cspgo-profile-file", 121 cl::desc("Specify a context sensitive PGO profile file")); 122 123 static cl::opt<bool> 124 RunCSIRInstr("lto-cspgo-gen", 125 cl::desc("Run PGO context sensitive IR instrumentation"), 126 cl::init(false), cl::Hidden); 127 128 static cl::opt<bool> 129 UseNewPM("use-new-pm", 130 cl::desc("Run LTO passes using the new pass manager"), 131 cl::init(false), cl::Hidden); 132 133 static cl::opt<bool> 134 DebugPassManager("debug-pass-manager", cl::init(false), cl::Hidden, 135 cl::desc("Print pass management debugging information")); 136 137 static cl::opt<std::string> 138 StatsFile("stats-file", cl::desc("Filename to write statistics to")); 139 140 static void check(Error E, std::string Msg) { 141 if (!E) 142 return; 143 handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) { 144 errs() << "llvm-lto2: " << Msg << ": " << EIB.message().c_str() << '\n'; 145 }); 146 exit(1); 147 } 148 149 template <typename T> static T check(Expected<T> E, std::string Msg) { 150 if (E) 151 return std::move(*E); 152 check(E.takeError(), Msg); 153 return T(); 154 } 155 156 static void check(std::error_code EC, std::string Msg) { 157 check(errorCodeToError(EC), Msg); 158 } 159 160 template <typename T> static T check(ErrorOr<T> E, std::string Msg) { 161 if (E) 162 return std::move(*E); 163 check(E.getError(), Msg); 164 return T(); 165 } 166 167 static int usage() { 168 errs() << "Available subcommands: dump-symtab run\n"; 169 return 1; 170 } 171 172 static int run(int argc, char **argv) { 173 cl::ParseCommandLineOptions(argc, argv, "Resolution-based LTO test harness"); 174 175 // FIXME: Workaround PR30396 which means that a symbol can appear 176 // more than once if it is defined in module-level assembly and 177 // has a GV declaration. We allow (file, symbol) pairs to have multiple 178 // resolutions and apply them in the order observed. 179 std::map<std::pair<std::string, std::string>, std::list<SymbolResolution>> 180 CommandLineResolutions; 181 for (std::string R : SymbolResolutions) { 182 StringRef Rest = R; 183 StringRef FileName, SymbolName; 184 std::tie(FileName, Rest) = Rest.split(','); 185 if (Rest.empty()) { 186 llvm::errs() << "invalid resolution: " << R << '\n'; 187 return 1; 188 } 189 std::tie(SymbolName, Rest) = Rest.split(','); 190 SymbolResolution Res; 191 for (char C : Rest) { 192 if (C == 'p') 193 Res.Prevailing = true; 194 else if (C == 'l') 195 Res.FinalDefinitionInLinkageUnit = true; 196 else if (C == 'x') 197 Res.VisibleToRegularObj = true; 198 else if (C == 'r') 199 Res.LinkerRedefined = true; 200 else { 201 llvm::errs() << "invalid character " << C << " in resolution: " << R 202 << '\n'; 203 return 1; 204 } 205 } 206 CommandLineResolutions[{FileName, SymbolName}].push_back(Res); 207 } 208 209 std::vector<std::unique_ptr<MemoryBuffer>> MBs; 210 211 Config Conf; 212 Conf.DiagHandler = [](const DiagnosticInfo &DI) { 213 DiagnosticPrinterRawOStream DP(errs()); 214 DI.print(DP); 215 errs() << '\n'; 216 if (DI.getSeverity() == DS_Error) 217 exit(1); 218 }; 219 220 Conf.CPU = MCPU; 221 Conf.Options = InitTargetOptionsFromCodeGenFlags(); 222 Conf.MAttrs = MAttrs; 223 if (auto RM = getRelocModel()) 224 Conf.RelocModel = *RM; 225 Conf.CodeModel = getCodeModel(); 226 227 Conf.DebugPassManager = DebugPassManager; 228 229 if (SaveTemps) 230 check(Conf.addSaveTemps(OutputFilename + "."), 231 "Config::addSaveTemps failed"); 232 233 // Optimization remarks. 234 Conf.RemarksFilename = RemarksFilename; 235 Conf.RemarksPasses = RemarksPasses; 236 Conf.RemarksWithHotness = RemarksWithHotness; 237 Conf.RemarksFormat = RemarksFormat; 238 239 Conf.SampleProfile = SamplePGOFile; 240 Conf.CSIRProfile = CSPGOFile; 241 Conf.RunCSIRInstr = RunCSIRInstr; 242 243 // Run a custom pipeline, if asked for. 244 Conf.OptPipeline = OptPipeline; 245 Conf.AAPipeline = AAPipeline; 246 247 Conf.OptLevel = OptLevel - '0'; 248 Conf.UseNewPM = UseNewPM; 249 switch (CGOptLevel) { 250 case '0': 251 Conf.CGOptLevel = CodeGenOpt::None; 252 break; 253 case '1': 254 Conf.CGOptLevel = CodeGenOpt::Less; 255 break; 256 case '2': 257 Conf.CGOptLevel = CodeGenOpt::Default; 258 break; 259 case '3': 260 Conf.CGOptLevel = CodeGenOpt::Aggressive; 261 break; 262 default: 263 llvm::errs() << "invalid cg optimization level: " << CGOptLevel << '\n'; 264 return 1; 265 } 266 267 if (FileType.getNumOccurrences()) 268 Conf.CGFileType = FileType; 269 270 Conf.OverrideTriple = OverrideTriple; 271 Conf.DefaultTriple = DefaultTriple; 272 Conf.StatsFile = StatsFile; 273 Conf.PTO.LoopVectorization = Conf.OptLevel > 1; 274 Conf.PTO.SLPVectorization = Conf.OptLevel > 1; 275 276 ThinBackend Backend; 277 if (ThinLTODistributedIndexes) 278 Backend = createWriteIndexesThinBackend(/* OldPrefix */ "", 279 /* NewPrefix */ "", 280 /* ShouldEmitImportsFiles */ true, 281 /* LinkedObjectsFile */ nullptr, 282 /* OnWrite */ {}); 283 else 284 Backend = createInProcessThinBackend(Threads); 285 LTO Lto(std::move(Conf), std::move(Backend)); 286 287 bool HasErrors = false; 288 for (std::string F : InputFilenames) { 289 std::unique_ptr<MemoryBuffer> MB = check(MemoryBuffer::getFile(F), F); 290 std::unique_ptr<InputFile> Input = 291 check(InputFile::create(MB->getMemBufferRef()), F); 292 293 std::vector<SymbolResolution> Res; 294 for (const InputFile::Symbol &Sym : Input->symbols()) { 295 auto I = CommandLineResolutions.find({F, Sym.getName()}); 296 // If it isn't found, look for "$", which would have been added 297 // (followed by a hash) when the symbol was promoted during module 298 // splitting if it was defined in one part and used in the other. 299 // Try looking up the symbol name before the "$". 300 if (I == CommandLineResolutions.end()) { 301 auto SplitName = Sym.getName().rsplit("$"); 302 I = CommandLineResolutions.find({F, SplitName.first}); 303 } 304 if (I == CommandLineResolutions.end()) { 305 llvm::errs() << argv[0] << ": missing symbol resolution for " << F 306 << ',' << Sym.getName() << '\n'; 307 HasErrors = true; 308 } else { 309 Res.push_back(I->second.front()); 310 I->second.pop_front(); 311 if (I->second.empty()) 312 CommandLineResolutions.erase(I); 313 } 314 } 315 316 if (HasErrors) 317 continue; 318 319 MBs.push_back(std::move(MB)); 320 check(Lto.add(std::move(Input), Res), F); 321 } 322 323 if (!CommandLineResolutions.empty()) { 324 HasErrors = true; 325 for (auto UnusedRes : CommandLineResolutions) 326 llvm::errs() << argv[0] << ": unused symbol resolution for " 327 << UnusedRes.first.first << ',' << UnusedRes.first.second 328 << '\n'; 329 } 330 if (HasErrors) 331 return 1; 332 333 auto AddStream = 334 [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> { 335 std::string Path = OutputFilename + "." + utostr(Task); 336 337 std::error_code EC; 338 auto S = std::make_unique<raw_fd_ostream>(Path, EC, sys::fs::OF_None); 339 check(EC, Path); 340 return std::make_unique<lto::NativeObjectStream>(std::move(S)); 341 }; 342 343 auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) { 344 *AddStream(Task)->OS << MB->getBuffer(); 345 }; 346 347 NativeObjectCache Cache; 348 if (!CacheDir.empty()) 349 Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache"); 350 351 check(Lto.run(AddStream, Cache), "LTO::run failed"); 352 return 0; 353 } 354 355 static int dumpSymtab(int argc, char **argv) { 356 for (StringRef F : make_range(argv + 1, argv + argc)) { 357 std::unique_ptr<MemoryBuffer> MB = check(MemoryBuffer::getFile(F), F); 358 BitcodeFileContents BFC = check(getBitcodeFileContents(*MB), F); 359 360 if (BFC.Symtab.size() >= sizeof(irsymtab::storage::Header)) { 361 auto *Hdr = reinterpret_cast<const irsymtab::storage::Header *>( 362 BFC.Symtab.data()); 363 outs() << "version: " << Hdr->Version << '\n'; 364 if (Hdr->Version == irsymtab::storage::Header::kCurrentVersion) 365 outs() << "producer: " << Hdr->Producer.get(BFC.StrtabForSymtab) 366 << '\n'; 367 } 368 369 std::unique_ptr<InputFile> Input = 370 check(InputFile::create(MB->getMemBufferRef()), F); 371 372 outs() << "target triple: " << Input->getTargetTriple() << '\n'; 373 Triple TT(Input->getTargetTriple()); 374 375 outs() << "source filename: " << Input->getSourceFileName() << '\n'; 376 377 if (TT.isOSBinFormatCOFF()) 378 outs() << "linker opts: " << Input->getCOFFLinkerOpts() << '\n'; 379 380 if (TT.isOSBinFormatELF()) { 381 outs() << "dependent libraries:"; 382 for (auto L : Input->getDependentLibraries()) 383 outs() << " \"" << L << "\""; 384 outs() << '\n'; 385 } 386 387 std::vector<StringRef> ComdatTable = Input->getComdatTable(); 388 for (const InputFile::Symbol &Sym : Input->symbols()) { 389 switch (Sym.getVisibility()) { 390 case GlobalValue::HiddenVisibility: 391 outs() << 'H'; 392 break; 393 case GlobalValue::ProtectedVisibility: 394 outs() << 'P'; 395 break; 396 case GlobalValue::DefaultVisibility: 397 outs() << 'D'; 398 break; 399 } 400 401 auto PrintBool = [&](char C, bool B) { outs() << (B ? C : '-'); }; 402 PrintBool('U', Sym.isUndefined()); 403 PrintBool('C', Sym.isCommon()); 404 PrintBool('W', Sym.isWeak()); 405 PrintBool('I', Sym.isIndirect()); 406 PrintBool('O', Sym.canBeOmittedFromSymbolTable()); 407 PrintBool('T', Sym.isTLS()); 408 PrintBool('X', Sym.isExecutable()); 409 outs() << ' ' << Sym.getName() << '\n'; 410 411 if (Sym.isCommon()) 412 outs() << " size " << Sym.getCommonSize() << " align " 413 << Sym.getCommonAlignment() << '\n'; 414 415 int Comdat = Sym.getComdatIndex(); 416 if (Comdat != -1) 417 outs() << " comdat " << ComdatTable[Comdat] << '\n'; 418 419 if (TT.isOSBinFormatCOFF() && Sym.isWeak() && Sym.isIndirect()) 420 outs() << " fallback " << Sym.getCOFFWeakExternalFallback() << '\n'; 421 422 if (!Sym.getSectionName().empty()) 423 outs() << " section " << Sym.getSectionName() << "\n"; 424 } 425 426 outs() << '\n'; 427 } 428 429 return 0; 430 } 431 432 int main(int argc, char **argv) { 433 InitLLVM X(argc, argv); 434 InitializeAllTargets(); 435 InitializeAllTargetMCs(); 436 InitializeAllAsmPrinters(); 437 InitializeAllAsmParsers(); 438 439 // FIXME: This should use llvm::cl subcommands, but it isn't currently 440 // possible to pass an argument not associated with a subcommand to a 441 // subcommand (e.g. -use-new-pm). 442 if (argc < 2) 443 return usage(); 444 445 StringRef Subcommand = argv[1]; 446 // Ensure that argv[0] is correct after adjusting argv/argc. 447 argv[1] = argv[0]; 448 if (Subcommand == "dump-symtab") 449 return dumpSymtab(argc - 1, argv + 1); 450 if (Subcommand == "run") 451 return run(argc - 1, argv + 1); 452 return usage(); 453 } 454