xref: /freebsd/contrib/llvm-project/llvm/tools/llvm-lto2/llvm-lto2.cpp (revision cfd6422a5217410fbd66f7a7a8a64d9d85e61229)
1 //===-- llvm-lto2: test harness for the resolution-based LTO interface ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This program takes in a list of bitcode files, links them and performs
10 // link-time optimization according to the provided symbol resolutions using the
11 // resolution-based LTO interface, and outputs one or more object files.
12 //
13 // This program is intended to eventually replace llvm-lto which uses the legacy
14 // LTO interface.
15 //
16 //===----------------------------------------------------------------------===//
17 
18 #include "llvm/Bitcode/BitcodeReader.h"
19 #include "llvm/CodeGen/CommandFlags.h"
20 #include "llvm/IR/DiagnosticPrinter.h"
21 #include "llvm/LTO/Caching.h"
22 #include "llvm/LTO/LTO.h"
23 #include "llvm/Passes/PassPlugin.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/FileSystem.h"
26 #include "llvm/Support/InitLLVM.h"
27 #include "llvm/Support/PluginLoader.h"
28 #include "llvm/Support/TargetSelect.h"
29 #include "llvm/Support/Threading.h"
30 
31 using namespace llvm;
32 using namespace lto;
33 
34 static codegen::RegisterCodeGenFlags CGF;
35 
36 static cl::opt<char>
37     OptLevel("O", cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
38                            "(default = '-O2')"),
39              cl::Prefix, cl::ZeroOrMore, cl::init('2'));
40 
41 static cl::opt<char> CGOptLevel(
42     "cg-opt-level",
43     cl::desc("Codegen optimization level (0, 1, 2 or 3, default = '2')"),
44     cl::init('2'));
45 
46 static cl::list<std::string> InputFilenames(cl::Positional, cl::OneOrMore,
47                                             cl::desc("<input bitcode files>"));
48 
49 static cl::opt<std::string> OutputFilename("o", cl::Required,
50                                            cl::desc("Output filename"),
51                                            cl::value_desc("filename"));
52 
53 static cl::opt<std::string> CacheDir("cache-dir", cl::desc("Cache Directory"),
54                                      cl::value_desc("directory"));
55 
56 static cl::opt<std::string> OptPipeline("opt-pipeline",
57                                         cl::desc("Optimizer Pipeline"),
58                                         cl::value_desc("pipeline"));
59 
60 static cl::opt<std::string> AAPipeline("aa-pipeline",
61                                        cl::desc("Alias Analysis Pipeline"),
62                                        cl::value_desc("aapipeline"));
63 
64 static cl::opt<bool> SaveTemps("save-temps", cl::desc("Save temporary files"));
65 
66 static cl::opt<bool>
67     ThinLTODistributedIndexes("thinlto-distributed-indexes", cl::init(false),
68                               cl::desc("Write out individual index and "
69                                        "import files for the "
70                                        "distributed backend case"));
71 
72 // Default to using all available threads in the system, but using only one
73 // thread per core (no SMT).
74 // Use -thinlto-threads=all to use hardware_concurrency() instead, which means
75 // to use all hardware threads or cores in the system.
76 static cl::opt<std::string> Threads("thinlto-threads");
77 
78 static cl::list<std::string> SymbolResolutions(
79     "r",
80     cl::desc("Specify a symbol resolution: filename,symbolname,resolution\n"
81              "where \"resolution\" is a sequence (which may be empty) of the\n"
82              "following characters:\n"
83              " p - prevailing: the linker has chosen this definition of the\n"
84              "     symbol\n"
85              " l - local: the definition of this symbol is unpreemptable at\n"
86              "     runtime and is known to be in this linkage unit\n"
87              " x - externally visible: the definition of this symbol is\n"
88              "     visible outside of the LTO unit\n"
89              "A resolution for each symbol must be specified."),
90     cl::ZeroOrMore);
91 
92 static cl::opt<std::string> OverrideTriple(
93     "override-triple",
94     cl::desc("Replace target triples in input files with this triple"));
95 
96 static cl::opt<std::string> DefaultTriple(
97     "default-triple",
98     cl::desc(
99         "Replace unspecified target triples in input files with this triple"));
100 
101 static cl::opt<bool> RemarksWithHotness(
102     "pass-remarks-with-hotness",
103     cl::desc("With PGO, include profile count in optimization remarks"),
104     cl::Hidden);
105 
106 static cl::opt<std::string>
107     RemarksFilename("pass-remarks-output",
108                     cl::desc("Output filename for pass remarks"),
109                     cl::value_desc("filename"));
110 
111 static cl::opt<std::string>
112     RemarksPasses("pass-remarks-filter",
113                   cl::desc("Only record optimization remarks from passes whose "
114                            "names match the given regular expression"),
115                   cl::value_desc("regex"));
116 
117 static cl::opt<std::string> RemarksFormat(
118     "pass-remarks-format",
119     cl::desc("The format used for serializing remarks (default: YAML)"),
120     cl::value_desc("format"), cl::init("yaml"));
121 
122 static cl::opt<std::string>
123     SamplePGOFile("lto-sample-profile-file",
124                   cl::desc("Specify a SamplePGO profile file"));
125 
126 static cl::opt<std::string>
127     CSPGOFile("lto-cspgo-profile-file",
128               cl::desc("Specify a context sensitive PGO profile file"));
129 
130 static cl::opt<bool>
131     RunCSIRInstr("lto-cspgo-gen",
132                  cl::desc("Run PGO context sensitive IR instrumentation"),
133                  cl::init(false), cl::Hidden);
134 
135 static cl::opt<bool>
136     UseNewPM("use-new-pm",
137              cl::desc("Run LTO passes using the new pass manager"),
138              cl::init(false), cl::Hidden);
139 
140 static cl::opt<bool>
141     DebugPassManager("debug-pass-manager", cl::init(false), cl::Hidden,
142                      cl::desc("Print pass management debugging information"));
143 
144 static cl::opt<std::string>
145     StatsFile("stats-file", cl::desc("Filename to write statistics to"));
146 
147 static cl::list<std::string>
148     PassPlugins("load-pass-plugin",
149                 cl::desc("Load passes from plugin library"));
150 
151 static void check(Error E, std::string Msg) {
152   if (!E)
153     return;
154   handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
155     errs() << "llvm-lto2: " << Msg << ": " << EIB.message().c_str() << '\n';
156   });
157   exit(1);
158 }
159 
160 template <typename T> static T check(Expected<T> E, std::string Msg) {
161   if (E)
162     return std::move(*E);
163   check(E.takeError(), Msg);
164   return T();
165 }
166 
167 static void check(std::error_code EC, std::string Msg) {
168   check(errorCodeToError(EC), Msg);
169 }
170 
171 template <typename T> static T check(ErrorOr<T> E, std::string Msg) {
172   if (E)
173     return std::move(*E);
174   check(E.getError(), Msg);
175   return T();
176 }
177 
178 static int usage() {
179   errs() << "Available subcommands: dump-symtab run\n";
180   return 1;
181 }
182 
183 static int run(int argc, char **argv) {
184   cl::ParseCommandLineOptions(argc, argv, "Resolution-based LTO test harness");
185 
186   // FIXME: Workaround PR30396 which means that a symbol can appear
187   // more than once if it is defined in module-level assembly and
188   // has a GV declaration. We allow (file, symbol) pairs to have multiple
189   // resolutions and apply them in the order observed.
190   std::map<std::pair<std::string, std::string>, std::list<SymbolResolution>>
191       CommandLineResolutions;
192   for (std::string R : SymbolResolutions) {
193     StringRef Rest = R;
194     StringRef FileName, SymbolName;
195     std::tie(FileName, Rest) = Rest.split(',');
196     if (Rest.empty()) {
197       llvm::errs() << "invalid resolution: " << R << '\n';
198       return 1;
199     }
200     std::tie(SymbolName, Rest) = Rest.split(',');
201     SymbolResolution Res;
202     for (char C : Rest) {
203       if (C == 'p')
204         Res.Prevailing = true;
205       else if (C == 'l')
206         Res.FinalDefinitionInLinkageUnit = true;
207       else if (C == 'x')
208         Res.VisibleToRegularObj = true;
209       else if (C == 'r')
210         Res.LinkerRedefined = true;
211       else {
212         llvm::errs() << "invalid character " << C << " in resolution: " << R
213                      << '\n';
214         return 1;
215       }
216     }
217     CommandLineResolutions[{std::string(FileName), std::string(SymbolName)}]
218         .push_back(Res);
219   }
220 
221   std::vector<std::unique_ptr<MemoryBuffer>> MBs;
222 
223   Config Conf;
224   Conf.DiagHandler = [](const DiagnosticInfo &DI) {
225     DiagnosticPrinterRawOStream DP(errs());
226     DI.print(DP);
227     errs() << '\n';
228     if (DI.getSeverity() == DS_Error)
229       exit(1);
230   };
231 
232   Conf.CPU = codegen::getMCPU();
233   Conf.Options = codegen::InitTargetOptionsFromCodeGenFlags();
234   Conf.MAttrs = codegen::getMAttrs();
235   if (auto RM = codegen::getExplicitRelocModel())
236     Conf.RelocModel = RM.getValue();
237   Conf.CodeModel = codegen::getExplicitCodeModel();
238 
239   Conf.DebugPassManager = DebugPassManager;
240 
241   if (SaveTemps)
242     check(Conf.addSaveTemps(OutputFilename + "."),
243           "Config::addSaveTemps failed");
244 
245   // Optimization remarks.
246   Conf.RemarksFilename = RemarksFilename;
247   Conf.RemarksPasses = RemarksPasses;
248   Conf.RemarksWithHotness = RemarksWithHotness;
249   Conf.RemarksFormat = RemarksFormat;
250 
251   Conf.SampleProfile = SamplePGOFile;
252   Conf.CSIRProfile = CSPGOFile;
253   Conf.RunCSIRInstr = RunCSIRInstr;
254 
255   // Run a custom pipeline, if asked for.
256   Conf.OptPipeline = OptPipeline;
257   Conf.AAPipeline = AAPipeline;
258 
259   Conf.OptLevel = OptLevel - '0';
260   Conf.UseNewPM = UseNewPM;
261   for (auto &PluginFN : PassPlugins)
262     Conf.PassPlugins.push_back(PluginFN);
263   switch (CGOptLevel) {
264   case '0':
265     Conf.CGOptLevel = CodeGenOpt::None;
266     break;
267   case '1':
268     Conf.CGOptLevel = CodeGenOpt::Less;
269     break;
270   case '2':
271     Conf.CGOptLevel = CodeGenOpt::Default;
272     break;
273   case '3':
274     Conf.CGOptLevel = CodeGenOpt::Aggressive;
275     break;
276   default:
277     llvm::errs() << "invalid cg optimization level: " << CGOptLevel << '\n';
278     return 1;
279   }
280 
281   if (auto FT = codegen::getExplicitFileType())
282     Conf.CGFileType = FT.getValue();
283 
284   Conf.OverrideTriple = OverrideTriple;
285   Conf.DefaultTriple = DefaultTriple;
286   Conf.StatsFile = StatsFile;
287   Conf.PTO.LoopVectorization = Conf.OptLevel > 1;
288   Conf.PTO.SLPVectorization = Conf.OptLevel > 1;
289 
290   ThinBackend Backend;
291   if (ThinLTODistributedIndexes)
292     Backend = createWriteIndexesThinBackend(/* OldPrefix */ "",
293                                             /* NewPrefix */ "",
294                                             /* ShouldEmitImportsFiles */ true,
295                                             /* LinkedObjectsFile */ nullptr,
296                                             /* OnWrite */ {});
297   else
298     Backend = createInProcessThinBackend(
299         llvm::heavyweight_hardware_concurrency(Threads));
300   LTO Lto(std::move(Conf), std::move(Backend));
301 
302   bool HasErrors = false;
303   for (std::string F : InputFilenames) {
304     std::unique_ptr<MemoryBuffer> MB = check(MemoryBuffer::getFile(F), F);
305     std::unique_ptr<InputFile> Input =
306         check(InputFile::create(MB->getMemBufferRef()), F);
307 
308     std::vector<SymbolResolution> Res;
309     for (const InputFile::Symbol &Sym : Input->symbols()) {
310       auto I = CommandLineResolutions.find({F, std::string(Sym.getName())});
311       // If it isn't found, look for "$", which would have been added
312       // (followed by a hash) when the symbol was promoted during module
313       // splitting if it was defined in one part and used in the other.
314       // Try looking up the symbol name before the "$".
315       if (I == CommandLineResolutions.end()) {
316         auto SplitName = Sym.getName().rsplit("$");
317         I = CommandLineResolutions.find({F, std::string(SplitName.first)});
318       }
319       if (I == CommandLineResolutions.end()) {
320         llvm::errs() << argv[0] << ": missing symbol resolution for " << F
321                      << ',' << Sym.getName() << '\n';
322         HasErrors = true;
323       } else {
324         Res.push_back(I->second.front());
325         I->second.pop_front();
326         if (I->second.empty())
327           CommandLineResolutions.erase(I);
328       }
329     }
330 
331     if (HasErrors)
332       continue;
333 
334     MBs.push_back(std::move(MB));
335     check(Lto.add(std::move(Input), Res), F);
336   }
337 
338   if (!CommandLineResolutions.empty()) {
339     HasErrors = true;
340     for (auto UnusedRes : CommandLineResolutions)
341       llvm::errs() << argv[0] << ": unused symbol resolution for "
342                    << UnusedRes.first.first << ',' << UnusedRes.first.second
343                    << '\n';
344   }
345   if (HasErrors)
346     return 1;
347 
348   auto AddStream =
349       [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
350     std::string Path = OutputFilename + "." + utostr(Task);
351 
352     std::error_code EC;
353     auto S = std::make_unique<raw_fd_ostream>(Path, EC, sys::fs::OF_None);
354     check(EC, Path);
355     return std::make_unique<lto::NativeObjectStream>(std::move(S));
356   };
357 
358   auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
359     *AddStream(Task)->OS << MB->getBuffer();
360   };
361 
362   NativeObjectCache Cache;
363   if (!CacheDir.empty())
364     Cache = check(localCache(CacheDir, AddBuffer), "failed to create cache");
365 
366   check(Lto.run(AddStream, Cache), "LTO::run failed");
367   return 0;
368 }
369 
370 static int dumpSymtab(int argc, char **argv) {
371   for (StringRef F : make_range(argv + 1, argv + argc)) {
372     std::unique_ptr<MemoryBuffer> MB =
373         check(MemoryBuffer::getFile(F), std::string(F));
374     BitcodeFileContents BFC =
375         check(getBitcodeFileContents(*MB), std::string(F));
376 
377     if (BFC.Symtab.size() >= sizeof(irsymtab::storage::Header)) {
378       auto *Hdr = reinterpret_cast<const irsymtab::storage::Header *>(
379           BFC.Symtab.data());
380       outs() << "version: " << Hdr->Version << '\n';
381       if (Hdr->Version == irsymtab::storage::Header::kCurrentVersion)
382         outs() << "producer: " << Hdr->Producer.get(BFC.StrtabForSymtab)
383                << '\n';
384     }
385 
386     std::unique_ptr<InputFile> Input =
387         check(InputFile::create(MB->getMemBufferRef()), std::string(F));
388 
389     outs() << "target triple: " << Input->getTargetTriple() << '\n';
390     Triple TT(Input->getTargetTriple());
391 
392     outs() << "source filename: " << Input->getSourceFileName() << '\n';
393 
394     if (TT.isOSBinFormatCOFF())
395       outs() << "linker opts: " << Input->getCOFFLinkerOpts() << '\n';
396 
397     if (TT.isOSBinFormatELF()) {
398       outs() << "dependent libraries:";
399       for (auto L : Input->getDependentLibraries())
400         outs() << " \"" << L << "\"";
401       outs() << '\n';
402     }
403 
404     std::vector<StringRef> ComdatTable = Input->getComdatTable();
405     for (const InputFile::Symbol &Sym : Input->symbols()) {
406       switch (Sym.getVisibility()) {
407       case GlobalValue::HiddenVisibility:
408         outs() << 'H';
409         break;
410       case GlobalValue::ProtectedVisibility:
411         outs() << 'P';
412         break;
413       case GlobalValue::DefaultVisibility:
414         outs() << 'D';
415         break;
416       }
417 
418       auto PrintBool = [&](char C, bool B) { outs() << (B ? C : '-'); };
419       PrintBool('U', Sym.isUndefined());
420       PrintBool('C', Sym.isCommon());
421       PrintBool('W', Sym.isWeak());
422       PrintBool('I', Sym.isIndirect());
423       PrintBool('O', Sym.canBeOmittedFromSymbolTable());
424       PrintBool('T', Sym.isTLS());
425       PrintBool('X', Sym.isExecutable());
426       outs() << ' ' << Sym.getName() << '\n';
427 
428       if (Sym.isCommon())
429         outs() << "         size " << Sym.getCommonSize() << " align "
430                << Sym.getCommonAlignment() << '\n';
431 
432       int Comdat = Sym.getComdatIndex();
433       if (Comdat != -1)
434         outs() << "         comdat " << ComdatTable[Comdat] << '\n';
435 
436       if (TT.isOSBinFormatCOFF() && Sym.isWeak() && Sym.isIndirect())
437         outs() << "         fallback " << Sym.getCOFFWeakExternalFallback() << '\n';
438 
439       if (!Sym.getSectionName().empty())
440         outs() << "         section " << Sym.getSectionName() << "\n";
441     }
442 
443     outs() << '\n';
444   }
445 
446   return 0;
447 }
448 
449 int main(int argc, char **argv) {
450   InitLLVM X(argc, argv);
451   InitializeAllTargets();
452   InitializeAllTargetMCs();
453   InitializeAllAsmPrinters();
454   InitializeAllAsmParsers();
455 
456   // FIXME: This should use llvm::cl subcommands, but it isn't currently
457   // possible to pass an argument not associated with a subcommand to a
458   // subcommand (e.g. -use-new-pm).
459   if (argc < 2)
460     return usage();
461 
462   StringRef Subcommand = argv[1];
463   // Ensure that argv[0] is correct after adjusting argv/argc.
464   argv[1] = argv[0];
465   if (Subcommand == "dump-symtab")
466     return dumpSymtab(argc - 1, argv + 1);
467   if (Subcommand == "run")
468     return run(argc - 1, argv + 1);
469   return usage();
470 }
471