xref: /freebsd/contrib/llvm-project/llvm/tools/llvm-lto/llvm-lto.cpp (revision 5fb307d29b364982acbde82cbf77db3cae486f8c)
1 //===- llvm-lto: a simple command-line program to link modules with LTO ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This program takes in a list of bitcode files, links them, performs link-time
10 // optimization, and outputs an object file.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm-c/lto.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/SmallString.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/ADT/StringSet.h"
21 #include "llvm/ADT/Twine.h"
22 #include "llvm/Bitcode/BitcodeReader.h"
23 #include "llvm/Bitcode/BitcodeWriter.h"
24 #include "llvm/CodeGen/CommandFlags.h"
25 #include "llvm/IR/DiagnosticInfo.h"
26 #include "llvm/IR/DiagnosticPrinter.h"
27 #include "llvm/IR/LLVMContext.h"
28 #include "llvm/IR/Module.h"
29 #include "llvm/IR/ModuleSummaryIndex.h"
30 #include "llvm/IR/Verifier.h"
31 #include "llvm/IRReader/IRReader.h"
32 #include "llvm/LTO/legacy/LTOCodeGenerator.h"
33 #include "llvm/LTO/legacy/LTOModule.h"
34 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
35 #include "llvm/Support/Allocator.h"
36 #include "llvm/Support/Casting.h"
37 #include "llvm/Support/CommandLine.h"
38 #include "llvm/Support/Error.h"
39 #include "llvm/Support/ErrorHandling.h"
40 #include "llvm/Support/ErrorOr.h"
41 #include "llvm/Support/FileSystem.h"
42 #include "llvm/Support/InitLLVM.h"
43 #include "llvm/Support/MemoryBuffer.h"
44 #include "llvm/Support/Path.h"
45 #include "llvm/Support/SourceMgr.h"
46 #include "llvm/Support/TargetSelect.h"
47 #include "llvm/Support/ToolOutputFile.h"
48 #include "llvm/Support/raw_ostream.h"
49 #include "llvm/Support/WithColor.h"
50 #include "llvm/Target/TargetOptions.h"
51 #include <algorithm>
52 #include <cassert>
53 #include <cstdint>
54 #include <cstdlib>
55 #include <list>
56 #include <map>
57 #include <memory>
58 #include <string>
59 #include <system_error>
60 #include <tuple>
61 #include <utility>
62 #include <vector>
63 
64 using namespace llvm;
65 
66 static codegen::RegisterCodeGenFlags CGF;
67 
68 static cl::OptionCategory LTOCategory("LTO Options");
69 
70 static cl::opt<char>
71     OptLevel("O",
72              cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
73                       "(default = '-O2')"),
74              cl::Prefix, cl::init('2'), cl::cat(LTOCategory));
75 
76 static cl::opt<bool>
77     IndexStats("thinlto-index-stats",
78                cl::desc("Print statistic for the index in every input files"),
79                cl::init(false), cl::cat(LTOCategory));
80 
81 static cl::opt<bool> DisableVerify(
82     "disable-verify", cl::init(false),
83     cl::desc("Do not run the verifier during the optimization pipeline"),
84     cl::cat(LTOCategory));
85 
86 static cl::opt<bool> EnableFreestanding(
87     "lto-freestanding", cl::init(false),
88     cl::desc("Enable Freestanding (disable builtins / TLI) during LTO"),
89     cl::cat(LTOCategory));
90 
91 static cl::opt<bool> UseDiagnosticHandler(
92     "use-diagnostic-handler", cl::init(false),
93     cl::desc("Use a diagnostic handler to test the handler interface"),
94     cl::cat(LTOCategory));
95 
96 static cl::opt<bool>
97     ThinLTO("thinlto", cl::init(false),
98             cl::desc("Only write combined global index for ThinLTO backends"),
99             cl::cat(LTOCategory));
100 
101 enum ThinLTOModes {
102   THINLINK,
103   THINDISTRIBUTE,
104   THINEMITIMPORTS,
105   THINPROMOTE,
106   THINIMPORT,
107   THININTERNALIZE,
108   THINOPT,
109   THINCODEGEN,
110   THINALL
111 };
112 
113 cl::opt<ThinLTOModes> ThinLTOMode(
114     "thinlto-action", cl::desc("Perform a single ThinLTO stage:"),
115     cl::values(
116         clEnumValN(
117             THINLINK, "thinlink",
118             "ThinLink: produces the index by linking only the summaries."),
119         clEnumValN(THINDISTRIBUTE, "distributedindexes",
120                    "Produces individual indexes for distributed backends."),
121         clEnumValN(THINEMITIMPORTS, "emitimports",
122                    "Emit imports files for distributed backends."),
123         clEnumValN(THINPROMOTE, "promote",
124                    "Perform pre-import promotion (requires -thinlto-index)."),
125         clEnumValN(THINIMPORT, "import",
126                    "Perform both promotion and "
127                    "cross-module importing (requires "
128                    "-thinlto-index)."),
129         clEnumValN(THININTERNALIZE, "internalize",
130                    "Perform internalization driven by -exported-symbol "
131                    "(requires -thinlto-index)."),
132         clEnumValN(THINOPT, "optimize", "Perform ThinLTO optimizations."),
133         clEnumValN(THINCODEGEN, "codegen", "CodeGen (expected to match llc)"),
134         clEnumValN(THINALL, "run", "Perform ThinLTO end-to-end")),
135     cl::cat(LTOCategory));
136 
137 static cl::opt<std::string>
138     ThinLTOIndex("thinlto-index",
139                  cl::desc("Provide the index produced by a ThinLink, required "
140                           "to perform the promotion and/or importing."),
141                  cl::cat(LTOCategory));
142 
143 static cl::opt<std::string> ThinLTOPrefixReplace(
144     "thinlto-prefix-replace",
145     cl::desc("Control where files for distributed backends are "
146              "created. Expects 'oldprefix;newprefix' and if path "
147              "prefix of output file is oldprefix it will be "
148              "replaced with newprefix."),
149     cl::cat(LTOCategory));
150 
151 static cl::opt<std::string> ThinLTOModuleId(
152     "thinlto-module-id",
153     cl::desc("For the module ID for the file to process, useful to "
154              "match what is in the index."),
155     cl::cat(LTOCategory));
156 
157 static cl::opt<std::string> ThinLTOCacheDir("thinlto-cache-dir",
158                                             cl::desc("Enable ThinLTO caching."),
159                                             cl::cat(LTOCategory));
160 
161 static cl::opt<int> ThinLTOCachePruningInterval(
162     "thinlto-cache-pruning-interval", cl::init(1200),
163     cl::desc("Set ThinLTO cache pruning interval."), cl::cat(LTOCategory));
164 
165 static cl::opt<uint64_t> ThinLTOCacheMaxSizeBytes(
166     "thinlto-cache-max-size-bytes",
167     cl::desc("Set ThinLTO cache pruning directory maximum size in bytes."),
168     cl::cat(LTOCategory));
169 
170 static cl::opt<int> ThinLTOCacheMaxSizeFiles(
171     "thinlto-cache-max-size-files", cl::init(1000000),
172     cl::desc("Set ThinLTO cache pruning directory maximum number of files."),
173     cl::cat(LTOCategory));
174 
175 static cl::opt<unsigned> ThinLTOCacheEntryExpiration(
176     "thinlto-cache-entry-expiration", cl::init(604800) /* 1w */,
177     cl::desc("Set ThinLTO cache entry expiration time."), cl::cat(LTOCategory));
178 
179 static cl::opt<std::string> ThinLTOSaveTempsPrefix(
180     "thinlto-save-temps",
181     cl::desc("Save ThinLTO temp files using filenames created by adding "
182              "suffixes to the given file path prefix."),
183     cl::cat(LTOCategory));
184 
185 static cl::opt<std::string> ThinLTOGeneratedObjectsDir(
186     "thinlto-save-objects",
187     cl::desc("Save ThinLTO generated object files using filenames created in "
188              "the given directory."),
189     cl::cat(LTOCategory));
190 
191 static cl::opt<bool> SaveLinkedModuleFile(
192     "save-linked-module", cl::init(false),
193     cl::desc("Write linked LTO module to file before optimize"),
194     cl::cat(LTOCategory));
195 
196 static cl::opt<bool>
197     SaveModuleFile("save-merged-module", cl::init(false),
198                    cl::desc("Write merged LTO module to file before CodeGen"),
199                    cl::cat(LTOCategory));
200 
201 static cl::list<std::string> InputFilenames(cl::Positional, cl::OneOrMore,
202                                             cl::desc("<input bitcode files>"),
203                                             cl::cat(LTOCategory));
204 
205 static cl::opt<std::string> OutputFilename("o", cl::init(""),
206                                            cl::desc("Override output filename"),
207                                            cl::value_desc("filename"),
208                                            cl::cat(LTOCategory));
209 
210 static cl::list<std::string> ExportedSymbols(
211     "exported-symbol",
212     cl::desc("List of symbols to export from the resulting object file"),
213     cl::cat(LTOCategory));
214 
215 static cl::list<std::string>
216     DSOSymbols("dso-symbol",
217                cl::desc("Symbol to put in the symtab in the resulting dso"),
218                cl::cat(LTOCategory));
219 
220 static cl::opt<bool> ListSymbolsOnly(
221     "list-symbols-only", cl::init(false),
222     cl::desc("Instead of running LTO, list the symbols in each IR file"),
223     cl::cat(LTOCategory));
224 
225 static cl::opt<bool> ListDependentLibrariesOnly(
226     "list-dependent-libraries-only", cl::init(false),
227     cl::desc(
228         "Instead of running LTO, list the dependent libraries in each IR file"),
229     cl::cat(LTOCategory));
230 
231 static cl::opt<bool> QueryHasCtorDtor(
232     "query-hasCtorDtor", cl::init(false),
233     cl::desc("Queries LTOModule::hasCtorDtor() on each IR file"));
234 
235 static cl::opt<bool>
236     SetMergedModule("set-merged-module", cl::init(false),
237                     cl::desc("Use the first input module as the merged module"),
238                     cl::cat(LTOCategory));
239 
240 static cl::opt<unsigned> Parallelism("j", cl::Prefix, cl::init(1),
241                                      cl::desc("Number of backend threads"),
242                                      cl::cat(LTOCategory));
243 
244 static cl::opt<bool> RestoreGlobalsLinkage(
245     "restore-linkage", cl::init(false),
246     cl::desc("Restore original linkage of globals prior to CodeGen"),
247     cl::cat(LTOCategory));
248 
249 static cl::opt<bool> CheckHasObjC(
250     "check-for-objc", cl::init(false),
251     cl::desc("Only check if the module has objective-C defined in it"),
252     cl::cat(LTOCategory));
253 
254 static cl::opt<bool> PrintMachOCPUOnly(
255     "print-macho-cpu-only", cl::init(false),
256     cl::desc("Instead of running LTO, print the mach-o cpu in each IR file"),
257     cl::cat(LTOCategory));
258 
259 static cl::opt<bool>
260     DebugPassManager("debug-pass-manager", cl::init(false), cl::Hidden,
261                      cl::desc("Print pass management debugging information"),
262                      cl::cat(LTOCategory));
263 
264 static cl::opt<bool>
265     LTOSaveBeforeOpt("lto-save-before-opt", cl::init(false),
266                      cl::desc("Save the IR before running optimizations"));
267 
268 namespace {
269 
270 struct ModuleInfo {
271   BitVector CanBeHidden;
272 };
273 
274 } // end anonymous namespace
275 
276 static void handleDiagnostics(lto_codegen_diagnostic_severity_t Severity,
277                               const char *Msg, void *) {
278   errs() << "llvm-lto: ";
279   switch (Severity) {
280   case LTO_DS_NOTE:
281     errs() << "note: ";
282     break;
283   case LTO_DS_REMARK:
284     errs() << "remark: ";
285     break;
286   case LTO_DS_ERROR:
287     errs() << "error: ";
288     break;
289   case LTO_DS_WARNING:
290     errs() << "warning: ";
291     break;
292   }
293   errs() << Msg << "\n";
294 }
295 
296 static std::string CurrentActivity;
297 
298 namespace {
299   struct LLVMLTODiagnosticHandler : public DiagnosticHandler {
300     bool handleDiagnostics(const DiagnosticInfo &DI) override {
301       raw_ostream &OS = errs();
302       OS << "llvm-lto: ";
303       switch (DI.getSeverity()) {
304       case DS_Error:
305         OS << "error";
306         break;
307       case DS_Warning:
308         OS << "warning";
309         break;
310       case DS_Remark:
311         OS << "remark";
312         break;
313       case DS_Note:
314         OS << "note";
315         break;
316       }
317       if (!CurrentActivity.empty())
318         OS << ' ' << CurrentActivity;
319       OS << ": ";
320 
321       DiagnosticPrinterRawOStream DP(OS);
322       DI.print(DP);
323       OS << '\n';
324 
325       if (DI.getSeverity() == DS_Error)
326         exit(1);
327       return true;
328     }
329   };
330   }
331 
332 static void error(const Twine &Msg) {
333   errs() << "llvm-lto: " << Msg << '\n';
334   exit(1);
335 }
336 
337 static void error(std::error_code EC, const Twine &Prefix) {
338   if (EC)
339     error(Prefix + ": " + EC.message());
340 }
341 
342 template <typename T>
343 static void error(const ErrorOr<T> &V, const Twine &Prefix) {
344   error(V.getError(), Prefix);
345 }
346 
347 static void maybeVerifyModule(const Module &Mod) {
348   if (!DisableVerify && verifyModule(Mod, &errs()))
349     error("Broken Module");
350 }
351 
352 static std::unique_ptr<LTOModule>
353 getLocalLTOModule(StringRef Path, std::unique_ptr<MemoryBuffer> &Buffer,
354                   const TargetOptions &Options) {
355   ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
356       MemoryBuffer::getFile(Path);
357   error(BufferOrErr, "error loading file '" + Path + "'");
358   Buffer = std::move(BufferOrErr.get());
359   CurrentActivity = ("loading file '" + Path + "'").str();
360   std::unique_ptr<LLVMContext> Context = std::make_unique<LLVMContext>();
361   Context->setDiagnosticHandler(std::make_unique<LLVMLTODiagnosticHandler>(),
362                                 true);
363   ErrorOr<std::unique_ptr<LTOModule>> Ret = LTOModule::createInLocalContext(
364       std::move(Context), Buffer->getBufferStart(), Buffer->getBufferSize(),
365       Options, Path);
366   CurrentActivity = "";
367   maybeVerifyModule((*Ret)->getModule());
368   return std::move(*Ret);
369 }
370 
371 /// Print some statistics on the index for each input files.
372 static void printIndexStats() {
373   for (auto &Filename : InputFilenames) {
374     ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename + "': ");
375     std::unique_ptr<ModuleSummaryIndex> Index =
376         ExitOnErr(getModuleSummaryIndexForFile(Filename));
377     // Skip files without a module summary.
378     if (!Index)
379       report_fatal_error(Twine(Filename) + " does not contain an index");
380 
381     unsigned Calls = 0, Refs = 0, Functions = 0, Alias = 0, Globals = 0;
382     for (auto &Summaries : *Index) {
383       for (auto &Summary : Summaries.second.SummaryList) {
384         Refs += Summary->refs().size();
385         if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
386           Functions++;
387           Calls += FuncSummary->calls().size();
388         } else if (isa<AliasSummary>(Summary.get()))
389           Alias++;
390         else
391           Globals++;
392       }
393     }
394     outs() << "Index " << Filename << " contains "
395            << (Alias + Globals + Functions) << " nodes (" << Functions
396            << " functions, " << Alias << " alias, " << Globals
397            << " globals) and " << (Calls + Refs) << " edges (" << Refs
398            << " refs and " << Calls << " calls)\n";
399   }
400 }
401 
402 /// Load each IR file and dump certain information based on active flags.
403 ///
404 /// The main point here is to provide lit-testable coverage for the LTOModule
405 /// functionality that's exposed by the C API. Moreover, this provides testing
406 /// coverage for modules that have been created in their own contexts.
407 static void testLTOModule(const TargetOptions &Options) {
408   for (auto &Filename : InputFilenames) {
409     std::unique_ptr<MemoryBuffer> Buffer;
410     std::unique_ptr<LTOModule> Module =
411         getLocalLTOModule(Filename, Buffer, Options);
412 
413     if (ListSymbolsOnly) {
414       // List the symbols.
415       outs() << Filename << ":\n";
416       for (int I = 0, E = Module->getSymbolCount(); I != E; ++I)
417         outs() << Module->getSymbolName(I) << "\n";
418     }
419     if (QueryHasCtorDtor)
420       outs() << Filename
421              << ": hasCtorDtor = " << (Module->hasCtorDtor() ? "true" : "false")
422              << "\n";
423   }
424 }
425 
426 static std::unique_ptr<MemoryBuffer> loadFile(StringRef Filename) {
427     ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename.str() +
428         "': ");
429     return ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename)));
430 }
431 
432 static void listDependentLibraries() {
433   for (auto &Filename : InputFilenames) {
434     auto Buffer = loadFile(Filename);
435     std::string E;
436     std::unique_ptr<lto::InputFile> Input(LTOModule::createInputFile(
437         Buffer->getBufferStart(), Buffer->getBufferSize(), Filename.c_str(),
438         E));
439     if (!Input)
440       error(E);
441 
442     // List the dependent libraries.
443     outs() << Filename << ":\n";
444     for (size_t I = 0, C = LTOModule::getDependentLibraryCount(Input.get());
445          I != C; ++I) {
446       size_t L = 0;
447       const char *S = LTOModule::getDependentLibrary(Input.get(), I, &L);
448       assert(S);
449       outs() << StringRef(S, L) << "\n";
450     }
451   }
452 }
453 
454 static void printMachOCPUOnly() {
455   LLVMContext Context;
456   Context.setDiagnosticHandler(std::make_unique<LLVMLTODiagnosticHandler>(),
457                                true);
458   TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Triple());
459   for (auto &Filename : InputFilenames) {
460     ErrorOr<std::unique_ptr<LTOModule>> ModuleOrErr =
461         LTOModule::createFromFile(Context, Filename, Options);
462     if (!ModuleOrErr)
463       error(ModuleOrErr, "llvm-lto: ");
464 
465     Expected<uint32_t> CPUType = (*ModuleOrErr)->getMachOCPUType();
466     Expected<uint32_t> CPUSubType = (*ModuleOrErr)->getMachOCPUSubType();
467     if (!CPUType)
468       error("Error while printing mach-o cputype: " +
469             toString(CPUType.takeError()));
470     if (!CPUSubType)
471       error("Error while printing mach-o cpusubtype: " +
472             toString(CPUSubType.takeError()));
473     outs() << llvm::format("%s:\ncputype: %u\ncpusubtype: %u\n",
474                            Filename.c_str(), *CPUType, *CPUSubType);
475   }
476 }
477 
478 /// Create a combined index file from the input IR files and write it.
479 ///
480 /// This is meant to enable testing of ThinLTO combined index generation,
481 /// currently available via the gold plugin via -thinlto.
482 static void createCombinedModuleSummaryIndex() {
483   ModuleSummaryIndex CombinedIndex(/*HaveGVs=*/false);
484   uint64_t NextModuleId = 0;
485   for (auto &Filename : InputFilenames) {
486     ExitOnError ExitOnErr("llvm-lto: error loading file '" + Filename + "': ");
487     std::unique_ptr<MemoryBuffer> MB =
488         ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Filename)));
489     ExitOnErr(readModuleSummaryIndex(*MB, CombinedIndex, NextModuleId++));
490   }
491   // In order to use this index for testing, specifically import testing, we
492   // need to update any indirect call edges created from SamplePGO, so that they
493   // point to the correct GUIDs.
494   updateIndirectCalls(CombinedIndex);
495   std::error_code EC;
496   assert(!OutputFilename.empty());
497   raw_fd_ostream OS(OutputFilename + ".thinlto.bc", EC,
498                     sys::fs::OpenFlags::OF_None);
499   error(EC, "error opening the file '" + OutputFilename + ".thinlto.bc'");
500   writeIndexToFile(CombinedIndex, OS);
501   OS.close();
502 }
503 
504 /// Parse the thinlto_prefix_replace option into the \p OldPrefix and
505 /// \p NewPrefix strings, if it was specified.
506 static void getThinLTOOldAndNewPrefix(std::string &OldPrefix,
507                                       std::string &NewPrefix) {
508   assert(ThinLTOPrefixReplace.empty() ||
509          ThinLTOPrefixReplace.find(';') != StringRef::npos);
510   StringRef PrefixReplace = ThinLTOPrefixReplace;
511   std::pair<StringRef, StringRef> Split = PrefixReplace.split(";");
512   OldPrefix = Split.first.str();
513   NewPrefix = Split.second.str();
514 }
515 
516 /// Given the original \p Path to an output file, replace any path
517 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
518 /// resulting directory if it does not yet exist.
519 static std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
520                                         StringRef NewPrefix) {
521   if (OldPrefix.empty() && NewPrefix.empty())
522     return std::string(Path);
523   SmallString<128> NewPath(Path);
524   llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
525   StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
526   if (!ParentPath.empty()) {
527     // Make sure the new directory exists, creating it if necessary.
528     if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
529       error(EC, "error creating the directory '" + ParentPath + "'");
530   }
531   return std::string(NewPath.str());
532 }
533 
534 namespace thinlto {
535 
536 std::vector<std::unique_ptr<MemoryBuffer>>
537 loadAllFilesForIndex(const ModuleSummaryIndex &Index) {
538   std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers;
539 
540   for (auto &ModPath : Index.modulePaths()) {
541     const auto &Filename = ModPath.first();
542     std::string CurrentActivity = ("loading file '" + Filename + "'").str();
543     auto InputOrErr = MemoryBuffer::getFile(Filename);
544     error(InputOrErr, "error " + CurrentActivity);
545     InputBuffers.push_back(std::move(*InputOrErr));
546   }
547   return InputBuffers;
548 }
549 
550 std::unique_ptr<ModuleSummaryIndex> loadCombinedIndex() {
551   if (ThinLTOIndex.empty())
552     report_fatal_error("Missing -thinlto-index for ThinLTO promotion stage");
553   ExitOnError ExitOnErr("llvm-lto: error loading file '" + ThinLTOIndex +
554                         "': ");
555   return ExitOnErr(getModuleSummaryIndexForFile(ThinLTOIndex));
556 }
557 
558 static std::unique_ptr<lto::InputFile> loadInputFile(MemoryBufferRef Buffer) {
559   ExitOnError ExitOnErr("llvm-lto: error loading input '" +
560                         Buffer.getBufferIdentifier().str() + "': ");
561   return ExitOnErr(lto::InputFile::create(Buffer));
562 }
563 
564 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile &File,
565                                                    LLVMContext &CTX) {
566   auto &Mod = File.getSingleBitcodeModule();
567   auto ModuleOrErr = Mod.parseModule(CTX);
568   if (!ModuleOrErr) {
569     handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
570       SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
571                                       SourceMgr::DK_Error, EIB.message());
572       Err.print("llvm-lto", errs());
573     });
574     report_fatal_error("Can't load module, abort.");
575   }
576   maybeVerifyModule(**ModuleOrErr);
577   if (ThinLTOModuleId.getNumOccurrences()) {
578     if (InputFilenames.size() != 1)
579       report_fatal_error("Can't override the module id for multiple files");
580     (*ModuleOrErr)->setModuleIdentifier(ThinLTOModuleId);
581   }
582   return std::move(*ModuleOrErr);
583 }
584 
585 static void writeModuleToFile(Module &TheModule, StringRef Filename) {
586   std::error_code EC;
587   raw_fd_ostream OS(Filename, EC, sys::fs::OpenFlags::OF_None);
588   error(EC, "error opening the file '" + Filename + "'");
589   maybeVerifyModule(TheModule);
590   WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
591 }
592 
593 class ThinLTOProcessing {
594 public:
595   ThinLTOCodeGenerator ThinGenerator;
596 
597   ThinLTOProcessing(const TargetOptions &Options) {
598     ThinGenerator.setCodePICModel(codegen::getExplicitRelocModel());
599     ThinGenerator.setTargetOptions(Options);
600     ThinGenerator.setCacheDir(ThinLTOCacheDir);
601     ThinGenerator.setCachePruningInterval(ThinLTOCachePruningInterval);
602     ThinGenerator.setCacheEntryExpiration(ThinLTOCacheEntryExpiration);
603     ThinGenerator.setCacheMaxSizeFiles(ThinLTOCacheMaxSizeFiles);
604     ThinGenerator.setCacheMaxSizeBytes(ThinLTOCacheMaxSizeBytes);
605     ThinGenerator.setFreestanding(EnableFreestanding);
606     ThinGenerator.setDebugPassManager(DebugPassManager);
607 
608     // Add all the exported symbols to the table of symbols to preserve.
609     for (unsigned i = 0; i < ExportedSymbols.size(); ++i)
610       ThinGenerator.preserveSymbol(ExportedSymbols[i]);
611   }
612 
613   void run() {
614     switch (ThinLTOMode) {
615     case THINLINK:
616       return thinLink();
617     case THINDISTRIBUTE:
618       return distributedIndexes();
619     case THINEMITIMPORTS:
620       return emitImports();
621     case THINPROMOTE:
622       return promote();
623     case THINIMPORT:
624       return import();
625     case THININTERNALIZE:
626       return internalize();
627     case THINOPT:
628       return optimize();
629     case THINCODEGEN:
630       return codegen();
631     case THINALL:
632       return runAll();
633     }
634   }
635 
636 private:
637   /// Load the input files, create the combined index, and write it out.
638   void thinLink() {
639     // Perform "ThinLink": just produce the index
640     if (OutputFilename.empty())
641       report_fatal_error(
642           "OutputFilename is necessary to store the combined index.\n");
643 
644     LLVMContext Ctx;
645     std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers;
646     for (unsigned i = 0; i < InputFilenames.size(); ++i) {
647       auto &Filename = InputFilenames[i];
648       std::string CurrentActivity = "loading file '" + Filename + "'";
649       auto InputOrErr = MemoryBuffer::getFile(Filename);
650       error(InputOrErr, "error " + CurrentActivity);
651       InputBuffers.push_back(std::move(*InputOrErr));
652       ThinGenerator.addModule(Filename, InputBuffers.back()->getBuffer());
653     }
654 
655     auto CombinedIndex = ThinGenerator.linkCombinedIndex();
656     if (!CombinedIndex)
657       report_fatal_error("ThinLink didn't create an index");
658     std::error_code EC;
659     raw_fd_ostream OS(OutputFilename, EC, sys::fs::OpenFlags::OF_None);
660     error(EC, "error opening the file '" + OutputFilename + "'");
661     writeIndexToFile(*CombinedIndex, OS);
662   }
663 
664   /// Load the combined index from disk, then compute and generate
665   /// individual index files suitable for ThinLTO distributed backend builds
666   /// on the files mentioned on the command line (these must match the index
667   /// content).
668   void distributedIndexes() {
669     if (InputFilenames.size() != 1 && !OutputFilename.empty())
670       report_fatal_error("Can't handle a single output filename and multiple "
671                          "input files, do not provide an output filename and "
672                          "the output files will be suffixed from the input "
673                          "ones.");
674 
675     std::string OldPrefix, NewPrefix;
676     getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
677 
678     auto Index = loadCombinedIndex();
679     for (auto &Filename : InputFilenames) {
680       LLVMContext Ctx;
681       auto Buffer = loadFile(Filename);
682       auto Input = loadInputFile(Buffer->getMemBufferRef());
683       auto TheModule = loadModuleFromInput(*Input, Ctx);
684 
685       // Build a map of module to the GUIDs and summary objects that should
686       // be written to its index.
687       std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
688       ThinGenerator.gatherImportedSummariesForModule(
689           *TheModule, *Index, ModuleToSummariesForIndex, *Input);
690 
691       std::string OutputName = OutputFilename;
692       if (OutputName.empty()) {
693         OutputName = Filename + ".thinlto.bc";
694       }
695       OutputName = getThinLTOOutputFile(OutputName, OldPrefix, NewPrefix);
696       std::error_code EC;
697       raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::OF_None);
698       error(EC, "error opening the file '" + OutputName + "'");
699       writeIndexToFile(*Index, OS, &ModuleToSummariesForIndex);
700     }
701   }
702 
703   /// Load the combined index from disk, compute the imports, and emit
704   /// the import file lists for each module to disk.
705   void emitImports() {
706     if (InputFilenames.size() != 1 && !OutputFilename.empty())
707       report_fatal_error("Can't handle a single output filename and multiple "
708                          "input files, do not provide an output filename and "
709                          "the output files will be suffixed from the input "
710                          "ones.");
711 
712     std::string OldPrefix, NewPrefix;
713     getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
714 
715     auto Index = loadCombinedIndex();
716     for (auto &Filename : InputFilenames) {
717       LLVMContext Ctx;
718       auto Buffer = loadFile(Filename);
719       auto Input = loadInputFile(Buffer->getMemBufferRef());
720       auto TheModule = loadModuleFromInput(*Input, Ctx);
721       std::string OutputName = OutputFilename;
722       if (OutputName.empty()) {
723         OutputName = Filename + ".imports";
724       }
725       OutputName =
726           getThinLTOOutputFile(OutputName, OldPrefix, NewPrefix);
727       ThinGenerator.emitImports(*TheModule, OutputName, *Index, *Input);
728     }
729   }
730 
731   /// Load the combined index from disk, then load every file referenced by
732   /// the index and add them to the generator, finally perform the promotion
733   /// on the files mentioned on the command line (these must match the index
734   /// content).
735   void promote() {
736     if (InputFilenames.size() != 1 && !OutputFilename.empty())
737       report_fatal_error("Can't handle a single output filename and multiple "
738                          "input files, do not provide an output filename and "
739                          "the output files will be suffixed from the input "
740                          "ones.");
741 
742     auto Index = loadCombinedIndex();
743     for (auto &Filename : InputFilenames) {
744       LLVMContext Ctx;
745       auto Buffer = loadFile(Filename);
746       auto Input = loadInputFile(Buffer->getMemBufferRef());
747       auto TheModule = loadModuleFromInput(*Input, Ctx);
748 
749       ThinGenerator.promote(*TheModule, *Index, *Input);
750 
751       std::string OutputName = OutputFilename;
752       if (OutputName.empty()) {
753         OutputName = Filename + ".thinlto.promoted.bc";
754       }
755       writeModuleToFile(*TheModule, OutputName);
756     }
757   }
758 
759   /// Load the combined index from disk, then load every file referenced by
760   /// the index and add them to the generator, then performs the promotion and
761   /// cross module importing on the files mentioned on the command line
762   /// (these must match the index content).
763   void import() {
764     if (InputFilenames.size() != 1 && !OutputFilename.empty())
765       report_fatal_error("Can't handle a single output filename and multiple "
766                          "input files, do not provide an output filename and "
767                          "the output files will be suffixed from the input "
768                          "ones.");
769 
770     auto Index = loadCombinedIndex();
771     auto InputBuffers = loadAllFilesForIndex(*Index);
772     for (auto &MemBuffer : InputBuffers)
773       ThinGenerator.addModule(MemBuffer->getBufferIdentifier(),
774                               MemBuffer->getBuffer());
775 
776     for (auto &Filename : InputFilenames) {
777       LLVMContext Ctx;
778       auto Buffer = loadFile(Filename);
779       auto Input = loadInputFile(Buffer->getMemBufferRef());
780       auto TheModule = loadModuleFromInput(*Input, Ctx);
781 
782       ThinGenerator.crossModuleImport(*TheModule, *Index, *Input);
783 
784       std::string OutputName = OutputFilename;
785       if (OutputName.empty()) {
786         OutputName = Filename + ".thinlto.imported.bc";
787       }
788       writeModuleToFile(*TheModule, OutputName);
789     }
790   }
791 
792   void internalize() {
793     if (InputFilenames.size() != 1 && !OutputFilename.empty())
794       report_fatal_error("Can't handle a single output filename and multiple "
795                          "input files, do not provide an output filename and "
796                          "the output files will be suffixed from the input "
797                          "ones.");
798 
799     if (ExportedSymbols.empty())
800       errs() << "Warning: -internalize will not perform without "
801                 "-exported-symbol\n";
802 
803     auto Index = loadCombinedIndex();
804     auto InputBuffers = loadAllFilesForIndex(*Index);
805     for (auto &MemBuffer : InputBuffers)
806       ThinGenerator.addModule(MemBuffer->getBufferIdentifier(),
807                               MemBuffer->getBuffer());
808 
809     for (auto &Filename : InputFilenames) {
810       LLVMContext Ctx;
811       auto Buffer = loadFile(Filename);
812       auto Input = loadInputFile(Buffer->getMemBufferRef());
813       auto TheModule = loadModuleFromInput(*Input, Ctx);
814 
815       ThinGenerator.internalize(*TheModule, *Index, *Input);
816 
817       std::string OutputName = OutputFilename;
818       if (OutputName.empty()) {
819         OutputName = Filename + ".thinlto.internalized.bc";
820       }
821       writeModuleToFile(*TheModule, OutputName);
822     }
823   }
824 
825   void optimize() {
826     if (InputFilenames.size() != 1 && !OutputFilename.empty())
827       report_fatal_error("Can't handle a single output filename and multiple "
828                          "input files, do not provide an output filename and "
829                          "the output files will be suffixed from the input "
830                          "ones.");
831     if (!ThinLTOIndex.empty())
832       errs() << "Warning: -thinlto-index ignored for optimize stage";
833 
834     for (auto &Filename : InputFilenames) {
835       LLVMContext Ctx;
836       auto Buffer = loadFile(Filename);
837       auto Input = loadInputFile(Buffer->getMemBufferRef());
838       auto TheModule = loadModuleFromInput(*Input, Ctx);
839 
840       ThinGenerator.optimize(*TheModule);
841 
842       std::string OutputName = OutputFilename;
843       if (OutputName.empty()) {
844         OutputName = Filename + ".thinlto.imported.bc";
845       }
846       writeModuleToFile(*TheModule, OutputName);
847     }
848   }
849 
850   void codegen() {
851     if (InputFilenames.size() != 1 && !OutputFilename.empty())
852       report_fatal_error("Can't handle a single output filename and multiple "
853                          "input files, do not provide an output filename and "
854                          "the output files will be suffixed from the input "
855                          "ones.");
856     if (!ThinLTOIndex.empty())
857       errs() << "Warning: -thinlto-index ignored for codegen stage";
858 
859     std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers;
860     for (auto &Filename : InputFilenames) {
861       LLVMContext Ctx;
862       auto InputOrErr = MemoryBuffer::getFile(Filename);
863       error(InputOrErr, "error " + CurrentActivity);
864       InputBuffers.push_back(std::move(*InputOrErr));
865       ThinGenerator.addModule(Filename, InputBuffers.back()->getBuffer());
866     }
867     ThinGenerator.setCodeGenOnly(true);
868     ThinGenerator.run();
869     for (auto BinName :
870          zip(ThinGenerator.getProducedBinaries(), InputFilenames)) {
871       std::string OutputName = OutputFilename;
872       if (OutputName.empty())
873         OutputName = std::get<1>(BinName) + ".thinlto.o";
874       else if (OutputName == "-") {
875         outs() << std::get<0>(BinName)->getBuffer();
876         return;
877       }
878 
879       std::error_code EC;
880       raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::OF_None);
881       error(EC, "error opening the file '" + OutputName + "'");
882       OS << std::get<0>(BinName)->getBuffer();
883     }
884   }
885 
886   /// Full ThinLTO process
887   void runAll() {
888     if (!OutputFilename.empty())
889       report_fatal_error("Do not provide an output filename for ThinLTO "
890                          " processing, the output files will be suffixed from "
891                          "the input ones.");
892 
893     if (!ThinLTOIndex.empty())
894       errs() << "Warning: -thinlto-index ignored for full ThinLTO process";
895 
896     LLVMContext Ctx;
897     std::vector<std::unique_ptr<MemoryBuffer>> InputBuffers;
898     for (unsigned i = 0; i < InputFilenames.size(); ++i) {
899       auto &Filename = InputFilenames[i];
900       std::string CurrentActivity = "loading file '" + Filename + "'";
901       auto InputOrErr = MemoryBuffer::getFile(Filename);
902       error(InputOrErr, "error " + CurrentActivity);
903       InputBuffers.push_back(std::move(*InputOrErr));
904       ThinGenerator.addModule(Filename, InputBuffers.back()->getBuffer());
905     }
906 
907     if (!ThinLTOSaveTempsPrefix.empty())
908       ThinGenerator.setSaveTempsDir(ThinLTOSaveTempsPrefix);
909 
910     if (!ThinLTOGeneratedObjectsDir.empty()) {
911       ThinGenerator.setGeneratedObjectsDirectory(ThinLTOGeneratedObjectsDir);
912       ThinGenerator.run();
913       return;
914     }
915 
916     ThinGenerator.run();
917 
918     auto &Binaries = ThinGenerator.getProducedBinaries();
919     if (Binaries.size() != InputFilenames.size())
920       report_fatal_error("Number of output objects does not match the number "
921                          "of inputs");
922 
923     for (unsigned BufID = 0; BufID < Binaries.size(); ++BufID) {
924       auto OutputName = InputFilenames[BufID] + ".thinlto.o";
925       std::error_code EC;
926       raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::OF_None);
927       error(EC, "error opening the file '" + OutputName + "'");
928       OS << Binaries[BufID]->getBuffer();
929     }
930   }
931 
932   /// Load the combined index from disk, then load every file referenced by
933 };
934 
935 } // end namespace thinlto
936 
937 int main(int argc, char **argv) {
938   InitLLVM X(argc, argv);
939   cl::HideUnrelatedOptions({&LTOCategory, &getColorCategory()});
940   cl::ParseCommandLineOptions(argc, argv, "llvm LTO linker\n");
941 
942   if (OptLevel < '0' || OptLevel > '3')
943     error("optimization level must be between 0 and 3");
944 
945   // Initialize the configured targets.
946   InitializeAllTargets();
947   InitializeAllTargetMCs();
948   InitializeAllAsmPrinters();
949   InitializeAllAsmParsers();
950 
951   // set up the TargetOptions for the machine
952   TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Triple());
953 
954   if (ListSymbolsOnly || QueryHasCtorDtor) {
955     testLTOModule(Options);
956     return 0;
957   }
958 
959   if (ListDependentLibrariesOnly) {
960     listDependentLibraries();
961     return 0;
962   }
963 
964   if (IndexStats) {
965     printIndexStats();
966     return 0;
967   }
968 
969   if (CheckHasObjC) {
970     for (auto &Filename : InputFilenames) {
971       ExitOnError ExitOnErr(std::string(*argv) + ": error loading file '" +
972                             Filename + "': ");
973       std::unique_ptr<MemoryBuffer> BufferOrErr =
974           ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(Filename)));
975       auto Buffer = std::move(BufferOrErr.get());
976       if (ExitOnErr(isBitcodeContainingObjCCategory(*Buffer)))
977         outs() << "Bitcode " << Filename << " contains ObjC\n";
978       else
979         outs() << "Bitcode " << Filename << " does not contain ObjC\n";
980     }
981     return 0;
982   }
983 
984   if (PrintMachOCPUOnly) {
985     printMachOCPUOnly();
986     return 0;
987   }
988 
989   if (ThinLTOMode.getNumOccurrences()) {
990     if (ThinLTOMode.getNumOccurrences() > 1)
991       report_fatal_error("You can't specify more than one -thinlto-action");
992     thinlto::ThinLTOProcessing ThinLTOProcessor(Options);
993     ThinLTOProcessor.run();
994     return 0;
995   }
996 
997   if (ThinLTO) {
998     createCombinedModuleSummaryIndex();
999     return 0;
1000   }
1001 
1002   unsigned BaseArg = 0;
1003 
1004   LLVMContext Context;
1005   Context.setDiagnosticHandler(std::make_unique<LLVMLTODiagnosticHandler>(),
1006                                true);
1007 
1008   LTOCodeGenerator CodeGen(Context);
1009   CodeGen.setDisableVerify(DisableVerify);
1010 
1011   if (UseDiagnosticHandler)
1012     CodeGen.setDiagnosticHandler(handleDiagnostics, nullptr);
1013 
1014   CodeGen.setCodePICModel(codegen::getExplicitRelocModel());
1015   CodeGen.setFreestanding(EnableFreestanding);
1016   CodeGen.setDebugPassManager(DebugPassManager);
1017 
1018   CodeGen.setDebugInfo(LTO_DEBUG_MODEL_DWARF);
1019   CodeGen.setTargetOptions(Options);
1020   CodeGen.setShouldRestoreGlobalsLinkage(RestoreGlobalsLinkage);
1021 
1022   StringSet<MallocAllocator> DSOSymbolsSet;
1023   for (unsigned i = 0; i < DSOSymbols.size(); ++i)
1024     DSOSymbolsSet.insert(DSOSymbols[i]);
1025 
1026   std::vector<std::string> KeptDSOSyms;
1027 
1028   for (unsigned i = BaseArg; i < InputFilenames.size(); ++i) {
1029     CurrentActivity = "loading file '" + InputFilenames[i] + "'";
1030     ErrorOr<std::unique_ptr<LTOModule>> ModuleOrErr =
1031         LTOModule::createFromFile(Context, InputFilenames[i], Options);
1032     std::unique_ptr<LTOModule> &Module = *ModuleOrErr;
1033     CurrentActivity = "";
1034 
1035     unsigned NumSyms = Module->getSymbolCount();
1036     for (unsigned I = 0; I < NumSyms; ++I) {
1037       StringRef Name = Module->getSymbolName(I);
1038       if (!DSOSymbolsSet.count(Name))
1039         continue;
1040       lto_symbol_attributes Attrs = Module->getSymbolAttributes(I);
1041       unsigned Scope = Attrs & LTO_SYMBOL_SCOPE_MASK;
1042       if (Scope != LTO_SYMBOL_SCOPE_DEFAULT_CAN_BE_HIDDEN)
1043         KeptDSOSyms.push_back(std::string(Name));
1044     }
1045 
1046     // We use the first input module as the destination module when
1047     // SetMergedModule is true.
1048     if (SetMergedModule && i == BaseArg) {
1049       // Transfer ownership to the code generator.
1050       CodeGen.setModule(std::move(Module));
1051     } else if (!CodeGen.addModule(Module.get())) {
1052       // Print a message here so that we know addModule() did not abort.
1053       error("error adding file '" + InputFilenames[i] + "'");
1054     }
1055   }
1056 
1057   // Add all the exported symbols to the table of symbols to preserve.
1058   for (unsigned i = 0; i < ExportedSymbols.size(); ++i)
1059     CodeGen.addMustPreserveSymbol(ExportedSymbols[i]);
1060 
1061   // Add all the dso symbols to the table of symbols to expose.
1062   for (unsigned i = 0; i < KeptDSOSyms.size(); ++i)
1063     CodeGen.addMustPreserveSymbol(KeptDSOSyms[i]);
1064 
1065   // Set cpu and attrs strings for the default target/subtarget.
1066   CodeGen.setCpu(codegen::getMCPU());
1067 
1068   CodeGen.setOptLevel(OptLevel - '0');
1069   CodeGen.setAttrs(codegen::getMAttrs());
1070 
1071   if (auto FT = codegen::getExplicitFileType())
1072     CodeGen.setFileType(*FT);
1073 
1074   if (!OutputFilename.empty()) {
1075     if (LTOSaveBeforeOpt)
1076       CodeGen.setSaveIRBeforeOptPath(OutputFilename + ".0.preopt.bc");
1077 
1078     if (SaveLinkedModuleFile) {
1079       std::string ModuleFilename = OutputFilename;
1080       ModuleFilename += ".linked.bc";
1081       std::string ErrMsg;
1082 
1083       if (!CodeGen.writeMergedModules(ModuleFilename))
1084         error("writing linked module failed.");
1085     }
1086 
1087     if (!CodeGen.optimize()) {
1088       // Diagnostic messages should have been printed by the handler.
1089       error("error optimizing the code");
1090     }
1091 
1092     if (SaveModuleFile) {
1093       std::string ModuleFilename = OutputFilename;
1094       ModuleFilename += ".merged.bc";
1095       std::string ErrMsg;
1096 
1097       if (!CodeGen.writeMergedModules(ModuleFilename))
1098         error("writing merged module failed.");
1099     }
1100 
1101     auto AddStream =
1102         [&](size_t Task,
1103             const Twine &ModuleName) -> std::unique_ptr<CachedFileStream> {
1104       std::string PartFilename = OutputFilename;
1105       if (Parallelism != 1)
1106         PartFilename += "." + utostr(Task);
1107 
1108       std::error_code EC;
1109       auto S =
1110           std::make_unique<raw_fd_ostream>(PartFilename, EC, sys::fs::OF_None);
1111       if (EC)
1112         error("error opening the file '" + PartFilename + "': " + EC.message());
1113       return std::make_unique<CachedFileStream>(std::move(S));
1114     };
1115 
1116     if (!CodeGen.compileOptimized(AddStream, Parallelism))
1117       // Diagnostic messages should have been printed by the handler.
1118       error("error compiling the code");
1119 
1120   } else {
1121     if (Parallelism != 1)
1122       error("-j must be specified together with -o");
1123 
1124     if (SaveModuleFile)
1125       error(": -save-merged-module must be specified with -o");
1126 
1127     const char *OutputName = nullptr;
1128     if (!CodeGen.compile_to_file(&OutputName))
1129       error("error compiling the code");
1130       // Diagnostic messages should have been printed by the handler.
1131 
1132     outs() << "Wrote native object file '" << OutputName << "'\n";
1133   }
1134 
1135   return 0;
1136 }
1137