xref: /freebsd/contrib/llvm-project/clang/lib/CodeGen/BackendUtil.cpp (revision b1879975794772ee51f0b4865753364c7d7626c3) 
1 //===--- BackendUtil.cpp - LLVM Backend Utilities -------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/CodeGen/BackendUtil.h"
10 #include "BackendConsumer.h"
11 #include "LinkInModulesPass.h"
12 #include "clang/Basic/CodeGenOptions.h"
13 #include "clang/Basic/Diagnostic.h"
14 #include "clang/Basic/LangOptions.h"
15 #include "clang/Basic/TargetOptions.h"
16 #include "clang/Frontend/FrontendDiagnostic.h"
17 #include "clang/Frontend/Utils.h"
18 #include "clang/Lex/HeaderSearchOptions.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/Analysis/AliasAnalysis.h"
23 #include "llvm/Analysis/GlobalsModRef.h"
24 #include "llvm/Analysis/TargetLibraryInfo.h"
25 #include "llvm/Analysis/TargetTransformInfo.h"
26 #include "llvm/Bitcode/BitcodeReader.h"
27 #include "llvm/Bitcode/BitcodeWriter.h"
28 #include "llvm/Bitcode/BitcodeWriterPass.h"
29 #include "llvm/CodeGen/RegAllocRegistry.h"
30 #include "llvm/CodeGen/SchedulerRegistry.h"
31 #include "llvm/CodeGen/TargetSubtargetInfo.h"
32 #include "llvm/Frontend/Driver/CodeGenOptions.h"
33 #include "llvm/IR/DataLayout.h"
34 #include "llvm/IR/DebugInfo.h"
35 #include "llvm/IR/LegacyPassManager.h"
36 #include "llvm/IR/Module.h"
37 #include "llvm/IR/ModuleSummaryIndex.h"
38 #include "llvm/IR/PassManager.h"
39 #include "llvm/IR/Verifier.h"
40 #include "llvm/IRPrinter/IRPrintingPasses.h"
41 #include "llvm/LTO/LTOBackend.h"
42 #include "llvm/MC/MCAsmInfo.h"
43 #include "llvm/MC/TargetRegistry.h"
44 #include "llvm/Object/OffloadBinary.h"
45 #include "llvm/Passes/PassBuilder.h"
46 #include "llvm/Passes/PassPlugin.h"
47 #include "llvm/Passes/StandardInstrumentations.h"
48 #include "llvm/ProfileData/InstrProfCorrelator.h"
49 #include "llvm/Support/BuryPointer.h"
50 #include "llvm/Support/CommandLine.h"
51 #include "llvm/Support/MemoryBuffer.h"
52 #include "llvm/Support/PrettyStackTrace.h"
53 #include "llvm/Support/TimeProfiler.h"
54 #include "llvm/Support/Timer.h"
55 #include "llvm/Support/ToolOutputFile.h"
56 #include "llvm/Support/VirtualFileSystem.h"
57 #include "llvm/Support/raw_ostream.h"
58 #include "llvm/Target/TargetMachine.h"
59 #include "llvm/Target/TargetOptions.h"
60 #include "llvm/TargetParser/SubtargetFeature.h"
61 #include "llvm/TargetParser/Triple.h"
62 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
63 #include "llvm/Transforms/IPO/EmbedBitcodePass.h"
64 #include "llvm/Transforms/IPO/LowerTypeTests.h"
65 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
66 #include "llvm/Transforms/InstCombine/InstCombine.h"
67 #include "llvm/Transforms/Instrumentation.h"
68 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
69 #include "llvm/Transforms/Instrumentation/AddressSanitizerOptions.h"
70 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
71 #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
72 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
73 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
74 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
75 #include "llvm/Transforms/Instrumentation/KCFI.h"
76 #include "llvm/Transforms/Instrumentation/LowerAllowCheckPass.h"
77 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
78 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
79 #include "llvm/Transforms/Instrumentation/NumericalStabilitySanitizer.h"
80 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
81 #include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h"
82 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
83 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
84 #include "llvm/Transforms/ObjCARC.h"
85 #include "llvm/Transforms/Scalar/EarlyCSE.h"
86 #include "llvm/Transforms/Scalar/GVN.h"
87 #include "llvm/Transforms/Scalar/JumpThreading.h"
88 #include "llvm/Transforms/Utils/Debugify.h"
89 #include "llvm/Transforms/Utils/ModuleUtils.h"
90 #include <memory>
91 #include <optional>
92 using namespace clang;
93 using namespace llvm;
94 
95 #define HANDLE_EXTENSION(Ext)                                                  \
96   llvm::PassPluginLibraryInfo get##Ext##PluginInfo();
97 #include "llvm/Support/Extension.def"
98 
99 namespace llvm {
100 extern cl::opt<bool> PrintPipelinePasses;
101 
102 // Experiment to move sanitizers earlier.
103 static cl::opt<bool> ClSanitizeOnOptimizerEarlyEP(
104     "sanitizer-early-opt-ep", cl::Optional,
105     cl::desc("Insert sanitizers on OptimizerEarlyEP."));
106 
107 // Experiment to mark cold functions as optsize/minsize/optnone.
108 // TODO: remove once this is exposed as a proper driver flag.
109 static cl::opt<PGOOptions::ColdFuncOpt> ClPGOColdFuncAttr(
110     "pgo-cold-func-opt", cl::init(PGOOptions::ColdFuncOpt::Default), cl::Hidden,
111     cl::desc(
112         "Function attribute to apply to cold functions as determined by PGO"),
113     cl::values(clEnumValN(PGOOptions::ColdFuncOpt::Default, "default",
114                           "Default (no attribute)"),
115                clEnumValN(PGOOptions::ColdFuncOpt::OptSize, "optsize",
116                           "Mark cold functions with optsize."),
117                clEnumValN(PGOOptions::ColdFuncOpt::MinSize, "minsize",
118                           "Mark cold functions with minsize."),
119                clEnumValN(PGOOptions::ColdFuncOpt::OptNone, "optnone",
120                           "Mark cold functions with optnone.")));
121 
122 extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
123 } // namespace llvm
124 
125 namespace {
126 
127 // Default filename used for profile generation.
128 std::string getDefaultProfileGenName() {
129   return DebugInfoCorrelate || ProfileCorrelate != InstrProfCorrelator::NONE
130              ? "default_%m.proflite"
131              : "default_%m.profraw";
132 }
133 
134 class EmitAssemblyHelper {
135   DiagnosticsEngine &Diags;
136   const HeaderSearchOptions &HSOpts;
137   const CodeGenOptions &CodeGenOpts;
138   const clang::TargetOptions &TargetOpts;
139   const LangOptions &LangOpts;
140   llvm::Module *TheModule;
141   IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS;
142 
143   Timer CodeGenerationTime;
144 
145   std::unique_ptr<raw_pwrite_stream> OS;
146 
147   Triple TargetTriple;
148 
149   TargetIRAnalysis getTargetIRAnalysis() const {
150     if (TM)
151       return TM->getTargetIRAnalysis();
152 
153     return TargetIRAnalysis();
154   }
155 
156   /// Generates the TargetMachine.
157   /// Leaves TM unchanged if it is unable to create the target machine.
158   /// Some of our clang tests specify triples which are not built
159   /// into clang. This is okay because these tests check the generated
160   /// IR, and they require DataLayout which depends on the triple.
161   /// In this case, we allow this method to fail and not report an error.
162   /// When MustCreateTM is used, we print an error if we are unable to load
163   /// the requested target.
164   void CreateTargetMachine(bool MustCreateTM);
165 
166   /// Add passes necessary to emit assembly or LLVM IR.
167   ///
168   /// \return True on success.
169   bool AddEmitPasses(legacy::PassManager &CodeGenPasses, BackendAction Action,
170                      raw_pwrite_stream &OS, raw_pwrite_stream *DwoOS);
171 
172   std::unique_ptr<llvm::ToolOutputFile> openOutputFile(StringRef Path) {
173     std::error_code EC;
174     auto F = std::make_unique<llvm::ToolOutputFile>(Path, EC,
175                                                      llvm::sys::fs::OF_None);
176     if (EC) {
177       Diags.Report(diag::err_fe_unable_to_open_output) << Path << EC.message();
178       F.reset();
179     }
180     return F;
181   }
182 
183   void RunOptimizationPipeline(
184       BackendAction Action, std::unique_ptr<raw_pwrite_stream> &OS,
185       std::unique_ptr<llvm::ToolOutputFile> &ThinLinkOS, BackendConsumer *BC);
186   void RunCodegenPipeline(BackendAction Action,
187                           std::unique_ptr<raw_pwrite_stream> &OS,
188                           std::unique_ptr<llvm::ToolOutputFile> &DwoOS);
189 
190   /// Check whether we should emit a module summary for regular LTO.
191   /// The module summary should be emitted by default for regular LTO
192   /// except for ld64 targets.
193   ///
194   /// \return True if the module summary should be emitted.
195   bool shouldEmitRegularLTOSummary() const {
196     return CodeGenOpts.PrepareForLTO && !CodeGenOpts.DisableLLVMPasses &&
197            TargetTriple.getVendor() != llvm::Triple::Apple;
198   }
199 
200   /// Check whether we should emit a flag for UnifiedLTO.
201   /// The UnifiedLTO module flag should be set when UnifiedLTO is enabled for
202   /// ThinLTO or Full LTO with module summaries.
203   bool shouldEmitUnifiedLTOModueFlag() const {
204     return CodeGenOpts.UnifiedLTO &&
205            (CodeGenOpts.PrepareForThinLTO || shouldEmitRegularLTOSummary());
206   }
207 
208 public:
209   EmitAssemblyHelper(DiagnosticsEngine &_Diags,
210                      const HeaderSearchOptions &HeaderSearchOpts,
211                      const CodeGenOptions &CGOpts,
212                      const clang::TargetOptions &TOpts,
213                      const LangOptions &LOpts, llvm::Module *M,
214                      IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS)
215       : Diags(_Diags), HSOpts(HeaderSearchOpts), CodeGenOpts(CGOpts),
216         TargetOpts(TOpts), LangOpts(LOpts), TheModule(M), VFS(std::move(VFS)),
217         CodeGenerationTime("codegen", "Code Generation Time"),
218         TargetTriple(TheModule->getTargetTriple()) {}
219 
220   ~EmitAssemblyHelper() {
221     if (CodeGenOpts.DisableFree)
222       BuryPointer(std::move(TM));
223   }
224 
225   std::unique_ptr<TargetMachine> TM;
226 
227   // Emit output using the new pass manager for the optimization pipeline.
228   void EmitAssembly(BackendAction Action, std::unique_ptr<raw_pwrite_stream> OS,
229                     BackendConsumer *BC);
230 };
231 } // namespace
232 
233 static SanitizerCoverageOptions
234 getSancovOptsFromCGOpts(const CodeGenOptions &CGOpts) {
235   SanitizerCoverageOptions Opts;
236   Opts.CoverageType =
237       static_cast<SanitizerCoverageOptions::Type>(CGOpts.SanitizeCoverageType);
238   Opts.IndirectCalls = CGOpts.SanitizeCoverageIndirectCalls;
239   Opts.TraceBB = CGOpts.SanitizeCoverageTraceBB;
240   Opts.TraceCmp = CGOpts.SanitizeCoverageTraceCmp;
241   Opts.TraceDiv = CGOpts.SanitizeCoverageTraceDiv;
242   Opts.TraceGep = CGOpts.SanitizeCoverageTraceGep;
243   Opts.Use8bitCounters = CGOpts.SanitizeCoverage8bitCounters;
244   Opts.TracePC = CGOpts.SanitizeCoverageTracePC;
245   Opts.TracePCGuard = CGOpts.SanitizeCoverageTracePCGuard;
246   Opts.NoPrune = CGOpts.SanitizeCoverageNoPrune;
247   Opts.Inline8bitCounters = CGOpts.SanitizeCoverageInline8bitCounters;
248   Opts.InlineBoolFlag = CGOpts.SanitizeCoverageInlineBoolFlag;
249   Opts.PCTable = CGOpts.SanitizeCoveragePCTable;
250   Opts.StackDepth = CGOpts.SanitizeCoverageStackDepth;
251   Opts.TraceLoads = CGOpts.SanitizeCoverageTraceLoads;
252   Opts.TraceStores = CGOpts.SanitizeCoverageTraceStores;
253   Opts.CollectControlFlow = CGOpts.SanitizeCoverageControlFlow;
254   return Opts;
255 }
256 
257 static SanitizerBinaryMetadataOptions
258 getSanitizerBinaryMetadataOptions(const CodeGenOptions &CGOpts) {
259   SanitizerBinaryMetadataOptions Opts;
260   Opts.Covered = CGOpts.SanitizeBinaryMetadataCovered;
261   Opts.Atomics = CGOpts.SanitizeBinaryMetadataAtomics;
262   Opts.UAR = CGOpts.SanitizeBinaryMetadataUAR;
263   return Opts;
264 }
265 
266 // Check if ASan should use GC-friendly instrumentation for globals.
267 // First of all, there is no point if -fdata-sections is off (expect for MachO,
268 // where this is not a factor). Also, on ELF this feature requires an assembler
269 // extension that only works with -integrated-as at the moment.
270 static bool asanUseGlobalsGC(const Triple &T, const CodeGenOptions &CGOpts) {
271   if (!CGOpts.SanitizeAddressGlobalsDeadStripping)
272     return false;
273   switch (T.getObjectFormat()) {
274   case Triple::MachO:
275   case Triple::COFF:
276     return true;
277   case Triple::ELF:
278     return !CGOpts.DisableIntegratedAS;
279   case Triple::GOFF:
280     llvm::report_fatal_error("ASan not implemented for GOFF");
281   case Triple::XCOFF:
282     llvm::report_fatal_error("ASan not implemented for XCOFF.");
283   case Triple::Wasm:
284   case Triple::DXContainer:
285   case Triple::SPIRV:
286   case Triple::UnknownObjectFormat:
287     break;
288   }
289   return false;
290 }
291 
292 static std::optional<llvm::CodeModel::Model>
293 getCodeModel(const CodeGenOptions &CodeGenOpts) {
294   unsigned CodeModel = llvm::StringSwitch<unsigned>(CodeGenOpts.CodeModel)
295                            .Case("tiny", llvm::CodeModel::Tiny)
296                            .Case("small", llvm::CodeModel::Small)
297                            .Case("kernel", llvm::CodeModel::Kernel)
298                            .Case("medium", llvm::CodeModel::Medium)
299                            .Case("large", llvm::CodeModel::Large)
300                            .Case("default", ~1u)
301                            .Default(~0u);
302   assert(CodeModel != ~0u && "invalid code model!");
303   if (CodeModel == ~1u)
304     return std::nullopt;
305   return static_cast<llvm::CodeModel::Model>(CodeModel);
306 }
307 
308 static CodeGenFileType getCodeGenFileType(BackendAction Action) {
309   if (Action == Backend_EmitObj)
310     return CodeGenFileType::ObjectFile;
311   else if (Action == Backend_EmitMCNull)
312     return CodeGenFileType::Null;
313   else {
314     assert(Action == Backend_EmitAssembly && "Invalid action!");
315     return CodeGenFileType::AssemblyFile;
316   }
317 }
318 
319 static bool actionRequiresCodeGen(BackendAction Action) {
320   return Action != Backend_EmitNothing && Action != Backend_EmitBC &&
321          Action != Backend_EmitLL;
322 }
323 
324 static bool initTargetOptions(DiagnosticsEngine &Diags,
325                               llvm::TargetOptions &Options,
326                               const CodeGenOptions &CodeGenOpts,
327                               const clang::TargetOptions &TargetOpts,
328                               const LangOptions &LangOpts,
329                               const HeaderSearchOptions &HSOpts) {
330   switch (LangOpts.getThreadModel()) {
331   case LangOptions::ThreadModelKind::POSIX:
332     Options.ThreadModel = llvm::ThreadModel::POSIX;
333     break;
334   case LangOptions::ThreadModelKind::Single:
335     Options.ThreadModel = llvm::ThreadModel::Single;
336     break;
337   }
338 
339   // Set float ABI type.
340   assert((CodeGenOpts.FloatABI == "soft" || CodeGenOpts.FloatABI == "softfp" ||
341           CodeGenOpts.FloatABI == "hard" || CodeGenOpts.FloatABI.empty()) &&
342          "Invalid Floating Point ABI!");
343   Options.FloatABIType =
344       llvm::StringSwitch<llvm::FloatABI::ABIType>(CodeGenOpts.FloatABI)
345           .Case("soft", llvm::FloatABI::Soft)
346           .Case("softfp", llvm::FloatABI::Soft)
347           .Case("hard", llvm::FloatABI::Hard)
348           .Default(llvm::FloatABI::Default);
349 
350   // Set FP fusion mode.
351   switch (LangOpts.getDefaultFPContractMode()) {
352   case LangOptions::FPM_Off:
353     // Preserve any contraction performed by the front-end.  (Strict performs
354     // splitting of the muladd intrinsic in the backend.)
355     Options.AllowFPOpFusion = llvm::FPOpFusion::Standard;
356     break;
357   case LangOptions::FPM_On:
358   case LangOptions::FPM_FastHonorPragmas:
359     Options.AllowFPOpFusion = llvm::FPOpFusion::Standard;
360     break;
361   case LangOptions::FPM_Fast:
362     Options.AllowFPOpFusion = llvm::FPOpFusion::Fast;
363     break;
364   }
365 
366   Options.BinutilsVersion =
367       llvm::TargetMachine::parseBinutilsVersion(CodeGenOpts.BinutilsVersion);
368   Options.UseInitArray = CodeGenOpts.UseInitArray;
369   Options.DisableIntegratedAS = CodeGenOpts.DisableIntegratedAS;
370 
371   // Set EABI version.
372   Options.EABIVersion = TargetOpts.EABIVersion;
373 
374   if (LangOpts.hasSjLjExceptions())
375     Options.ExceptionModel = llvm::ExceptionHandling::SjLj;
376   if (LangOpts.hasSEHExceptions())
377     Options.ExceptionModel = llvm::ExceptionHandling::WinEH;
378   if (LangOpts.hasDWARFExceptions())
379     Options.ExceptionModel = llvm::ExceptionHandling::DwarfCFI;
380   if (LangOpts.hasWasmExceptions())
381     Options.ExceptionModel = llvm::ExceptionHandling::Wasm;
382 
383   Options.NoInfsFPMath = LangOpts.NoHonorInfs;
384   Options.NoNaNsFPMath = LangOpts.NoHonorNaNs;
385   Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS;
386   Options.UnsafeFPMath = LangOpts.AllowFPReassoc && LangOpts.AllowRecip &&
387                          LangOpts.NoSignedZero && LangOpts.ApproxFunc &&
388                          (LangOpts.getDefaultFPContractMode() ==
389                               LangOptions::FPModeKind::FPM_Fast ||
390                           LangOpts.getDefaultFPContractMode() ==
391                               LangOptions::FPModeKind::FPM_FastHonorPragmas);
392   Options.ApproxFuncFPMath = LangOpts.ApproxFunc;
393 
394   Options.BBAddrMap = CodeGenOpts.BBAddrMap;
395   Options.BBSections =
396       llvm::StringSwitch<llvm::BasicBlockSection>(CodeGenOpts.BBSections)
397           .Case("all", llvm::BasicBlockSection::All)
398           .Case("labels", llvm::BasicBlockSection::Labels)
399           .StartsWith("list=", llvm::BasicBlockSection::List)
400           .Case("none", llvm::BasicBlockSection::None)
401           .Default(llvm::BasicBlockSection::None);
402 
403   if (Options.BBSections == llvm::BasicBlockSection::List) {
404     ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
405         MemoryBuffer::getFile(CodeGenOpts.BBSections.substr(5));
406     if (!MBOrErr) {
407       Diags.Report(diag::err_fe_unable_to_load_basic_block_sections_file)
408           << MBOrErr.getError().message();
409       return false;
410     }
411     Options.BBSectionsFuncListBuf = std::move(*MBOrErr);
412   }
413 
414   Options.EnableMachineFunctionSplitter = CodeGenOpts.SplitMachineFunctions;
415   Options.FunctionSections = CodeGenOpts.FunctionSections;
416   Options.DataSections = CodeGenOpts.DataSections;
417   Options.IgnoreXCOFFVisibility = LangOpts.IgnoreXCOFFVisibility;
418   Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames;
419   Options.UniqueBasicBlockSectionNames =
420       CodeGenOpts.UniqueBasicBlockSectionNames;
421   Options.SeparateNamedSections = CodeGenOpts.SeparateNamedSections;
422   Options.TLSSize = CodeGenOpts.TLSSize;
423   Options.EnableTLSDESC = CodeGenOpts.EnableTLSDESC;
424   Options.EmulatedTLS = CodeGenOpts.EmulatedTLS;
425   Options.DebuggerTuning = CodeGenOpts.getDebuggerTuning();
426   Options.EmitStackSizeSection = CodeGenOpts.StackSizeSection;
427   Options.StackUsageOutput = CodeGenOpts.StackUsageOutput;
428   Options.EmitAddrsig = CodeGenOpts.Addrsig;
429   Options.ForceDwarfFrameSection = CodeGenOpts.ForceDwarfFrameSection;
430   Options.EmitCallSiteInfo = CodeGenOpts.EmitCallSiteInfo;
431   Options.EnableAIXExtendedAltivecABI = LangOpts.EnableAIXExtendedAltivecABI;
432   Options.XRayFunctionIndex = CodeGenOpts.XRayFunctionIndex;
433   Options.LoopAlignment = CodeGenOpts.LoopAlignment;
434   Options.DebugStrictDwarf = CodeGenOpts.DebugStrictDwarf;
435   Options.ObjectFilenameForDebug = CodeGenOpts.ObjectFilenameForDebug;
436   Options.Hotpatch = CodeGenOpts.HotPatch;
437   Options.JMCInstrument = CodeGenOpts.JMCInstrument;
438   Options.XCOFFReadOnlyPointers = CodeGenOpts.XCOFFReadOnlyPointers;
439 
440   switch (CodeGenOpts.getSwiftAsyncFramePointer()) {
441   case CodeGenOptions::SwiftAsyncFramePointerKind::Auto:
442     Options.SwiftAsyncFramePointer =
443         SwiftAsyncFramePointerMode::DeploymentBased;
444     break;
445 
446   case CodeGenOptions::SwiftAsyncFramePointerKind::Always:
447     Options.SwiftAsyncFramePointer = SwiftAsyncFramePointerMode::Always;
448     break;
449 
450   case CodeGenOptions::SwiftAsyncFramePointerKind::Never:
451     Options.SwiftAsyncFramePointer = SwiftAsyncFramePointerMode::Never;
452     break;
453   }
454 
455   Options.MCOptions.SplitDwarfFile = CodeGenOpts.SplitDwarfFile;
456   Options.MCOptions.EmitDwarfUnwind = CodeGenOpts.getEmitDwarfUnwind();
457   Options.MCOptions.EmitCompactUnwindNonCanonical =
458       CodeGenOpts.EmitCompactUnwindNonCanonical;
459   Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll;
460   Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels;
461   Options.MCOptions.MCUseDwarfDirectory =
462       CodeGenOpts.NoDwarfDirectoryAsm
463           ? llvm::MCTargetOptions::DisableDwarfDirectory
464           : llvm::MCTargetOptions::EnableDwarfDirectory;
465   Options.MCOptions.MCNoExecStack = CodeGenOpts.NoExecStack;
466   Options.MCOptions.MCIncrementalLinkerCompatible =
467       CodeGenOpts.IncrementalLinkerCompatible;
468   Options.MCOptions.MCFatalWarnings = CodeGenOpts.FatalWarnings;
469   Options.MCOptions.MCNoWarn = CodeGenOpts.NoWarn;
470   Options.MCOptions.AsmVerbose = CodeGenOpts.AsmVerbose;
471   Options.MCOptions.Dwarf64 = CodeGenOpts.Dwarf64;
472   Options.MCOptions.PreserveAsmComments = CodeGenOpts.PreserveAsmComments;
473   Options.MCOptions.Crel = CodeGenOpts.Crel;
474   Options.MCOptions.X86RelaxRelocations = CodeGenOpts.RelaxELFRelocations;
475   Options.MCOptions.CompressDebugSections =
476       CodeGenOpts.getCompressDebugSections();
477   Options.MCOptions.ABIName = TargetOpts.ABI;
478   for (const auto &Entry : HSOpts.UserEntries)
479     if (!Entry.IsFramework &&
480         (Entry.Group == frontend::IncludeDirGroup::Quoted ||
481          Entry.Group == frontend::IncludeDirGroup::Angled ||
482          Entry.Group == frontend::IncludeDirGroup::System))
483       Options.MCOptions.IASSearchPaths.push_back(
484           Entry.IgnoreSysRoot ? Entry.Path : HSOpts.Sysroot + Entry.Path);
485   Options.MCOptions.Argv0 = CodeGenOpts.Argv0;
486   Options.MCOptions.CommandLineArgs = CodeGenOpts.CommandLineArgs;
487   Options.MCOptions.AsSecureLogFile = CodeGenOpts.AsSecureLogFile;
488   Options.MCOptions.PPCUseFullRegisterNames =
489       CodeGenOpts.PPCUseFullRegisterNames;
490   Options.MisExpect = CodeGenOpts.MisExpect;
491 
492   return true;
493 }
494 
495 static std::optional<GCOVOptions>
496 getGCOVOptions(const CodeGenOptions &CodeGenOpts, const LangOptions &LangOpts) {
497   if (CodeGenOpts.CoverageNotesFile.empty() &&
498       CodeGenOpts.CoverageDataFile.empty())
499     return std::nullopt;
500   // Not using 'GCOVOptions::getDefault' allows us to avoid exiting if
501   // LLVM's -default-gcov-version flag is set to something invalid.
502   GCOVOptions Options;
503   Options.EmitNotes = !CodeGenOpts.CoverageNotesFile.empty();
504   Options.EmitData = !CodeGenOpts.CoverageDataFile.empty();
505   llvm::copy(CodeGenOpts.CoverageVersion, std::begin(Options.Version));
506   Options.NoRedZone = CodeGenOpts.DisableRedZone;
507   Options.Filter = CodeGenOpts.ProfileFilterFiles;
508   Options.Exclude = CodeGenOpts.ProfileExcludeFiles;
509   Options.Atomic = CodeGenOpts.AtomicProfileUpdate;
510   return Options;
511 }
512 
513 static std::optional<InstrProfOptions>
514 getInstrProfOptions(const CodeGenOptions &CodeGenOpts,
515                     const LangOptions &LangOpts) {
516   if (!CodeGenOpts.hasProfileClangInstr())
517     return std::nullopt;
518   InstrProfOptions Options;
519   Options.NoRedZone = CodeGenOpts.DisableRedZone;
520   Options.InstrProfileOutput = CodeGenOpts.InstrProfileOutput;
521   Options.Atomic = CodeGenOpts.AtomicProfileUpdate;
522   return Options;
523 }
524 
525 static void setCommandLineOpts(const CodeGenOptions &CodeGenOpts) {
526   SmallVector<const char *, 16> BackendArgs;
527   BackendArgs.push_back("clang"); // Fake program name.
528   if (!CodeGenOpts.DebugPass.empty()) {
529     BackendArgs.push_back("-debug-pass");
530     BackendArgs.push_back(CodeGenOpts.DebugPass.c_str());
531   }
532   if (!CodeGenOpts.LimitFloatPrecision.empty()) {
533     BackendArgs.push_back("-limit-float-precision");
534     BackendArgs.push_back(CodeGenOpts.LimitFloatPrecision.c_str());
535   }
536   // Check for the default "clang" invocation that won't set any cl::opt values.
537   // Skip trying to parse the command line invocation to avoid the issues
538   // described below.
539   if (BackendArgs.size() == 1)
540     return;
541   BackendArgs.push_back(nullptr);
542   // FIXME: The command line parser below is not thread-safe and shares a global
543   // state, so this call might crash or overwrite the options of another Clang
544   // instance in the same process.
545   llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1,
546                                     BackendArgs.data());
547 }
548 
549 void EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) {
550   // Create the TargetMachine for generating code.
551   std::string Error;
552   std::string Triple = TheModule->getTargetTriple();
553   const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
554   if (!TheTarget) {
555     if (MustCreateTM)
556       Diags.Report(diag::err_fe_unable_to_create_target) << Error;
557     return;
558   }
559 
560   std::optional<llvm::CodeModel::Model> CM = getCodeModel(CodeGenOpts);
561   std::string FeaturesStr =
562       llvm::join(TargetOpts.Features.begin(), TargetOpts.Features.end(), ",");
563   llvm::Reloc::Model RM = CodeGenOpts.RelocationModel;
564   std::optional<CodeGenOptLevel> OptLevelOrNone =
565       CodeGenOpt::getLevel(CodeGenOpts.OptimizationLevel);
566   assert(OptLevelOrNone && "Invalid optimization level!");
567   CodeGenOptLevel OptLevel = *OptLevelOrNone;
568 
569   llvm::TargetOptions Options;
570   if (!initTargetOptions(Diags, Options, CodeGenOpts, TargetOpts, LangOpts,
571                          HSOpts))
572     return;
573   TM.reset(TheTarget->createTargetMachine(Triple, TargetOpts.CPU, FeaturesStr,
574                                           Options, RM, CM, OptLevel));
575   TM->setLargeDataThreshold(CodeGenOpts.LargeDataThreshold);
576 }
577 
578 bool EmitAssemblyHelper::AddEmitPasses(legacy::PassManager &CodeGenPasses,
579                                        BackendAction Action,
580                                        raw_pwrite_stream &OS,
581                                        raw_pwrite_stream *DwoOS) {
582   // Add LibraryInfo.
583   std::unique_ptr<TargetLibraryInfoImpl> TLII(
584       llvm::driver::createTLII(TargetTriple, CodeGenOpts.getVecLib()));
585   CodeGenPasses.add(new TargetLibraryInfoWrapperPass(*TLII));
586 
587   // Normal mode, emit a .s or .o file by running the code generator. Note,
588   // this also adds codegenerator level optimization passes.
589   CodeGenFileType CGFT = getCodeGenFileType(Action);
590 
591   // Add ObjC ARC final-cleanup optimizations. This is done as part of the
592   // "codegen" passes so that it isn't run multiple times when there is
593   // inlining happening.
594   if (CodeGenOpts.OptimizationLevel > 0)
595     CodeGenPasses.add(createObjCARCContractPass());
596 
597   if (TM->addPassesToEmitFile(CodeGenPasses, OS, DwoOS, CGFT,
598                               /*DisableVerify=*/!CodeGenOpts.VerifyModule)) {
599     Diags.Report(diag::err_fe_unable_to_interface_with_target);
600     return false;
601   }
602 
603   return true;
604 }
605 
606 static OptimizationLevel mapToLevel(const CodeGenOptions &Opts) {
607   switch (Opts.OptimizationLevel) {
608   default:
609     llvm_unreachable("Invalid optimization level!");
610 
611   case 0:
612     return OptimizationLevel::O0;
613 
614   case 1:
615     return OptimizationLevel::O1;
616 
617   case 2:
618     switch (Opts.OptimizeSize) {
619     default:
620       llvm_unreachable("Invalid optimization level for size!");
621 
622     case 0:
623       return OptimizationLevel::O2;
624 
625     case 1:
626       return OptimizationLevel::Os;
627 
628     case 2:
629       return OptimizationLevel::Oz;
630     }
631 
632   case 3:
633     return OptimizationLevel::O3;
634   }
635 }
636 
637 static void addKCFIPass(const Triple &TargetTriple, const LangOptions &LangOpts,
638                         PassBuilder &PB) {
639   // If the back-end supports KCFI operand bundle lowering, skip KCFIPass.
640   if (TargetTriple.getArch() == llvm::Triple::x86_64 ||
641       TargetTriple.isAArch64(64) || TargetTriple.isRISCV())
642     return;
643 
644   // Ensure we lower KCFI operand bundles with -O0.
645   PB.registerOptimizerLastEPCallback(
646       [&](ModulePassManager &MPM, OptimizationLevel Level) {
647         if (Level == OptimizationLevel::O0 &&
648             LangOpts.Sanitize.has(SanitizerKind::KCFI))
649           MPM.addPass(createModuleToFunctionPassAdaptor(KCFIPass()));
650       });
651 
652   // When optimizations are requested, run KCIFPass after InstCombine to
653   // avoid unnecessary checks.
654   PB.registerPeepholeEPCallback(
655       [&](FunctionPassManager &FPM, OptimizationLevel Level) {
656         if (Level != OptimizationLevel::O0 &&
657             LangOpts.Sanitize.has(SanitizerKind::KCFI))
658           FPM.addPass(KCFIPass());
659       });
660 }
661 
662 static void addSanitizers(const Triple &TargetTriple,
663                           const CodeGenOptions &CodeGenOpts,
664                           const LangOptions &LangOpts, PassBuilder &PB) {
665   auto SanitizersCallback = [&](ModulePassManager &MPM,
666                                 OptimizationLevel Level) {
667     if (CodeGenOpts.hasSanitizeCoverage()) {
668       auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts);
669       MPM.addPass(SanitizerCoveragePass(
670           SancovOpts, CodeGenOpts.SanitizeCoverageAllowlistFiles,
671           CodeGenOpts.SanitizeCoverageIgnorelistFiles));
672     }
673 
674     if (CodeGenOpts.hasSanitizeBinaryMetadata()) {
675       MPM.addPass(SanitizerBinaryMetadataPass(
676           getSanitizerBinaryMetadataOptions(CodeGenOpts),
677           CodeGenOpts.SanitizeMetadataIgnorelistFiles));
678     }
679 
680     auto MSanPass = [&](SanitizerMask Mask, bool CompileKernel) {
681       if (LangOpts.Sanitize.has(Mask)) {
682         int TrackOrigins = CodeGenOpts.SanitizeMemoryTrackOrigins;
683         bool Recover = CodeGenOpts.SanitizeRecover.has(Mask);
684 
685         MemorySanitizerOptions options(TrackOrigins, Recover, CompileKernel,
686                                        CodeGenOpts.SanitizeMemoryParamRetval);
687         MPM.addPass(MemorySanitizerPass(options));
688         if (Level != OptimizationLevel::O0) {
689           // MemorySanitizer inserts complex instrumentation that mostly follows
690           // the logic of the original code, but operates on "shadow" values. It
691           // can benefit from re-running some general purpose optimization
692           // passes.
693           MPM.addPass(RequireAnalysisPass<GlobalsAA, llvm::Module>());
694           FunctionPassManager FPM;
695           FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
696           FPM.addPass(InstCombinePass());
697           FPM.addPass(JumpThreadingPass());
698           FPM.addPass(GVNPass());
699           FPM.addPass(InstCombinePass());
700           MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
701         }
702       }
703     };
704     MSanPass(SanitizerKind::Memory, false);
705     MSanPass(SanitizerKind::KernelMemory, true);
706 
707     if (LangOpts.Sanitize.has(SanitizerKind::Thread)) {
708       MPM.addPass(ModuleThreadSanitizerPass());
709       MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
710     }
711 
712     if (LangOpts.Sanitize.has(SanitizerKind::NumericalStability))
713       MPM.addPass(NumericalStabilitySanitizerPass());
714 
715     auto ASanPass = [&](SanitizerMask Mask, bool CompileKernel) {
716       if (LangOpts.Sanitize.has(Mask)) {
717         bool UseGlobalGC = asanUseGlobalsGC(TargetTriple, CodeGenOpts);
718         bool UseOdrIndicator = CodeGenOpts.SanitizeAddressUseOdrIndicator;
719         llvm::AsanDtorKind DestructorKind =
720             CodeGenOpts.getSanitizeAddressDtor();
721         AddressSanitizerOptions Opts;
722         Opts.CompileKernel = CompileKernel;
723         Opts.Recover = CodeGenOpts.SanitizeRecover.has(Mask);
724         Opts.UseAfterScope = CodeGenOpts.SanitizeAddressUseAfterScope;
725         Opts.UseAfterReturn = CodeGenOpts.getSanitizeAddressUseAfterReturn();
726         MPM.addPass(AddressSanitizerPass(Opts, UseGlobalGC, UseOdrIndicator,
727                                          DestructorKind));
728       }
729     };
730     ASanPass(SanitizerKind::Address, false);
731     ASanPass(SanitizerKind::KernelAddress, true);
732 
733     auto HWASanPass = [&](SanitizerMask Mask, bool CompileKernel) {
734       if (LangOpts.Sanitize.has(Mask)) {
735         bool Recover = CodeGenOpts.SanitizeRecover.has(Mask);
736         MPM.addPass(HWAddressSanitizerPass(
737             {CompileKernel, Recover,
738              /*DisableOptimization=*/CodeGenOpts.OptimizationLevel == 0}));
739       }
740     };
741     HWASanPass(SanitizerKind::HWAddress, false);
742     HWASanPass(SanitizerKind::KernelHWAddress, true);
743 
744     if (LangOpts.Sanitize.has(SanitizerKind::DataFlow)) {
745       MPM.addPass(DataFlowSanitizerPass(LangOpts.NoSanitizeFiles));
746     }
747   };
748   if (ClSanitizeOnOptimizerEarlyEP) {
749     PB.registerOptimizerEarlyEPCallback(
750         [SanitizersCallback](ModulePassManager &MPM, OptimizationLevel Level) {
751           ModulePassManager NewMPM;
752           SanitizersCallback(NewMPM, Level);
753           if (!NewMPM.isEmpty()) {
754             // Sanitizers can abandon<GlobalsAA>.
755             NewMPM.addPass(RequireAnalysisPass<GlobalsAA, llvm::Module>());
756             MPM.addPass(std::move(NewMPM));
757           }
758         });
759   } else {
760     // LastEP does not need GlobalsAA.
761     PB.registerOptimizerLastEPCallback(SanitizersCallback);
762   }
763 
764   if (LowerAllowCheckPass::IsRequested()) {
765     // We can optimize after inliner, and PGO profile matching. The hook below
766     // is called at the end `buildFunctionSimplificationPipeline`, which called
767     // from `buildInlinerPipeline`, which called after profile matching.
768     PB.registerScalarOptimizerLateEPCallback(
769         [](FunctionPassManager &FPM, OptimizationLevel Level) {
770           FPM.addPass(LowerAllowCheckPass());
771         });
772   }
773 }
774 
775 void EmitAssemblyHelper::RunOptimizationPipeline(
776     BackendAction Action, std::unique_ptr<raw_pwrite_stream> &OS,
777     std::unique_ptr<llvm::ToolOutputFile> &ThinLinkOS, BackendConsumer *BC) {
778   std::optional<PGOOptions> PGOOpt;
779 
780   if (CodeGenOpts.hasProfileIRInstr())
781     // -fprofile-generate.
782     PGOOpt = PGOOptions(
783         CodeGenOpts.InstrProfileOutput.empty() ? getDefaultProfileGenName()
784                                                : CodeGenOpts.InstrProfileOutput,
785         "", "", CodeGenOpts.MemoryProfileUsePath, nullptr, PGOOptions::IRInstr,
786         PGOOptions::NoCSAction, ClPGOColdFuncAttr,
787         CodeGenOpts.DebugInfoForProfiling,
788         /*PseudoProbeForProfiling=*/false, CodeGenOpts.AtomicProfileUpdate);
789   else if (CodeGenOpts.hasProfileIRUse()) {
790     // -fprofile-use.
791     auto CSAction = CodeGenOpts.hasProfileCSIRUse() ? PGOOptions::CSIRUse
792                                                     : PGOOptions::NoCSAction;
793     PGOOpt = PGOOptions(CodeGenOpts.ProfileInstrumentUsePath, "",
794                         CodeGenOpts.ProfileRemappingFile,
795                         CodeGenOpts.MemoryProfileUsePath, VFS,
796                         PGOOptions::IRUse, CSAction, ClPGOColdFuncAttr,
797                         CodeGenOpts.DebugInfoForProfiling);
798   } else if (!CodeGenOpts.SampleProfileFile.empty())
799     // -fprofile-sample-use
800     PGOOpt = PGOOptions(
801         CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile,
802         CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::SampleUse,
803         PGOOptions::NoCSAction, ClPGOColdFuncAttr,
804         CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling);
805   else if (!CodeGenOpts.MemoryProfileUsePath.empty())
806     // -fmemory-profile-use (without any of the above options)
807     PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath, VFS,
808                         PGOOptions::NoAction, PGOOptions::NoCSAction,
809                         ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling);
810   else if (CodeGenOpts.PseudoProbeForProfiling)
811     // -fpseudo-probe-for-profiling
812     PGOOpt =
813         PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
814                    PGOOptions::NoAction, PGOOptions::NoCSAction,
815                    ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling, true);
816   else if (CodeGenOpts.DebugInfoForProfiling)
817     // -fdebug-info-for-profiling
818     PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
819                         PGOOptions::NoAction, PGOOptions::NoCSAction,
820                         ClPGOColdFuncAttr, true);
821 
822   // Check to see if we want to generate a CS profile.
823   if (CodeGenOpts.hasProfileCSIRInstr()) {
824     assert(!CodeGenOpts.hasProfileCSIRUse() &&
825            "Cannot have both CSProfileUse pass and CSProfileGen pass at "
826            "the same time");
827     if (PGOOpt) {
828       assert(PGOOpt->Action != PGOOptions::IRInstr &&
829              PGOOpt->Action != PGOOptions::SampleUse &&
830              "Cannot run CSProfileGen pass with ProfileGen or SampleUse "
831              " pass");
832       PGOOpt->CSProfileGenFile = CodeGenOpts.InstrProfileOutput.empty()
833                                      ? getDefaultProfileGenName()
834                                      : CodeGenOpts.InstrProfileOutput;
835       PGOOpt->CSAction = PGOOptions::CSIRInstr;
836     } else
837       PGOOpt = PGOOptions("",
838                           CodeGenOpts.InstrProfileOutput.empty()
839                               ? getDefaultProfileGenName()
840                               : CodeGenOpts.InstrProfileOutput,
841                           "", /*MemoryProfile=*/"", nullptr,
842                           PGOOptions::NoAction, PGOOptions::CSIRInstr,
843                           ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling);
844   }
845   if (TM)
846     TM->setPGOOption(PGOOpt);
847 
848   PipelineTuningOptions PTO;
849   PTO.LoopUnrolling = CodeGenOpts.UnrollLoops;
850   // For historical reasons, loop interleaving is set to mirror setting for loop
851   // unrolling.
852   PTO.LoopInterleaving = CodeGenOpts.UnrollLoops;
853   PTO.LoopVectorization = CodeGenOpts.VectorizeLoop;
854   PTO.SLPVectorization = CodeGenOpts.VectorizeSLP;
855   PTO.MergeFunctions = CodeGenOpts.MergeFunctions;
856   // Only enable CGProfilePass when using integrated assembler, since
857   // non-integrated assemblers don't recognize .cgprofile section.
858   PTO.CallGraphProfile = !CodeGenOpts.DisableIntegratedAS;
859   PTO.UnifiedLTO = CodeGenOpts.UnifiedLTO;
860 
861   LoopAnalysisManager LAM;
862   FunctionAnalysisManager FAM;
863   CGSCCAnalysisManager CGAM;
864   ModuleAnalysisManager MAM;
865 
866   bool DebugPassStructure = CodeGenOpts.DebugPass == "Structure";
867   PassInstrumentationCallbacks PIC;
868   PrintPassOptions PrintPassOpts;
869   PrintPassOpts.Indent = DebugPassStructure;
870   PrintPassOpts.SkipAnalyses = DebugPassStructure;
871   StandardInstrumentations SI(
872       TheModule->getContext(),
873       (CodeGenOpts.DebugPassManager || DebugPassStructure),
874       CodeGenOpts.VerifyEach, PrintPassOpts);
875   SI.registerCallbacks(PIC, &MAM);
876   PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC);
877 
878   // Handle the assignment tracking feature options.
879   switch (CodeGenOpts.getAssignmentTrackingMode()) {
880   case CodeGenOptions::AssignmentTrackingOpts::Forced:
881     PB.registerPipelineStartEPCallback(
882         [&](ModulePassManager &MPM, OptimizationLevel Level) {
883           MPM.addPass(AssignmentTrackingPass());
884         });
885     break;
886   case CodeGenOptions::AssignmentTrackingOpts::Enabled:
887     // Disable assignment tracking in LTO builds for now as the performance
888     // cost is too high. Disable for LLDB tuning due to llvm.org/PR43126.
889     if (!CodeGenOpts.PrepareForThinLTO && !CodeGenOpts.PrepareForLTO &&
890         CodeGenOpts.getDebuggerTuning() != llvm::DebuggerKind::LLDB) {
891       PB.registerPipelineStartEPCallback(
892           [&](ModulePassManager &MPM, OptimizationLevel Level) {
893             // Only use assignment tracking if optimisations are enabled.
894             if (Level != OptimizationLevel::O0)
895               MPM.addPass(AssignmentTrackingPass());
896           });
897     }
898     break;
899   case CodeGenOptions::AssignmentTrackingOpts::Disabled:
900     break;
901   }
902 
903   // Enable verify-debuginfo-preserve-each for new PM.
904   DebugifyEachInstrumentation Debugify;
905   DebugInfoPerPass DebugInfoBeforePass;
906   if (CodeGenOpts.EnableDIPreservationVerify) {
907     Debugify.setDebugifyMode(DebugifyMode::OriginalDebugInfo);
908     Debugify.setDebugInfoBeforePass(DebugInfoBeforePass);
909 
910     if (!CodeGenOpts.DIBugsReportFilePath.empty())
911       Debugify.setOrigDIVerifyBugsReportFilePath(
912           CodeGenOpts.DIBugsReportFilePath);
913     Debugify.registerCallbacks(PIC, MAM);
914   }
915   // Attempt to load pass plugins and register their callbacks with PB.
916   for (auto &PluginFN : CodeGenOpts.PassPlugins) {
917     auto PassPlugin = PassPlugin::Load(PluginFN);
918     if (PassPlugin) {
919       PassPlugin->registerPassBuilderCallbacks(PB);
920     } else {
921       Diags.Report(diag::err_fe_unable_to_load_plugin)
922           << PluginFN << toString(PassPlugin.takeError());
923     }
924   }
925   for (const auto &PassCallback : CodeGenOpts.PassBuilderCallbacks)
926     PassCallback(PB);
927 #define HANDLE_EXTENSION(Ext)                                                  \
928   get##Ext##PluginInfo().RegisterPassBuilderCallbacks(PB);
929 #include "llvm/Support/Extension.def"
930 
931   // Register the target library analysis directly and give it a customized
932   // preset TLI.
933   std::unique_ptr<TargetLibraryInfoImpl> TLII(
934       llvm::driver::createTLII(TargetTriple, CodeGenOpts.getVecLib()));
935   FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
936 
937   // Register all the basic analyses with the managers.
938   PB.registerModuleAnalyses(MAM);
939   PB.registerCGSCCAnalyses(CGAM);
940   PB.registerFunctionAnalyses(FAM);
941   PB.registerLoopAnalyses(LAM);
942   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
943 
944   ModulePassManager MPM;
945   // Add a verifier pass, before any other passes, to catch CodeGen issues.
946   if (CodeGenOpts.VerifyModule)
947     MPM.addPass(VerifierPass());
948 
949   if (!CodeGenOpts.DisableLLVMPasses) {
950     // Map our optimization levels into one of the distinct levels used to
951     // configure the pipeline.
952     OptimizationLevel Level = mapToLevel(CodeGenOpts);
953 
954     const bool PrepareForThinLTO = CodeGenOpts.PrepareForThinLTO;
955     const bool PrepareForLTO = CodeGenOpts.PrepareForLTO;
956 
957     if (LangOpts.ObjCAutoRefCount) {
958       PB.registerPipelineStartEPCallback(
959           [](ModulePassManager &MPM, OptimizationLevel Level) {
960             if (Level != OptimizationLevel::O0)
961               MPM.addPass(
962                   createModuleToFunctionPassAdaptor(ObjCARCExpandPass()));
963           });
964       PB.registerPipelineEarlySimplificationEPCallback(
965           [](ModulePassManager &MPM, OptimizationLevel Level) {
966             if (Level != OptimizationLevel::O0)
967               MPM.addPass(ObjCARCAPElimPass());
968           });
969       PB.registerScalarOptimizerLateEPCallback(
970           [](FunctionPassManager &FPM, OptimizationLevel Level) {
971             if (Level != OptimizationLevel::O0)
972               FPM.addPass(ObjCARCOptPass());
973           });
974     }
975 
976     // If we reached here with a non-empty index file name, then the index
977     // file was empty and we are not performing ThinLTO backend compilation
978     // (used in testing in a distributed build environment).
979     bool IsThinLTOPostLink = !CodeGenOpts.ThinLTOIndexFile.empty();
980     // If so drop any the type test assume sequences inserted for whole program
981     // vtables so that codegen doesn't complain.
982     if (IsThinLTOPostLink)
983       PB.registerPipelineStartEPCallback(
984           [](ModulePassManager &MPM, OptimizationLevel Level) {
985             MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr,
986                                            /*ImportSummary=*/nullptr,
987                                            /*DropTypeTests=*/true));
988           });
989 
990     // Register callbacks to schedule sanitizer passes at the appropriate part
991     // of the pipeline.
992     if (LangOpts.Sanitize.has(SanitizerKind::LocalBounds))
993       PB.registerScalarOptimizerLateEPCallback(
994           [](FunctionPassManager &FPM, OptimizationLevel Level) {
995             FPM.addPass(BoundsCheckingPass());
996           });
997 
998     // Don't add sanitizers if we are here from ThinLTO PostLink. That already
999     // done on PreLink stage.
1000     if (!IsThinLTOPostLink) {
1001       addSanitizers(TargetTriple, CodeGenOpts, LangOpts, PB);
1002       addKCFIPass(TargetTriple, LangOpts, PB);
1003     }
1004 
1005     if (std::optional<GCOVOptions> Options =
1006             getGCOVOptions(CodeGenOpts, LangOpts))
1007       PB.registerPipelineStartEPCallback(
1008           [Options](ModulePassManager &MPM, OptimizationLevel Level) {
1009             MPM.addPass(GCOVProfilerPass(*Options));
1010           });
1011     if (std::optional<InstrProfOptions> Options =
1012             getInstrProfOptions(CodeGenOpts, LangOpts))
1013       PB.registerPipelineStartEPCallback(
1014           [Options](ModulePassManager &MPM, OptimizationLevel Level) {
1015             MPM.addPass(InstrProfilingLoweringPass(*Options, false));
1016           });
1017 
1018     // TODO: Consider passing the MemoryProfileOutput to the pass builder via
1019     // the PGOOptions, and set this up there.
1020     if (!CodeGenOpts.MemoryProfileOutput.empty()) {
1021       PB.registerOptimizerLastEPCallback(
1022           [](ModulePassManager &MPM, OptimizationLevel Level) {
1023             MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
1024             MPM.addPass(ModuleMemProfilerPass());
1025           });
1026     }
1027 
1028     if (CodeGenOpts.FatLTO) {
1029       MPM.addPass(PB.buildFatLTODefaultPipeline(
1030           Level, PrepareForThinLTO,
1031           PrepareForThinLTO || shouldEmitRegularLTOSummary()));
1032     } else if (PrepareForThinLTO) {
1033       MPM.addPass(PB.buildThinLTOPreLinkDefaultPipeline(Level));
1034     } else if (PrepareForLTO) {
1035       MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(Level));
1036     } else {
1037       MPM.addPass(PB.buildPerModuleDefaultPipeline(Level));
1038     }
1039   }
1040 
1041   // Link against bitcodes supplied via the -mlink-builtin-bitcode option
1042   if (CodeGenOpts.LinkBitcodePostopt)
1043     MPM.addPass(LinkInModulesPass(BC));
1044 
1045   // Add a verifier pass if requested. We don't have to do this if the action
1046   // requires code generation because there will already be a verifier pass in
1047   // the code-generation pipeline.
1048   // Since we already added a verifier pass above, this
1049   // might even not run the analysis, if previous passes caused no changes.
1050   if (!actionRequiresCodeGen(Action) && CodeGenOpts.VerifyModule)
1051     MPM.addPass(VerifierPass());
1052 
1053   if (Action == Backend_EmitBC || Action == Backend_EmitLL ||
1054       CodeGenOpts.FatLTO) {
1055     if (CodeGenOpts.PrepareForThinLTO && !CodeGenOpts.DisableLLVMPasses) {
1056       if (!TheModule->getModuleFlag("EnableSplitLTOUnit"))
1057         TheModule->addModuleFlag(llvm::Module::Error, "EnableSplitLTOUnit",
1058                                  CodeGenOpts.EnableSplitLTOUnit);
1059       if (Action == Backend_EmitBC) {
1060         if (!CodeGenOpts.ThinLinkBitcodeFile.empty()) {
1061           ThinLinkOS = openOutputFile(CodeGenOpts.ThinLinkBitcodeFile);
1062           if (!ThinLinkOS)
1063             return;
1064         }
1065         MPM.addPass(ThinLTOBitcodeWriterPass(
1066             *OS, ThinLinkOS ? &ThinLinkOS->os() : nullptr));
1067       } else if (Action == Backend_EmitLL) {
1068         MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists,
1069                                     /*EmitLTOSummary=*/true));
1070       }
1071     } else {
1072       // Emit a module summary by default for Regular LTO except for ld64
1073       // targets
1074       bool EmitLTOSummary = shouldEmitRegularLTOSummary();
1075       if (EmitLTOSummary) {
1076         if (!TheModule->getModuleFlag("ThinLTO") && !CodeGenOpts.UnifiedLTO)
1077           TheModule->addModuleFlag(llvm::Module::Error, "ThinLTO", uint32_t(0));
1078         if (!TheModule->getModuleFlag("EnableSplitLTOUnit"))
1079           TheModule->addModuleFlag(llvm::Module::Error, "EnableSplitLTOUnit",
1080                                    uint32_t(1));
1081       }
1082       if (Action == Backend_EmitBC) {
1083         MPM.addPass(BitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists,
1084                                       EmitLTOSummary));
1085       } else if (Action == Backend_EmitLL) {
1086         MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists,
1087                                     EmitLTOSummary));
1088       }
1089     }
1090 
1091     if (shouldEmitUnifiedLTOModueFlag())
1092       TheModule->addModuleFlag(llvm::Module::Error, "UnifiedLTO", uint32_t(1));
1093   }
1094 
1095   // Print a textual, '-passes=' compatible, representation of pipeline if
1096   // requested.
1097   if (PrintPipelinePasses) {
1098     MPM.printPipeline(outs(), [&PIC](StringRef ClassName) {
1099       auto PassName = PIC.getPassNameForClassName(ClassName);
1100       return PassName.empty() ? ClassName : PassName;
1101     });
1102     outs() << "\n";
1103     return;
1104   }
1105 
1106   if (LangOpts.HIPStdPar && !LangOpts.CUDAIsDevice &&
1107       LangOpts.HIPStdParInterposeAlloc)
1108     MPM.addPass(HipStdParAllocationInterpositionPass());
1109 
1110   // Now that we have all of the passes ready, run them.
1111   {
1112     PrettyStackTraceString CrashInfo("Optimizer");
1113     llvm::TimeTraceScope TimeScope("Optimizer");
1114     MPM.run(*TheModule, MAM);
1115   }
1116 }
1117 
1118 void EmitAssemblyHelper::RunCodegenPipeline(
1119     BackendAction Action, std::unique_ptr<raw_pwrite_stream> &OS,
1120     std::unique_ptr<llvm::ToolOutputFile> &DwoOS) {
1121   // We still use the legacy PM to run the codegen pipeline since the new PM
1122   // does not work with the codegen pipeline.
1123   // FIXME: make the new PM work with the codegen pipeline.
1124   legacy::PassManager CodeGenPasses;
1125 
1126   // Append any output we need to the pass manager.
1127   switch (Action) {
1128   case Backend_EmitAssembly:
1129   case Backend_EmitMCNull:
1130   case Backend_EmitObj:
1131     CodeGenPasses.add(
1132         createTargetTransformInfoWrapperPass(getTargetIRAnalysis()));
1133     if (!CodeGenOpts.SplitDwarfOutput.empty()) {
1134       DwoOS = openOutputFile(CodeGenOpts.SplitDwarfOutput);
1135       if (!DwoOS)
1136         return;
1137     }
1138     if (!AddEmitPasses(CodeGenPasses, Action, *OS,
1139                        DwoOS ? &DwoOS->os() : nullptr))
1140       // FIXME: Should we handle this error differently?
1141       return;
1142     break;
1143   default:
1144     return;
1145   }
1146 
1147   // If -print-pipeline-passes is requested, don't run the legacy pass manager.
1148   // FIXME: when codegen is switched to use the new pass manager, it should also
1149   // emit pass names here.
1150   if (PrintPipelinePasses) {
1151     return;
1152   }
1153 
1154   {
1155     PrettyStackTraceString CrashInfo("Code generation");
1156     llvm::TimeTraceScope TimeScope("CodeGenPasses");
1157     CodeGenPasses.run(*TheModule);
1158   }
1159 }
1160 
1161 void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
1162                                       std::unique_ptr<raw_pwrite_stream> OS,
1163                                       BackendConsumer *BC) {
1164   TimeRegion Region(CodeGenOpts.TimePasses ? &CodeGenerationTime : nullptr);
1165   setCommandLineOpts(CodeGenOpts);
1166 
1167   bool RequiresCodeGen = actionRequiresCodeGen(Action);
1168   CreateTargetMachine(RequiresCodeGen);
1169 
1170   if (RequiresCodeGen && !TM)
1171     return;
1172   if (TM)
1173     TheModule->setDataLayout(TM->createDataLayout());
1174 
1175   // Before executing passes, print the final values of the LLVM options.
1176   cl::PrintOptionValues();
1177 
1178   std::unique_ptr<llvm::ToolOutputFile> ThinLinkOS, DwoOS;
1179   RunOptimizationPipeline(Action, OS, ThinLinkOS, BC);
1180   RunCodegenPipeline(Action, OS, DwoOS);
1181 
1182   if (ThinLinkOS)
1183     ThinLinkOS->keep();
1184   if (DwoOS)
1185     DwoOS->keep();
1186 }
1187 
1188 static void runThinLTOBackend(
1189     DiagnosticsEngine &Diags, ModuleSummaryIndex *CombinedIndex,
1190     llvm::Module *M, const HeaderSearchOptions &HeaderOpts,
1191     const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts,
1192     const LangOptions &LOpts, std::unique_ptr<raw_pwrite_stream> OS,
1193     std::string SampleProfile, std::string ProfileRemapping,
1194     BackendAction Action) {
1195   DenseMap<StringRef, DenseMap<GlobalValue::GUID, GlobalValueSummary *>>
1196       ModuleToDefinedGVSummaries;
1197   CombinedIndex->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
1198 
1199   setCommandLineOpts(CGOpts);
1200 
1201   // We can simply import the values mentioned in the combined index, since
1202   // we should only invoke this using the individual indexes written out
1203   // via a WriteIndexesThinBackend.
1204   FunctionImporter::ImportMapTy ImportList;
1205   if (!lto::initImportList(*M, *CombinedIndex, ImportList))
1206     return;
1207 
1208   auto AddStream = [&](size_t Task, const Twine &ModuleName) {
1209     return std::make_unique<CachedFileStream>(std::move(OS),
1210                                               CGOpts.ObjectFilenameForDebug);
1211   };
1212   lto::Config Conf;
1213   if (CGOpts.SaveTempsFilePrefix != "") {
1214     if (Error E = Conf.addSaveTemps(CGOpts.SaveTempsFilePrefix + ".",
1215                                     /* UseInputModulePath */ false)) {
1216       handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
1217         errs() << "Error setting up ThinLTO save-temps: " << EIB.message()
1218                << '\n';
1219       });
1220     }
1221   }
1222   Conf.CPU = TOpts.CPU;
1223   Conf.CodeModel = getCodeModel(CGOpts);
1224   Conf.MAttrs = TOpts.Features;
1225   Conf.RelocModel = CGOpts.RelocationModel;
1226   std::optional<CodeGenOptLevel> OptLevelOrNone =
1227       CodeGenOpt::getLevel(CGOpts.OptimizationLevel);
1228   assert(OptLevelOrNone && "Invalid optimization level!");
1229   Conf.CGOptLevel = *OptLevelOrNone;
1230   Conf.OptLevel = CGOpts.OptimizationLevel;
1231   initTargetOptions(Diags, Conf.Options, CGOpts, TOpts, LOpts, HeaderOpts);
1232   Conf.SampleProfile = std::move(SampleProfile);
1233   Conf.PTO.LoopUnrolling = CGOpts.UnrollLoops;
1234   // For historical reasons, loop interleaving is set to mirror setting for loop
1235   // unrolling.
1236   Conf.PTO.LoopInterleaving = CGOpts.UnrollLoops;
1237   Conf.PTO.LoopVectorization = CGOpts.VectorizeLoop;
1238   Conf.PTO.SLPVectorization = CGOpts.VectorizeSLP;
1239   // Only enable CGProfilePass when using integrated assembler, since
1240   // non-integrated assemblers don't recognize .cgprofile section.
1241   Conf.PTO.CallGraphProfile = !CGOpts.DisableIntegratedAS;
1242 
1243   // Context sensitive profile.
1244   if (CGOpts.hasProfileCSIRInstr()) {
1245     Conf.RunCSIRInstr = true;
1246     Conf.CSIRProfile = std::move(CGOpts.InstrProfileOutput);
1247   } else if (CGOpts.hasProfileCSIRUse()) {
1248     Conf.RunCSIRInstr = false;
1249     Conf.CSIRProfile = std::move(CGOpts.ProfileInstrumentUsePath);
1250   }
1251 
1252   Conf.ProfileRemapping = std::move(ProfileRemapping);
1253   Conf.DebugPassManager = CGOpts.DebugPassManager;
1254   Conf.VerifyEach = CGOpts.VerifyEach;
1255   Conf.RemarksWithHotness = CGOpts.DiagnosticsWithHotness;
1256   Conf.RemarksFilename = CGOpts.OptRecordFile;
1257   Conf.RemarksPasses = CGOpts.OptRecordPasses;
1258   Conf.RemarksFormat = CGOpts.OptRecordFormat;
1259   Conf.SplitDwarfFile = CGOpts.SplitDwarfFile;
1260   Conf.SplitDwarfOutput = CGOpts.SplitDwarfOutput;
1261   switch (Action) {
1262   case Backend_EmitNothing:
1263     Conf.PreCodeGenModuleHook = [](size_t Task, const llvm::Module &Mod) {
1264       return false;
1265     };
1266     break;
1267   case Backend_EmitLL:
1268     Conf.PreCodeGenModuleHook = [&](size_t Task, const llvm::Module &Mod) {
1269       M->print(*OS, nullptr, CGOpts.EmitLLVMUseLists);
1270       return false;
1271     };
1272     break;
1273   case Backend_EmitBC:
1274     Conf.PreCodeGenModuleHook = [&](size_t Task, const llvm::Module &Mod) {
1275       WriteBitcodeToFile(*M, *OS, CGOpts.EmitLLVMUseLists);
1276       return false;
1277     };
1278     break;
1279   default:
1280     Conf.CGFileType = getCodeGenFileType(Action);
1281     break;
1282   }
1283   if (Error E =
1284           thinBackend(Conf, -1, AddStream, *M, *CombinedIndex, ImportList,
1285                       ModuleToDefinedGVSummaries[M->getModuleIdentifier()],
1286                       /* ModuleMap */ nullptr, CGOpts.CmdArgs)) {
1287     handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) {
1288       errs() << "Error running ThinLTO backend: " << EIB.message() << '\n';
1289     });
1290   }
1291 }
1292 
1293 void clang::EmitBackendOutput(
1294     DiagnosticsEngine &Diags, const HeaderSearchOptions &HeaderOpts,
1295     const CodeGenOptions &CGOpts, const clang::TargetOptions &TOpts,
1296     const LangOptions &LOpts, StringRef TDesc, llvm::Module *M,
1297     BackendAction Action, IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
1298     std::unique_ptr<raw_pwrite_stream> OS, BackendConsumer *BC) {
1299 
1300   llvm::TimeTraceScope TimeScope("Backend");
1301 
1302   std::unique_ptr<llvm::Module> EmptyModule;
1303   if (!CGOpts.ThinLTOIndexFile.empty()) {
1304     // If we are performing a ThinLTO importing compile, load the function index
1305     // into memory and pass it into runThinLTOBackend, which will run the
1306     // function importer and invoke LTO passes.
1307     std::unique_ptr<ModuleSummaryIndex> CombinedIndex;
1308     if (Error E = llvm::getModuleSummaryIndexForFile(
1309                       CGOpts.ThinLTOIndexFile,
1310                       /*IgnoreEmptyThinLTOIndexFile*/ true)
1311                       .moveInto(CombinedIndex)) {
1312       logAllUnhandledErrors(std::move(E), errs(),
1313                             "Error loading index file '" +
1314                             CGOpts.ThinLTOIndexFile + "': ");
1315       return;
1316     }
1317 
1318     // A null CombinedIndex means we should skip ThinLTO compilation
1319     // (LLVM will optionally ignore empty index files, returning null instead
1320     // of an error).
1321     if (CombinedIndex) {
1322       if (!CombinedIndex->skipModuleByDistributedBackend()) {
1323         runThinLTOBackend(Diags, CombinedIndex.get(), M, HeaderOpts, CGOpts,
1324                           TOpts, LOpts, std::move(OS), CGOpts.SampleProfileFile,
1325                           CGOpts.ProfileRemappingFile, Action);
1326         return;
1327       }
1328       // Distributed indexing detected that nothing from the module is needed
1329       // for the final linking. So we can skip the compilation. We sill need to
1330       // output an empty object file to make sure that a linker does not fail
1331       // trying to read it. Also for some features, like CFI, we must skip
1332       // the compilation as CombinedIndex does not contain all required
1333       // information.
1334       EmptyModule = std::make_unique<llvm::Module>("empty", M->getContext());
1335       EmptyModule->setTargetTriple(M->getTargetTriple());
1336       M = EmptyModule.get();
1337     }
1338   }
1339 
1340   EmitAssemblyHelper AsmHelper(Diags, HeaderOpts, CGOpts, TOpts, LOpts, M, VFS);
1341   AsmHelper.EmitAssembly(Action, std::move(OS), BC);
1342 
1343   // Verify clang's TargetInfo DataLayout against the LLVM TargetMachine's
1344   // DataLayout.
1345   if (AsmHelper.TM) {
1346     std::string DLDesc = M->getDataLayout().getStringRepresentation();
1347     if (DLDesc != TDesc) {
1348       unsigned DiagID = Diags.getCustomDiagID(
1349           DiagnosticsEngine::Error, "backend data layout '%0' does not match "
1350                                     "expected target description '%1'");
1351       Diags.Report(DiagID) << DLDesc << TDesc;
1352     }
1353   }
1354 }
1355 
1356 // With -fembed-bitcode, save a copy of the llvm IR as data in the
1357 // __LLVM,__bitcode section.
1358 void clang::EmbedBitcode(llvm::Module *M, const CodeGenOptions &CGOpts,
1359                          llvm::MemoryBufferRef Buf) {
1360   if (CGOpts.getEmbedBitcode() == CodeGenOptions::Embed_Off)
1361     return;
1362   llvm::embedBitcodeInModule(
1363       *M, Buf, CGOpts.getEmbedBitcode() != CodeGenOptions::Embed_Marker,
1364       CGOpts.getEmbedBitcode() != CodeGenOptions::Embed_Bitcode,
1365       CGOpts.CmdArgs);
1366 }
1367 
1368 void clang::EmbedObject(llvm::Module *M, const CodeGenOptions &CGOpts,
1369                         DiagnosticsEngine &Diags) {
1370   if (CGOpts.OffloadObjects.empty())
1371     return;
1372 
1373   for (StringRef OffloadObject : CGOpts.OffloadObjects) {
1374     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> ObjectOrErr =
1375         llvm::MemoryBuffer::getFileOrSTDIN(OffloadObject);
1376     if (ObjectOrErr.getError()) {
1377       auto DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
1378                                           "could not open '%0' for embedding");
1379       Diags.Report(DiagID) << OffloadObject;
1380       return;
1381     }
1382 
1383     llvm::embedBufferInModule(*M, **ObjectOrErr, ".llvm.offloading",
1384                               Align(object::OffloadBinary::getAlignment()));
1385   }
1386 }
1387